diff --git a/owl-bot-staging/v1/.coveragerc b/owl-bot-staging/v1/.coveragerc new file mode 100644 index 00000000..23d2208d --- /dev/null +++ b/owl-bot-staging/v1/.coveragerc @@ -0,0 +1,13 @@ +[run] +branch = True + +[report] +show_missing = True +omit = + google/cloud/bigquery_storage/__init__.py + google/cloud/bigquery_storage/gapic_version.py +exclude_lines = + # Re-enable the standard pragma + pragma: NO COVER + # Ignore debug-only repr + def __repr__ diff --git a/owl-bot-staging/v1/.flake8 b/owl-bot-staging/v1/.flake8 new file mode 100644 index 00000000..29227d4c --- /dev/null +++ b/owl-bot-staging/v1/.flake8 @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! +[flake8] +ignore = E203, E266, E501, W503 +exclude = + # Exclude generated code. + **/proto/** + **/gapic/** + **/services/** + **/types/** + *_pb2.py + + # Standard linting exemptions. + **/.nox/** + __pycache__, + .git, + *.pyc, + conf.py diff --git a/owl-bot-staging/v1/MANIFEST.in b/owl-bot-staging/v1/MANIFEST.in new file mode 100644 index 00000000..88a0105e --- /dev/null +++ b/owl-bot-staging/v1/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include google/cloud/bigquery_storage *.py +recursive-include google/cloud/bigquery_storage_v1 *.py diff --git a/owl-bot-staging/v1/README.rst b/owl-bot-staging/v1/README.rst new file mode 100644 index 00000000..13a45c14 --- /dev/null +++ b/owl-bot-staging/v1/README.rst @@ -0,0 +1,49 @@ +Python Client for Google Cloud Bigquery Storage API +================================================= + +Quick Start +----------- + +In order to use this library, you first need to go through the following steps: + +1. `Select or create a Cloud Platform project.`_ +2. `Enable billing for your project.`_ +3. Enable the Google Cloud Bigquery Storage API. +4. `Setup Authentication.`_ + +.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project +.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project +.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html + +Installation +~~~~~~~~~~~~ + +Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to +create isolated Python environments. The basic problem it addresses is one of +dependencies and versions, and indirectly permissions. + +With `virtualenv`_, it's possible to install this library without needing system +install permissions, and without clashing with the installed system +dependencies. + +.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ + + +Mac/Linux +^^^^^^^^^ + +.. code-block:: console + + python3 -m venv + source /bin/activate + /bin/pip install /path/to/library + + +Windows +^^^^^^^ + +.. code-block:: console + + python3 -m venv + \Scripts\activate + \Scripts\pip.exe install \path\to\library diff --git a/owl-bot-staging/v1/docs/_static/custom.css b/owl-bot-staging/v1/docs/_static/custom.css new file mode 100644 index 00000000..06423be0 --- /dev/null +++ b/owl-bot-staging/v1/docs/_static/custom.css @@ -0,0 +1,3 @@ +dl.field-list > dt { + min-width: 100px +} diff --git a/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_read.rst b/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_read.rst new file mode 100644 index 00000000..74f0a5fa --- /dev/null +++ b/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_read.rst @@ -0,0 +1,6 @@ +BigQueryRead +------------------------------ + +.. automodule:: google.cloud.bigquery_storage_v1.services.big_query_read + :members: + :inherited-members: diff --git a/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_write.rst b/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_write.rst new file mode 100644 index 00000000..e0c869ca --- /dev/null +++ b/owl-bot-staging/v1/docs/bigquery_storage_v1/big_query_write.rst @@ -0,0 +1,6 @@ +BigQueryWrite +------------------------------- + +.. automodule:: google.cloud.bigquery_storage_v1.services.big_query_write + :members: + :inherited-members: diff --git a/owl-bot-staging/v1/docs/bigquery_storage_v1/services_.rst b/owl-bot-staging/v1/docs/bigquery_storage_v1/services_.rst new file mode 100644 index 00000000..50f35459 --- /dev/null +++ b/owl-bot-staging/v1/docs/bigquery_storage_v1/services_.rst @@ -0,0 +1,7 @@ +Services for Google Cloud Bigquery Storage v1 API +================================================= +.. toctree:: + :maxdepth: 2 + + big_query_read + big_query_write diff --git a/owl-bot-staging/v1/docs/bigquery_storage_v1/types_.rst b/owl-bot-staging/v1/docs/bigquery_storage_v1/types_.rst new file mode 100644 index 00000000..3f722c57 --- /dev/null +++ b/owl-bot-staging/v1/docs/bigquery_storage_v1/types_.rst @@ -0,0 +1,6 @@ +Types for Google Cloud Bigquery Storage v1 API +============================================== + +.. automodule:: google.cloud.bigquery_storage_v1.types + :members: + :show-inheritance: diff --git a/owl-bot-staging/v1/docs/conf.py b/owl-bot-staging/v1/docs/conf.py new file mode 100644 index 00000000..33c4ea53 --- /dev/null +++ b/owl-bot-staging/v1/docs/conf.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# google-cloud-bigquery-storage documentation build configuration file +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath("..")) + +__version__ = "0.1.0" + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +needs_sphinx = "4.0.1" + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.viewcode", +] + +# autodoc/autosummary flags +autoclass_content = "both" +autodoc_default_flags = ["members"] +autosummary_generate = True + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# Allow markdown includes (so releases.md can include CHANGLEOG.md) +# http://www.sphinx-doc.org/en/master/markdown.html +source_parsers = {".md": "recommonmark.parser.CommonMarkParser"} + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +source_suffix = [".rst", ".md"] + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The root toctree document. +root_doc = "index" + +# General information about the project. +project = u"google-cloud-bigquery-storage" +copyright = u"2023, Google, LLC" +author = u"Google APIs" # TODO: autogenerate this bit + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The full version, including alpha/beta/rc tags. +release = __version__ +# The short X.Y version. +version = ".".join(release.split(".")[0:2]) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ["_build"] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "alabaster" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + "description": "Google Cloud Client Libraries for Python", + "github_user": "googleapis", + "github_repo": "google-cloud-python", + "github_banner": True, + "font_family": "'Roboto', Georgia, sans", + "head_font_family": "'Roboto', Georgia, serif", + "code_font_family": "'Roboto Mono', 'Consolas', monospace", +} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = "google-cloud-bigquery-storage-doc" + +# -- Options for warnings ------------------------------------------------------ + + +suppress_warnings = [ + # Temporarily suppress this to avoid "more than one target found for + # cross-reference" warning, which are intractable for us to avoid while in + # a mono-repo. + # See https://github.com/sphinx-doc/sphinx/blob + # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843 + "ref.python" +] + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', + # Latex figure (float) alignment + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + root_doc, + "google-cloud-bigquery-storage.tex", + u"google-cloud-bigquery-storage Documentation", + author, + "manual", + ) +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ( + root_doc, + "google-cloud-bigquery-storage", + u"Google Cloud Bigquery Storage Documentation", + [author], + 1, + ) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + root_doc, + "google-cloud-bigquery-storage", + u"google-cloud-bigquery-storage Documentation", + author, + "google-cloud-bigquery-storage", + "GAPIC library for Google Cloud Bigquery Storage API", + "APIs", + ) +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + "python": ("http://python.readthedocs.org/en/latest/", None), + "gax": ("https://gax-python.readthedocs.org/en/latest/", None), + "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), + "google-gax": ("https://gax-python.readthedocs.io/en/latest/", None), + "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None), + "grpc": ("https://grpc.io/grpc/python/", None), + "requests": ("http://requests.kennethreitz.org/en/stable/", None), + "proto": ("https://proto-plus-python.readthedocs.io/en/stable", None), + "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), +} + + +# Napoleon settings +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = True +napoleon_use_admonition_for_examples = False +napoleon_use_admonition_for_notes = False +napoleon_use_admonition_for_references = False +napoleon_use_ivar = False +napoleon_use_param = True +napoleon_use_rtype = True diff --git a/owl-bot-staging/v1/docs/index.rst b/owl-bot-staging/v1/docs/index.rst new file mode 100644 index 00000000..a5e3a707 --- /dev/null +++ b/owl-bot-staging/v1/docs/index.rst @@ -0,0 +1,7 @@ +API Reference +------------- +.. toctree:: + :maxdepth: 2 + + bigquery_storage_v1/services + bigquery_storage_v1/types diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage/__init__.py new file mode 100644 index 00000000..1abac372 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage/__init__.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from google.cloud.bigquery_storage import gapic_version as package_version + +__version__ = package_version.__version__ + + +from google.cloud.bigquery_storage_v1.services.big_query_read.client import BigQueryReadClient +from google.cloud.bigquery_storage_v1.services.big_query_read.async_client import BigQueryReadAsyncClient +from google.cloud.bigquery_storage_v1.services.big_query_write.client import BigQueryWriteClient +from google.cloud.bigquery_storage_v1.services.big_query_write.async_client import BigQueryWriteAsyncClient + +from google.cloud.bigquery_storage_v1.types.arrow import ArrowRecordBatch +from google.cloud.bigquery_storage_v1.types.arrow import ArrowSchema +from google.cloud.bigquery_storage_v1.types.arrow import ArrowSerializationOptions +from google.cloud.bigquery_storage_v1.types.avro import AvroRows +from google.cloud.bigquery_storage_v1.types.avro import AvroSchema +from google.cloud.bigquery_storage_v1.types.avro import AvroSerializationOptions +from google.cloud.bigquery_storage_v1.types.protobuf import ProtoRows +from google.cloud.bigquery_storage_v1.types.protobuf import ProtoSchema +from google.cloud.bigquery_storage_v1.types.storage import AppendRowsRequest +from google.cloud.bigquery_storage_v1.types.storage import AppendRowsResponse +from google.cloud.bigquery_storage_v1.types.storage import BatchCommitWriteStreamsRequest +from google.cloud.bigquery_storage_v1.types.storage import BatchCommitWriteStreamsResponse +from google.cloud.bigquery_storage_v1.types.storage import CreateReadSessionRequest +from google.cloud.bigquery_storage_v1.types.storage import CreateWriteStreamRequest +from google.cloud.bigquery_storage_v1.types.storage import FinalizeWriteStreamRequest +from google.cloud.bigquery_storage_v1.types.storage import FinalizeWriteStreamResponse +from google.cloud.bigquery_storage_v1.types.storage import FlushRowsRequest +from google.cloud.bigquery_storage_v1.types.storage import FlushRowsResponse +from google.cloud.bigquery_storage_v1.types.storage import GetWriteStreamRequest +from google.cloud.bigquery_storage_v1.types.storage import ReadRowsRequest +from google.cloud.bigquery_storage_v1.types.storage import ReadRowsResponse +from google.cloud.bigquery_storage_v1.types.storage import RowError +from google.cloud.bigquery_storage_v1.types.storage import SplitReadStreamRequest +from google.cloud.bigquery_storage_v1.types.storage import SplitReadStreamResponse +from google.cloud.bigquery_storage_v1.types.storage import StorageError +from google.cloud.bigquery_storage_v1.types.storage import StreamStats +from google.cloud.bigquery_storage_v1.types.storage import ThrottleState +from google.cloud.bigquery_storage_v1.types.stream import ReadSession +from google.cloud.bigquery_storage_v1.types.stream import ReadStream +from google.cloud.bigquery_storage_v1.types.stream import WriteStream +from google.cloud.bigquery_storage_v1.types.stream import DataFormat +from google.cloud.bigquery_storage_v1.types.stream import WriteStreamView +from google.cloud.bigquery_storage_v1.types.table import TableFieldSchema +from google.cloud.bigquery_storage_v1.types.table import TableSchema + +__all__ = ('BigQueryReadClient', + 'BigQueryReadAsyncClient', + 'BigQueryWriteClient', + 'BigQueryWriteAsyncClient', + 'ArrowRecordBatch', + 'ArrowSchema', + 'ArrowSerializationOptions', + 'AvroRows', + 'AvroSchema', + 'AvroSerializationOptions', + 'ProtoRows', + 'ProtoSchema', + 'AppendRowsRequest', + 'AppendRowsResponse', + 'BatchCommitWriteStreamsRequest', + 'BatchCommitWriteStreamsResponse', + 'CreateReadSessionRequest', + 'CreateWriteStreamRequest', + 'FinalizeWriteStreamRequest', + 'FinalizeWriteStreamResponse', + 'FlushRowsRequest', + 'FlushRowsResponse', + 'GetWriteStreamRequest', + 'ReadRowsRequest', + 'ReadRowsResponse', + 'RowError', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + 'StorageError', + 'StreamStats', + 'ThrottleState', + 'ReadSession', + 'ReadStream', + 'WriteStream', + 'DataFormat', + 'WriteStreamView', + 'TableFieldSchema', + 'TableSchema', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage/gapic_version.py b/owl-bot-staging/v1/google/cloud/bigquery_storage/gapic_version.py new file mode 100644 index 00000000..558c8aab --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage/gapic_version.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__version__ = "0.0.0" # {x-release-please-version} diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage/py.typed b/owl-bot-staging/v1/google/cloud/bigquery_storage/py.typed new file mode 100644 index 00000000..e71b4749 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/__init__.py new file mode 100644 index 00000000..698c2854 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/__init__.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from google.cloud.bigquery_storage_v1 import gapic_version as package_version + +__version__ = package_version.__version__ + + +from .services.big_query_read import BigQueryReadClient +from .services.big_query_read import BigQueryReadAsyncClient +from .services.big_query_write import BigQueryWriteClient +from .services.big_query_write import BigQueryWriteAsyncClient + +from .types.arrow import ArrowRecordBatch +from .types.arrow import ArrowSchema +from .types.arrow import ArrowSerializationOptions +from .types.avro import AvroRows +from .types.avro import AvroSchema +from .types.avro import AvroSerializationOptions +from .types.protobuf import ProtoRows +from .types.protobuf import ProtoSchema +from .types.storage import AppendRowsRequest +from .types.storage import AppendRowsResponse +from .types.storage import BatchCommitWriteStreamsRequest +from .types.storage import BatchCommitWriteStreamsResponse +from .types.storage import CreateReadSessionRequest +from .types.storage import CreateWriteStreamRequest +from .types.storage import FinalizeWriteStreamRequest +from .types.storage import FinalizeWriteStreamResponse +from .types.storage import FlushRowsRequest +from .types.storage import FlushRowsResponse +from .types.storage import GetWriteStreamRequest +from .types.storage import ReadRowsRequest +from .types.storage import ReadRowsResponse +from .types.storage import RowError +from .types.storage import SplitReadStreamRequest +from .types.storage import SplitReadStreamResponse +from .types.storage import StorageError +from .types.storage import StreamStats +from .types.storage import ThrottleState +from .types.stream import ReadSession +from .types.stream import ReadStream +from .types.stream import WriteStream +from .types.stream import DataFormat +from .types.stream import WriteStreamView +from .types.table import TableFieldSchema +from .types.table import TableSchema + +__all__ = ( + 'BigQueryReadAsyncClient', + 'BigQueryWriteAsyncClient', +'AppendRowsRequest', +'AppendRowsResponse', +'ArrowRecordBatch', +'ArrowSchema', +'ArrowSerializationOptions', +'AvroRows', +'AvroSchema', +'AvroSerializationOptions', +'BatchCommitWriteStreamsRequest', +'BatchCommitWriteStreamsResponse', +'BigQueryReadClient', +'BigQueryWriteClient', +'CreateReadSessionRequest', +'CreateWriteStreamRequest', +'DataFormat', +'FinalizeWriteStreamRequest', +'FinalizeWriteStreamResponse', +'FlushRowsRequest', +'FlushRowsResponse', +'GetWriteStreamRequest', +'ProtoRows', +'ProtoSchema', +'ReadRowsRequest', +'ReadRowsResponse', +'ReadSession', +'ReadStream', +'RowError', +'SplitReadStreamRequest', +'SplitReadStreamResponse', +'StorageError', +'StreamStats', +'TableFieldSchema', +'TableSchema', +'ThrottleState', +'WriteStream', +'WriteStreamView', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_metadata.json b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_metadata.json new file mode 100644 index 00000000..4fc72131 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_metadata.json @@ -0,0 +1,127 @@ + { + "comment": "This file maps proto services/RPCs to the corresponding library clients/methods", + "language": "python", + "libraryPackage": "google.cloud.bigquery_storage_v1", + "protoPackage": "google.cloud.bigquery.storage.v1", + "schema": "1.0", + "services": { + "BigQueryRead": { + "clients": { + "grpc": { + "libraryClient": "BigQueryReadClient", + "rpcs": { + "CreateReadSession": { + "methods": [ + "create_read_session" + ] + }, + "ReadRows": { + "methods": [ + "read_rows" + ] + }, + "SplitReadStream": { + "methods": [ + "split_read_stream" + ] + } + } + }, + "grpc-async": { + "libraryClient": "BigQueryReadAsyncClient", + "rpcs": { + "CreateReadSession": { + "methods": [ + "create_read_session" + ] + }, + "ReadRows": { + "methods": [ + "read_rows" + ] + }, + "SplitReadStream": { + "methods": [ + "split_read_stream" + ] + } + } + } + } + }, + "BigQueryWrite": { + "clients": { + "grpc": { + "libraryClient": "BigQueryWriteClient", + "rpcs": { + "AppendRows": { + "methods": [ + "append_rows" + ] + }, + "BatchCommitWriteStreams": { + "methods": [ + "batch_commit_write_streams" + ] + }, + "CreateWriteStream": { + "methods": [ + "create_write_stream" + ] + }, + "FinalizeWriteStream": { + "methods": [ + "finalize_write_stream" + ] + }, + "FlushRows": { + "methods": [ + "flush_rows" + ] + }, + "GetWriteStream": { + "methods": [ + "get_write_stream" + ] + } + } + }, + "grpc-async": { + "libraryClient": "BigQueryWriteAsyncClient", + "rpcs": { + "AppendRows": { + "methods": [ + "append_rows" + ] + }, + "BatchCommitWriteStreams": { + "methods": [ + "batch_commit_write_streams" + ] + }, + "CreateWriteStream": { + "methods": [ + "create_write_stream" + ] + }, + "FinalizeWriteStream": { + "methods": [ + "finalize_write_stream" + ] + }, + "FlushRows": { + "methods": [ + "flush_rows" + ] + }, + "GetWriteStream": { + "methods": [ + "get_write_stream" + ] + } + } + } + } + } + } +} diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_version.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_version.py new file mode 100644 index 00000000..558c8aab --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/gapic_version.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__version__ = "0.0.0" # {x-release-please-version} diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/py.typed b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/py.typed new file mode 100644 index 00000000..e71b4749 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/__init__.py new file mode 100644 index 00000000..8f6cf068 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/__init__.py new file mode 100644 index 00000000..194280aa --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .client import BigQueryReadClient +from .async_client import BigQueryReadAsyncClient + +__all__ = ( + 'BigQueryReadClient', + 'BigQueryReadAsyncClient', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py new file mode 100644 index 00000000..d5925ada --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/async_client.py @@ -0,0 +1,612 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import re +from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, AsyncIterable, Awaitable, Sequence, Tuple, Type, Union + +from google.cloud.bigquery_storage_v1 import gapic_version as package_version + +from google.api_core.client_options import ClientOptions +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry_async as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + + +try: + OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.AsyncRetry, object, None] # type: ignore + +from google.cloud.bigquery_storage_v1.types import arrow +from google.cloud.bigquery_storage_v1.types import avro +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.protobuf import timestamp_pb2 # type: ignore +from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport +from .client import BigQueryReadClient + + +class BigQueryReadAsyncClient: + """BigQuery Read API. + + The Read API can be used to read data from BigQuery. + """ + + _client: BigQueryReadClient + + # Copy defaults from the synchronous client for use here. + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + DEFAULT_ENDPOINT = BigQueryReadClient.DEFAULT_ENDPOINT + DEFAULT_MTLS_ENDPOINT = BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + _DEFAULT_ENDPOINT_TEMPLATE = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE + _DEFAULT_UNIVERSE = BigQueryReadClient._DEFAULT_UNIVERSE + + read_session_path = staticmethod(BigQueryReadClient.read_session_path) + parse_read_session_path = staticmethod(BigQueryReadClient.parse_read_session_path) + read_stream_path = staticmethod(BigQueryReadClient.read_stream_path) + parse_read_stream_path = staticmethod(BigQueryReadClient.parse_read_stream_path) + table_path = staticmethod(BigQueryReadClient.table_path) + parse_table_path = staticmethod(BigQueryReadClient.parse_table_path) + common_billing_account_path = staticmethod(BigQueryReadClient.common_billing_account_path) + parse_common_billing_account_path = staticmethod(BigQueryReadClient.parse_common_billing_account_path) + common_folder_path = staticmethod(BigQueryReadClient.common_folder_path) + parse_common_folder_path = staticmethod(BigQueryReadClient.parse_common_folder_path) + common_organization_path = staticmethod(BigQueryReadClient.common_organization_path) + parse_common_organization_path = staticmethod(BigQueryReadClient.parse_common_organization_path) + common_project_path = staticmethod(BigQueryReadClient.common_project_path) + parse_common_project_path = staticmethod(BigQueryReadClient.parse_common_project_path) + common_location_path = staticmethod(BigQueryReadClient.common_location_path) + parse_common_location_path = staticmethod(BigQueryReadClient.parse_common_location_path) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_info.__func__(BigQueryReadAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_file.__func__(BigQueryReadAsyncClient, filename, *args, **kwargs) # type: ignore + + from_service_account_json = from_service_account_file + + @classmethod + def get_mtls_endpoint_and_cert_source(cls, client_options: Optional[ClientOptions] = None): + """Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variable is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + return BigQueryReadClient.get_mtls_endpoint_and_cert_source(client_options) # type: ignore + + @property + def transport(self) -> BigQueryReadTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryReadTransport: The transport used by the client instance. + """ + return self._client.transport + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._client._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used + by the client instance. + """ + return self._client._universe_domain + + get_transport_class = BigQueryReadClient.get_transport_class + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Optional[Union[str, BigQueryReadTransport, Callable[..., BigQueryReadTransport]]] = "grpc_asyncio", + client_options: Optional[ClientOptions] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query read async client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Optional[Union[str,BigQueryReadTransport,Callable[..., BigQueryReadTransport]]]): + The transport to use, or a Callable that constructs and returns a new transport to use. + If a Callable is given, it will be called with the same set of initialization + arguments as used in the BigQueryReadTransport constructor. + If set to None, a transport is chosen automatically. + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide a client certificate for mTLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client = BigQueryReadClient( + credentials=credentials, + transport=transport, + client_options=client_options, + client_info=client_info, + + ) + + async def create_read_session(self, + request: Optional[Union[storage.CreateReadSessionRequest, dict]] = None, + *, + parent: Optional[str] = None, + read_session: Optional[stream.ReadSession] = None, + max_stream_count: Optional[int] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.ReadSession: + r"""Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a + push-down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + async def sample_create_read_session(): + # Create a client + client = bigquery_storage_v1.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.CreateReadSessionRequest( + parent="parent_value", + ) + + # Make the request + response = await client.create_read_session(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1.types.CreateReadSessionRequest, dict]]): + The request object. Request message for ``CreateReadSession``. + parent (:class:`str`): + Required. The request project that owns the session, in + the form of ``projects/{project_id}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + read_session (:class:`google.cloud.bigquery_storage_v1.types.ReadSession`): + Required. Session to be created. + This corresponds to the ``read_session`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + max_stream_count (:class:`int`): + Max initial number of streams. If unset or zero, the + server will provide a value of streams so as to produce + reasonable throughput. Must be non-negative. The number + of streams may be lower than the requested number, + depending on the amount parallelism that is reasonable + for the table. There is a default system max limit of + 1,000. + + This must be greater than or equal to + preferred_min_stream_count. Typically, clients should + either leave this unset to let the system to determine + an upper bound OR set this a size for the maximum "units + of work" it can gracefully handle. + + This corresponds to the ``max_stream_count`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.ReadSession: + Information about the ReadSession. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, read_session, max_stream_count]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.CreateReadSessionRequest): + request = storage.CreateReadSessionRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if read_session is not None: + request.read_session = read_session + if max_stream_count is not None: + request.max_stream_count = max_stream_count + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.create_read_session] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_session.table", request.read_session.table), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def read_rows(self, + request: Optional[Union[storage.ReadRowsRequest, dict]] = None, + *, + read_stream: Optional[str] = None, + offset: Optional[int] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Awaitable[AsyncIterable[storage.ReadRowsResponse]]: + r"""Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + async def sample_read_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.ReadRowsRequest( + read_stream="read_stream_value", + ) + + # Make the request + stream = await client.read_rows(request=request) + + # Handle the response + async for response in stream: + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1.types.ReadRowsRequest, dict]]): + The request object. Request message for ``ReadRows``. + read_stream (:class:`str`): + Required. Stream to read rows from. + This corresponds to the ``read_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + offset (:class:`int`): + The offset requested must be less + than the last row read from Read. + Requesting a larger offset is undefined. + If not specified, start reading from + offset zero. + + This corresponds to the ``offset`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + AsyncIterable[google.cloud.bigquery_storage_v1.types.ReadRowsResponse]: + Response from calling ReadRows may include row data, progress and + throttling information. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([read_stream, offset]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.ReadRowsRequest): + request = storage.ReadRowsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if read_stream is not None: + request.read_stream = read_stream + if offset is not None: + request.offset = offset + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.read_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_stream", request.read_stream), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def split_read_stream(self, + request: Optional[Union[storage.SplitReadStreamRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.SplitReadStreamResponse: + r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + async def sample_split_read_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.SplitReadStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.split_read_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1.types.SplitReadStreamRequest, dict]]): + The request object. Request message for ``SplitReadStream``. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.SplitReadStreamResponse: + Response message for SplitReadStream. + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.SplitReadStreamRequest): + request = storage.SplitReadStreamRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.split_read_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def __aenter__(self) -> "BigQueryReadAsyncClient": + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.transport.close() + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +__all__ = ( + "BigQueryReadAsyncClient", +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/client.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/client.py new file mode 100644 index 00000000..f6811d19 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/client.py @@ -0,0 +1,983 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import os +import re +from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, Iterable, Sequence, Tuple, Type, Union, cast +import warnings + +from google.cloud.bigquery_storage_v1 import gapic_version as package_version + +from google.api_core import client_options as client_options_lib +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport import mtls # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.oauth2 import service_account # type: ignore + +try: + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.Retry, object, None] # type: ignore + +from google.cloud.bigquery_storage_v1.types import arrow +from google.cloud.bigquery_storage_v1.types import avro +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.protobuf import timestamp_pb2 # type: ignore +from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .transports.grpc import BigQueryReadGrpcTransport +from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport + + +class BigQueryReadClientMeta(type): + """Metaclass for the BigQueryRead client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + _transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] + _transport_registry["grpc"] = BigQueryReadGrpcTransport + _transport_registry["grpc_asyncio"] = BigQueryReadGrpcAsyncIOTransport + + def get_transport_class(cls, + label: Optional[str] = None, + ) -> Type[BigQueryReadTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class BigQueryReadClient(metaclass=BigQueryReadClientMeta): + """BigQuery Read API. + + The Read API can be used to read data from BigQuery. + """ + + @staticmethod + def _get_default_mtls_endpoint(api_endpoint): + """Converts api endpoint to mTLS endpoint. + + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to + "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. + Args: + api_endpoint (Optional[str]): the api endpoint to convert. + Returns: + str: converted mTLS api endpoint. + """ + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" + DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_ENDPOINT + ) + + _DEFAULT_ENDPOINT_TEMPLATE = "bigquerystorage.{UNIVERSE_DOMAIN}" + _DEFAULT_UNIVERSE = "googleapis.com" + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file( + filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryReadTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryReadTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def read_session_path(project: str,location: str,session: str,) -> str: + """Returns a fully-qualified read_session string.""" + return "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) + + @staticmethod + def parse_read_session_path(path: str) -> Dict[str,str]: + """Parses a read_session path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def read_stream_path(project: str,location: str,session: str,stream: str,) -> str: + """Returns a fully-qualified read_stream string.""" + return "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) + + @staticmethod + def parse_read_stream_path(path: str) -> Dict[str,str]: + """Parses a read_stream path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)/streams/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def table_path(project: str,dataset: str,table: str,) -> str: + """Returns a fully-qualified table string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + + @staticmethod + def parse_table_path(path: str) -> Dict[str,str]: + """Parses a table path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_billing_account_path(billing_account: str, ) -> str: + """Returns a fully-qualified billing_account string.""" + return "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + + @staticmethod + def parse_common_billing_account_path(path: str) -> Dict[str,str]: + """Parse a billing_account path into its component segments.""" + m = re.match(r"^billingAccounts/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_folder_path(folder: str, ) -> str: + """Returns a fully-qualified folder string.""" + return "folders/{folder}".format(folder=folder, ) + + @staticmethod + def parse_common_folder_path(path: str) -> Dict[str,str]: + """Parse a folder path into its component segments.""" + m = re.match(r"^folders/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_organization_path(organization: str, ) -> str: + """Returns a fully-qualified organization string.""" + return "organizations/{organization}".format(organization=organization, ) + + @staticmethod + def parse_common_organization_path(path: str) -> Dict[str,str]: + """Parse a organization path into its component segments.""" + m = re.match(r"^organizations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_project_path(project: str, ) -> str: + """Returns a fully-qualified project string.""" + return "projects/{project}".format(project=project, ) + + @staticmethod + def parse_common_project_path(path: str) -> Dict[str,str]: + """Parse a project path into its component segments.""" + m = re.match(r"^projects/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_location_path(project: str, location: str, ) -> str: + """Returns a fully-qualified location string.""" + return "projects/{project}/locations/{location}".format(project=project, location=location, ) + + @staticmethod + def parse_common_location_path(path: str) -> Dict[str,str]: + """Parse a location path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @classmethod + def get_mtls_endpoint_and_cert_source(cls, client_options: Optional[client_options_lib.ClientOptions] = None): + """Deprecated. Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variable is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + + warnings.warn("get_mtls_endpoint_and_cert_source is deprecated. Use the api_endpoint property instead.", + DeprecationWarning) + if client_options is None: + client_options = client_options_lib.ClientOptions() + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_client_cert not in ("true", "false"): + raise ValueError("Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`") + + # Figure out the client cert source to use. + client_cert_source = None + if use_client_cert == "true": + if client_options.client_cert_source: + client_cert_source = client_options.client_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + api_endpoint = cls.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = cls.DEFAULT_ENDPOINT + + return api_endpoint, client_cert_source + + @staticmethod + def _read_environment_variables(): + """Returns the environment variables used by the client. + + Returns: + Tuple[bool, str, str]: returns the GOOGLE_API_USE_CLIENT_CERTIFICATE, + GOOGLE_API_USE_MTLS_ENDPOINT, and GOOGLE_CLOUD_UNIVERSE_DOMAIN environment variables. + + Raises: + ValueError: If GOOGLE_API_USE_CLIENT_CERTIFICATE is not + any of ["true", "false"]. + google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT + is not any of ["auto", "never", "always"]. + """ + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false").lower() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_client_cert not in ("true", "false"): + raise ValueError("Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`") + return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + + @staticmethod + def _get_client_cert_source(provided_cert_source, use_cert_flag): + """Return the client cert source to be used by the client. + + Args: + provided_cert_source (bytes): The client certificate source provided. + use_cert_flag (bool): A flag indicating whether to use the client certificate. + + Returns: + bytes or None: The client cert source to be used by the client. + """ + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + return client_cert_source + + @staticmethod + def _get_api_endpoint(api_override, client_cert_source, universe_domain, use_mtls_endpoint): + """Return the API endpoint used by the client. + + Args: + api_override (str): The API endpoint override. If specified, this is always + the return value of this function and the other arguments are not used. + client_cert_source (bytes): The client certificate source used by the client. + universe_domain (str): The universe domain used by the client. + use_mtls_endpoint (str): How to use the mTLS endpoint, which depends also on the other parameters. + Possible values are "always", "auto", or "never". + + Returns: + str: The API endpoint to be used by the client. + """ + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + _default_universe = BigQueryReadClient._DEFAULT_UNIVERSE + if universe_domain != _default_universe: + raise MutualTLSChannelError(f"mTLS is not supported in any universe other than {_default_universe}.") + api_endpoint = BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=universe_domain) + return api_endpoint + + @staticmethod + def _get_universe_domain(client_universe_domain: Optional[str], universe_domain_env: Optional[str]) -> str: + """Return the universe domain used by the client. + + Args: + client_universe_domain (Optional[str]): The universe domain configured via the client options. + universe_domain_env (Optional[str]): The universe domain configured via the "GOOGLE_CLOUD_UNIVERSE_DOMAIN" environment variable. + + Returns: + str: The universe domain to be used by the client. + + Raises: + ValueError: If the universe domain is an empty string. + """ + universe_domain = BigQueryReadClient._DEFAULT_UNIVERSE + if client_universe_domain is not None: + universe_domain = client_universe_domain + elif universe_domain_env is not None: + universe_domain = universe_domain_env + if len(universe_domain.strip()) == 0: + raise ValueError("Universe Domain cannot be an empty string.") + return universe_domain + + @staticmethod + def _compare_universes(client_universe: str, + credentials: ga_credentials.Credentials) -> bool: + """Returns True iff the universe domains used by the client and credentials match. + + Args: + client_universe (str): The universe domain configured via the client options. + credentials (ga_credentials.Credentials): The credentials being used in the client. + + Returns: + bool: True iff client_universe matches the universe in credentials. + + Raises: + ValueError: when client_universe does not match the universe in credentials. + """ + + default_universe = BigQueryReadClient._DEFAULT_UNIVERSE + credentials_universe = getattr(credentials, "universe_domain", default_universe) + + if client_universe != credentials_universe: + raise ValueError("The configured universe domain " + f"({client_universe}) does not match the universe domain " + f"found in the credentials ({credentials_universe}). " + "If you haven't configured the universe domain explicitly, " + f"`{default_universe}` is the default.") + return True + + def _validate_universe_domain(self): + """Validates client's and credentials' universe domains are consistent. + + Returns: + bool: True iff the configured universe domain is valid. + + Raises: + ValueError: If the configured universe domain is not valid. + """ + self._is_universe_domain_valid = (self._is_universe_domain_valid or + BigQueryReadClient._compare_universes(self.universe_domain, self.transport._credentials)) + return self._is_universe_domain_valid + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used by the client instance. + """ + return self._universe_domain + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Optional[Union[str, BigQueryReadTransport, Callable[..., BigQueryReadTransport]]] = None, + client_options: Optional[Union[client_options_lib.ClientOptions, dict]] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query read client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Optional[Union[str,BigQueryReadTransport,Callable[..., BigQueryReadTransport]]]): + The transport to use, or a Callable that constructs and returns a new transport. + If a Callable is given, it will be called with the same set of initialization + arguments as used in the BigQueryReadTransport constructor. + If set to None, a transport is chosen automatically. + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide a client certificate for mTLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that the ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client_options = client_options + if isinstance(self._client_options, dict): + self._client_options = client_options_lib.from_dict(self._client_options) + if self._client_options is None: + self._client_options = client_options_lib.ClientOptions() + self._client_options = cast(client_options_lib.ClientOptions, self._client_options) + + universe_domain_opt = getattr(self._client_options, 'universe_domain', None) + + self._use_client_cert, self._use_mtls_endpoint, self._universe_domain_env = BigQueryReadClient._read_environment_variables() + self._client_cert_source = BigQueryReadClient._get_client_cert_source(self._client_options.client_cert_source, self._use_client_cert) + self._universe_domain = BigQueryReadClient._get_universe_domain(universe_domain_opt, self._universe_domain_env) + self._api_endpoint = None # updated below, depending on `transport` + + # Initialize the universe domain validation. + self._is_universe_domain_valid = False + + api_key_value = getattr(self._client_options, "api_key", None) + if api_key_value and credentials: + raise ValueError("client_options.api_key and credentials are mutually exclusive") + + # Save or instantiate the transport. + # Ordinarily, we provide the transport, but allowing a custom transport + # instance provides an extensibility point for unusual situations. + transport_provided = isinstance(transport, BigQueryReadTransport) + if transport_provided: + # transport is a BigQueryReadTransport instance. + if credentials or self._client_options.credentials_file or api_key_value: + raise ValueError("When providing a transport instance, " + "provide its credentials directly.") + if self._client_options.scopes: + raise ValueError( + "When providing a transport instance, provide its scopes " + "directly." + ) + self._transport = cast(BigQueryReadTransport, transport) + self._api_endpoint = self._transport.host + + self._api_endpoint = (self._api_endpoint or + BigQueryReadClient._get_api_endpoint( + self._client_options.api_endpoint, + self._client_cert_source, + self._universe_domain, + self._use_mtls_endpoint)) + + if not transport_provided: + import google.auth._default # type: ignore + + if api_key_value and hasattr(google.auth._default, "get_api_key_credentials"): + credentials = google.auth._default.get_api_key_credentials(api_key_value) + + transport_init: Union[Type[BigQueryReadTransport], Callable[..., BigQueryReadTransport]] = ( + BigQueryReadClient.get_transport_class(transport) + if isinstance(transport, str) or transport is None + else cast(Callable[..., BigQueryReadTransport], transport) + ) + # initialize with the provided callable or the passed in class + self._transport = transport_init( + credentials=credentials, + credentials_file=self._client_options.credentials_file, + host=self._api_endpoint, + scopes=self._client_options.scopes, + client_cert_source_for_mtls=self._client_cert_source, + quota_project_id=self._client_options.quota_project_id, + client_info=client_info, + always_use_jwt_access=True, + api_audience=self._client_options.api_audience, + ) + + def create_read_session(self, + request: Optional[Union[storage.CreateReadSessionRequest, dict]] = None, + *, + parent: Optional[str] = None, + read_session: Optional[stream.ReadSession] = None, + max_stream_count: Optional[int] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.ReadSession: + r"""Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a + push-down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + def sample_create_read_session(): + # Create a client + client = bigquery_storage_v1.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.CreateReadSessionRequest( + parent="parent_value", + ) + + # Make the request + response = client.create_read_session(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1.types.CreateReadSessionRequest, dict]): + The request object. Request message for ``CreateReadSession``. + parent (str): + Required. The request project that owns the session, in + the form of ``projects/{project_id}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + read_session (google.cloud.bigquery_storage_v1.types.ReadSession): + Required. Session to be created. + This corresponds to the ``read_session`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + max_stream_count (int): + Max initial number of streams. If unset or zero, the + server will provide a value of streams so as to produce + reasonable throughput. Must be non-negative. The number + of streams may be lower than the requested number, + depending on the amount parallelism that is reasonable + for the table. There is a default system max limit of + 1,000. + + This must be greater than or equal to + preferred_min_stream_count. Typically, clients should + either leave this unset to let the system to determine + an upper bound OR set this a size for the maximum "units + of work" it can gracefully handle. + + This corresponds to the ``max_stream_count`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.ReadSession: + Information about the ReadSession. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, read_session, max_stream_count]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.CreateReadSessionRequest): + request = storage.CreateReadSessionRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if read_session is not None: + request.read_session = read_session + if max_stream_count is not None: + request.max_stream_count = max_stream_count + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.create_read_session] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_session.table", request.read_session.table), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def read_rows(self, + request: Optional[Union[storage.ReadRowsRequest, dict]] = None, + *, + read_stream: Optional[str] = None, + offset: Optional[int] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Iterable[storage.ReadRowsResponse]: + r"""Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + def sample_read_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.ReadRowsRequest( + read_stream="read_stream_value", + ) + + # Make the request + stream = client.read_rows(request=request) + + # Handle the response + for response in stream: + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1.types.ReadRowsRequest, dict]): + The request object. Request message for ``ReadRows``. + read_stream (str): + Required. Stream to read rows from. + This corresponds to the ``read_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + offset (int): + The offset requested must be less + than the last row read from Read. + Requesting a larger offset is undefined. + If not specified, start reading from + offset zero. + + This corresponds to the ``offset`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + Iterable[google.cloud.bigquery_storage_v1.types.ReadRowsResponse]: + Response from calling ReadRows may include row data, progress and + throttling information. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([read_stream, offset]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.ReadRowsRequest): + request = storage.ReadRowsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if read_stream is not None: + request.read_stream = read_stream + if offset is not None: + request.offset = offset + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.read_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_stream", request.read_stream), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def split_read_stream(self, + request: Optional[Union[storage.SplitReadStreamRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.SplitReadStreamResponse: + r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + def sample_split_read_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.SplitReadStreamRequest( + name="name_value", + ) + + # Make the request + response = client.split_read_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1.types.SplitReadStreamRequest, dict]): + The request object. Request message for ``SplitReadStream``. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.SplitReadStreamResponse: + Response message for SplitReadStream. + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.SplitReadStreamRequest): + request = storage.SplitReadStreamRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.split_read_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def __enter__(self) -> "BigQueryReadClient": + return self + + def __exit__(self, type, value, traceback): + """Releases underlying transport's resources. + + .. warning:: + ONLY use as a context manager if the transport is NOT shared + with other clients! Exiting the with block will CLOSE the transport + and may cause errors in other clients! + """ + self.transport.close() + + + + + + + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +__all__ = ( + "BigQueryReadClient", +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/README.rst b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/README.rst new file mode 100644 index 00000000..687d374a --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/README.rst @@ -0,0 +1,9 @@ + +transport inheritance structure +_______________________________ + +`BigQueryReadTransport` is the ABC for all transports. +- public child `BigQueryReadGrpcTransport` for sync gRPC transport (defined in `grpc.py`). +- public child `BigQueryReadGrpcAsyncIOTransport` for async gRPC transport (defined in `grpc_asyncio.py`). +- private child `_BaseBigQueryReadRestTransport` for base REST transport with inner classes `_BaseMETHOD` (defined in `rest_base.py`). +- public child `BigQueryReadRestTransport` for sync REST transport with inner classes `METHOD` derived from the parent's corresponding `_BaseMETHOD` classes (defined in `rest.py`). diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/__init__.py new file mode 100644 index 00000000..7547977d --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from typing import Dict, Type + +from .base import BigQueryReadTransport +from .grpc import BigQueryReadGrpcTransport +from .grpc_asyncio import BigQueryReadGrpcAsyncIOTransport + + +# Compile a registry of transports. +_transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] +_transport_registry['grpc'] = BigQueryReadGrpcTransport +_transport_registry['grpc_asyncio'] = BigQueryReadGrpcAsyncIOTransport + +__all__ = ( + 'BigQueryReadTransport', + 'BigQueryReadGrpcTransport', + 'BigQueryReadGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py new file mode 100644 index 00000000..8378649a --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/base.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union + +from google.cloud.bigquery_storage_v1 import gapic_version as package_version + +import google.auth # type: ignore +import google.api_core +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +class BigQueryReadTransport(abc.ABC): + """Abstract transport class for BigQueryRead.""" + + AUTH_SCOPES = ( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', + ) + + DEFAULT_HOST: str = 'bigquerystorage.googleapis.com' + def __init__( + self, *, + host: str = DEFAULT_HOST, + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + **kwargs, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A list of scopes. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + """ + + scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES} + + # Save the scopes. + self._scopes = scopes + if not hasattr(self, "_ignore_credentials"): + self._ignore_credentials: bool = False + + # If no credentials are provided, then determine the appropriate + # defaults. + if credentials and credentials_file: + raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive") + + if credentials_file is not None: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, + **scopes_kwargs, + quota_project_id=quota_project_id + ) + elif credentials is None and not self._ignore_credentials: + credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id) + # Don't apply audience if the credentials file passed from user. + if hasattr(credentials, "with_gdch_audience"): + credentials = credentials.with_gdch_audience(api_audience if api_audience else host) + + # If the credentials are service account credentials, then always try to use self signed JWT. + if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"): + credentials = credentials.with_always_use_jwt_access(True) + + # Save the credentials. + self._credentials = credentials + + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ':' not in host: + host += ':443' + self._host = host + + @property + def host(self): + return self._host + + def _prep_wrapped_messages(self, client_info): + # Precompute the wrapped methods. + self._wrapped_methods = { + self.create_read_session: gapic_v1.method.wrap_method( + self.create_read_session, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.read_rows: gapic_v1.method.wrap_method( + self.read_rows, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.split_read_stream: gapic_v1.method.wrap_method( + self.split_read_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + def close(self): + """Closes resources associated with the transport. + + .. warning:: + Only call this method if the transport is NOT shared + with other clients - this may cause errors in other clients! + """ + raise NotImplementedError() + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + Union[ + stream.ReadSession, + Awaitable[stream.ReadSession] + ]]: + raise NotImplementedError() + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + Union[ + storage.ReadRowsResponse, + Awaitable[storage.ReadRowsResponse] + ]]: + raise NotImplementedError() + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + Union[ + storage.SplitReadStreamResponse, + Awaitable[storage.SplitReadStreamResponse] + ]]: + raise NotImplementedError() + + @property + def kind(self) -> str: + raise NotImplementedError() + + +__all__ = ( + 'BigQueryReadTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py new file mode 100644 index 00000000..4e201186 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc.py @@ -0,0 +1,366 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import grpc_helpers +from google.api_core import gapic_v1 +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO + + +class BigQueryReadGrpcTransport(BigQueryReadTransport): + """gRPC backend transport for BigQueryRead. + + BigQuery Read API. + + The Read API can be used to read data from BigQuery. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + _stubs: Dict[str, Callable] + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]] = None, + api_mtls_endpoint: Optional[str] = None, + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + ssl_channel_credentials: Optional[grpc.ChannelCredentials] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if a ``channel`` instance is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if a ``channel`` instance is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if a ``channel`` instance is provided. + channel (Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]]): + A ``Channel`` instance through which to make calls, or a Callable + that constructs and returns one. If set to None, ``self.create_channel`` + is used to create the channel. If a Callable is given, it will be called + with the same arguments as used in ``self.create_channel``. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or application default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for the grpc channel. It is ignored if a ``channel`` instance is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if a ``channel`` instance or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if isinstance(channel, grpc.Channel): + # Ignore credentials if a channel was passed. + credentials = None + self._ignore_credentials = True + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, + ) + + if not self._grpc_channel: + # initialize with the provided callable or the default channel + channel_init = channel or type(self).create_channel + self._grpc_channel = channel_init( + self._host, + # use the credentials which are saved + credentials=self._credentials, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> grpc.Channel: + """Create and return a gRPC channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + grpc.Channel: A gRPC channel object. + + Raises: + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + + return grpc_helpers.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + @property + def grpc_channel(self) -> grpc.Channel: + """Return the channel designed to connect to this service. + """ + return self._grpc_channel + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + stream.ReadSession]: + r"""Return a callable for the create read session method over gRPC. + + Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a + push-down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + Returns: + Callable[[~.CreateReadSessionRequest], + ~.ReadSession]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_read_session' not in self._stubs: + self._stubs['create_read_session'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryRead/CreateReadSession', + request_serializer=storage.CreateReadSessionRequest.serialize, + response_deserializer=stream.ReadSession.deserialize, + ) + return self._stubs['create_read_session'] + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + storage.ReadRowsResponse]: + r"""Return a callable for the read rows method over gRPC. + + Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Returns: + Callable[[~.ReadRowsRequest], + ~.ReadRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'read_rows' not in self._stubs: + self._stubs['read_rows'] = self.grpc_channel.unary_stream( + '/google.cloud.bigquery.storage.v1.BigQueryRead/ReadRows', + request_serializer=storage.ReadRowsRequest.serialize, + response_deserializer=storage.ReadRowsResponse.deserialize, + ) + return self._stubs['read_rows'] + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + storage.SplitReadStreamResponse]: + r"""Return a callable for the split read stream method over gRPC. + + Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Returns: + Callable[[~.SplitReadStreamRequest], + ~.SplitReadStreamResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'split_read_stream' not in self._stubs: + self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryRead/SplitReadStream', + request_serializer=storage.SplitReadStreamRequest.serialize, + response_deserializer=storage.SplitReadStreamResponse.deserialize, + ) + return self._stubs['split_read_stream'] + + def close(self): + self.grpc_channel.close() + + @property + def kind(self) -> str: + return "grpc" + + +__all__ = ( + 'BigQueryReadGrpcTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py new file mode 100644 index 00000000..daff7362 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_read/transports/grpc_asyncio.py @@ -0,0 +1,426 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import inspect +import warnings +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers_async +from google.api_core import exceptions as core_exceptions +from google.api_core import retry_async as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore +from grpc.experimental import aio # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .grpc import BigQueryReadGrpcTransport + + +class BigQueryReadGrpcAsyncIOTransport(BigQueryReadTransport): + """gRPC AsyncIO backend transport for BigQueryRead. + + BigQuery Read API. + + The Read API can be used to read data from BigQuery. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + + _grpc_channel: aio.Channel + _stubs: Dict[str, Callable] = {} + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> aio.Channel: + """Create and return a gRPC AsyncIO channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + aio.Channel: A gRPC AsyncIO channel object. + """ + + return grpc_helpers_async.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: Optional[Union[aio.Channel, Callable[..., aio.Channel]]] = None, + api_mtls_endpoint: Optional[str] = None, + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + ssl_channel_credentials: Optional[grpc.ChannelCredentials] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if a ``channel`` instance is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if a ``channel`` instance is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + channel (Optional[Union[aio.Channel, Callable[..., aio.Channel]]]): + A ``Channel`` instance through which to make calls, or a Callable + that constructs and returns one. If set to None, ``self.create_channel`` + is used to create the channel. If a Callable is given, it will be called + with the same arguments as used in ``self.create_channel``. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or application default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for the grpc channel. It is ignored if a ``channel`` instance is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if a ``channel`` instance or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if isinstance(channel, aio.Channel): + # Ignore credentials if a channel was passed. + credentials = None + self._ignore_credentials = True + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, + ) + + if not self._grpc_channel: + # initialize with the provided callable or the default channel + channel_init = channel or type(self).create_channel + self._grpc_channel = channel_init( + self._host, + # use the credentials which are saved + credentials=self._credentials, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._wrap_with_kind = "kind" in inspect.signature(gapic_v1.method_async.wrap_method).parameters + self._prep_wrapped_messages(client_info) + + @property + def grpc_channel(self) -> aio.Channel: + """Create the channel designed to connect to this service. + + This property caches on the instance; repeated calls return + the same channel. + """ + # Return the channel from cache. + return self._grpc_channel + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + Awaitable[stream.ReadSession]]: + r"""Return a callable for the create read session method over gRPC. + + Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a + push-down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + Returns: + Callable[[~.CreateReadSessionRequest], + Awaitable[~.ReadSession]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_read_session' not in self._stubs: + self._stubs['create_read_session'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryRead/CreateReadSession', + request_serializer=storage.CreateReadSessionRequest.serialize, + response_deserializer=stream.ReadSession.deserialize, + ) + return self._stubs['create_read_session'] + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + Awaitable[storage.ReadRowsResponse]]: + r"""Return a callable for the read rows method over gRPC. + + Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Returns: + Callable[[~.ReadRowsRequest], + Awaitable[~.ReadRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'read_rows' not in self._stubs: + self._stubs['read_rows'] = self.grpc_channel.unary_stream( + '/google.cloud.bigquery.storage.v1.BigQueryRead/ReadRows', + request_serializer=storage.ReadRowsRequest.serialize, + response_deserializer=storage.ReadRowsResponse.deserialize, + ) + return self._stubs['read_rows'] + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + Awaitable[storage.SplitReadStreamResponse]]: + r"""Return a callable for the split read stream method over gRPC. + + Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Returns: + Callable[[~.SplitReadStreamRequest], + Awaitable[~.SplitReadStreamResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'split_read_stream' not in self._stubs: + self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryRead/SplitReadStream', + request_serializer=storage.SplitReadStreamRequest.serialize, + response_deserializer=storage.SplitReadStreamResponse.deserialize, + ) + return self._stubs['split_read_stream'] + + def _prep_wrapped_messages(self, client_info): + """ Precompute the wrapped methods, overriding the base class method to use async wrappers.""" + self._wrapped_methods = { + self.create_read_session: self._wrap_method( + self.create_read_session, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.read_rows: self._wrap_method( + self.read_rows, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.split_read_stream: self._wrap_method( + self.split_read_stream, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + def _wrap_method(self, func, *args, **kwargs): + if self._wrap_with_kind: # pragma: NO COVER + kwargs["kind"] = self.kind + return gapic_v1.method_async.wrap_method(func, *args, **kwargs) + + def close(self): + return self.grpc_channel.close() + + @property + def kind(self) -> str: + return "grpc_asyncio" + + +__all__ = ( + 'BigQueryReadGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/__init__.py new file mode 100644 index 00000000..9000288a --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .client import BigQueryWriteClient +from .async_client import BigQueryWriteAsyncClient + +__all__ = ( + 'BigQueryWriteClient', + 'BigQueryWriteAsyncClient', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/async_client.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/async_client.py new file mode 100644 index 00000000..a08d518f --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/async_client.py @@ -0,0 +1,916 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import re +from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, AsyncIterable, Awaitable, AsyncIterator, Sequence, Tuple, Type, Union + +from google.cloud.bigquery_storage_v1 import gapic_version as package_version + +from google.api_core.client_options import ClientOptions +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry_async as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + + +try: + OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.AsyncRetry, object, None] # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.cloud.bigquery_storage_v1.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +from .transports.base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .transports.grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport +from .client import BigQueryWriteClient + + +class BigQueryWriteAsyncClient: + """BigQuery Write API. + + The Write API can be used to write data to BigQuery. + + For supplementary information about the Write API, see: + + https://cloud.google.com/bigquery/docs/write-api + """ + + _client: BigQueryWriteClient + + # Copy defaults from the synchronous client for use here. + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + DEFAULT_ENDPOINT = BigQueryWriteClient.DEFAULT_ENDPOINT + DEFAULT_MTLS_ENDPOINT = BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + _DEFAULT_ENDPOINT_TEMPLATE = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE + _DEFAULT_UNIVERSE = BigQueryWriteClient._DEFAULT_UNIVERSE + + table_path = staticmethod(BigQueryWriteClient.table_path) + parse_table_path = staticmethod(BigQueryWriteClient.parse_table_path) + write_stream_path = staticmethod(BigQueryWriteClient.write_stream_path) + parse_write_stream_path = staticmethod(BigQueryWriteClient.parse_write_stream_path) + common_billing_account_path = staticmethod(BigQueryWriteClient.common_billing_account_path) + parse_common_billing_account_path = staticmethod(BigQueryWriteClient.parse_common_billing_account_path) + common_folder_path = staticmethod(BigQueryWriteClient.common_folder_path) + parse_common_folder_path = staticmethod(BigQueryWriteClient.parse_common_folder_path) + common_organization_path = staticmethod(BigQueryWriteClient.common_organization_path) + parse_common_organization_path = staticmethod(BigQueryWriteClient.parse_common_organization_path) + common_project_path = staticmethod(BigQueryWriteClient.common_project_path) + parse_common_project_path = staticmethod(BigQueryWriteClient.parse_common_project_path) + common_location_path = staticmethod(BigQueryWriteClient.common_location_path) + parse_common_location_path = staticmethod(BigQueryWriteClient.parse_common_location_path) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteAsyncClient: The constructed client. + """ + return BigQueryWriteClient.from_service_account_info.__func__(BigQueryWriteAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteAsyncClient: The constructed client. + """ + return BigQueryWriteClient.from_service_account_file.__func__(BigQueryWriteAsyncClient, filename, *args, **kwargs) # type: ignore + + from_service_account_json = from_service_account_file + + @classmethod + def get_mtls_endpoint_and_cert_source(cls, client_options: Optional[ClientOptions] = None): + """Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variable is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + return BigQueryWriteClient.get_mtls_endpoint_and_cert_source(client_options) # type: ignore + + @property + def transport(self) -> BigQueryWriteTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryWriteTransport: The transport used by the client instance. + """ + return self._client.transport + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._client._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used + by the client instance. + """ + return self._client._universe_domain + + get_transport_class = BigQueryWriteClient.get_transport_class + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Optional[Union[str, BigQueryWriteTransport, Callable[..., BigQueryWriteTransport]]] = "grpc_asyncio", + client_options: Optional[ClientOptions] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query write async client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Optional[Union[str,BigQueryWriteTransport,Callable[..., BigQueryWriteTransport]]]): + The transport to use, or a Callable that constructs and returns a new transport to use. + If a Callable is given, it will be called with the same set of initialization + arguments as used in the BigQueryWriteTransport constructor. + If set to None, a transport is chosen automatically. + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide a client certificate for mTLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client = BigQueryWriteClient( + credentials=credentials, + transport=transport, + client_options=client_options, + client_info=client_info, + + ) + + async def create_write_stream(self, + request: Optional[Union[storage.CreateWriteStreamRequest, dict]] = None, + *, + parent: Optional[str] = None, + write_stream: Optional[stream.WriteStream] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Creates a write stream to the given table. Additionally, every + table has a special stream named '_default' to which data can be + written. This stream doesn't need to be created using + CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + async def sample_create_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.CreateWriteStreamRequest( + parent="parent_value", + ) + + # Make the request + response = await client.create_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1.types.CreateWriteStreamRequest, dict]]): + The request object. Request message for ``CreateWriteStream``. + parent (:class:`str`): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + write_stream (:class:`google.cloud.bigquery_storage_v1.types.WriteStream`): + Required. Stream to be created. + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, write_stream]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.CreateWriteStreamRequest): + request = storage.CreateWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.create_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def append_rows(self, + requests: Optional[AsyncIterator[storage.AppendRowsRequest]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Awaitable[AsyncIterable[storage.AppendRowsResponse]]: + r"""Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provides an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC connection. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains an optional offset at which the append + happened. No offset information will be returned for appends to + a default stream. + + Responses are received in the same order in which requests are + sent. There will be one response for each successful inserted + request. Responses may optionally embed error information if the + originating AppendRequest was not successfully processed. + + The specifics of when successfully appended data is made visible + to the table are governed by the type of stream: + + - For COMMITTED streams (which includes the default stream), + data is visible immediately upon successful append. + + - For BUFFERED streams, data is made visible via a subsequent + ``FlushRows`` rpc which advances a cursor to a newer offset + in the stream. + + - For PENDING streams, data is not made visible until the + stream itself is finalized (via the ``FinalizeWriteStream`` + rpc), and the stream is explicitly committed via the + ``BatchCommitWriteStreams`` rpc. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + async def sample_append_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.AppendRowsRequest( + write_stream="write_stream_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1.AppendRowsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = await client.append_rows(requests=request_generator()) + + # Handle the response + async for response in stream: + print(response) + + Args: + requests (AsyncIterator[`google.cloud.bigquery_storage_v1.types.AppendRowsRequest`]): + The request object AsyncIterator. Request message for ``AppendRows``. + + Because AppendRows is a bidirectional streaming RPC, + certain parts of the AppendRowsRequest need only be + specified for the first request before switching table + destinations. You can also switch table destinations + within the same connection for the default stream. + + The size of a single AppendRowsRequest must be less than + 10 MB in size. Requests larger than this return an + error, typically ``INVALID_ARGUMENT``. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + AsyncIterable[google.cloud.bigquery_storage_v1.types.AppendRowsResponse]: + Response message for AppendRows. + """ + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.append_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = rpc( + requests, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def get_write_stream(self, + request: Optional[Union[storage.GetWriteStreamRequest, dict]] = None, + *, + name: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Gets information about a write stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + async def sample_get_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.GetWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.get_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1.types.GetWriteStreamRequest, dict]]): + The request object. Request message for ``GetWriteStreamRequest``. + name (:class:`str`): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.GetWriteStreamRequest): + request = storage.GetWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.get_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def finalize_write_stream(self, + request: Optional[Union[storage.FinalizeWriteStreamRequest, dict]] = None, + *, + name: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FinalizeWriteStreamResponse: + r"""Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + async def sample_finalize_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.FinalizeWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.finalize_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamRequest, dict]]): + The request object. Request message for invoking ``FinalizeWriteStream``. + name (:class:`str`): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamResponse: + Response message for FinalizeWriteStream. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.FinalizeWriteStreamRequest): + request = storage.FinalizeWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.finalize_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def batch_commit_write_streams(self, + request: Optional[Union[storage.BatchCommitWriteStreamsRequest, dict]] = None, + *, + parent: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.BatchCommitWriteStreamsResponse: + r"""Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. + + Streams must be finalized before commit and cannot be committed + multiple times. Once a stream is committed, data in the stream + becomes available for read operations. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + async def sample_batch_commit_write_streams(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.BatchCommitWriteStreamsRequest( + parent="parent_value", + write_streams=['write_streams_value1', 'write_streams_value2'], + ) + + # Make the request + response = await client.batch_commit_write_streams(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsRequest, dict]]): + The request object. Request message for ``BatchCommitWriteStreams``. + parent (:class:`str`): + Required. Parent table that all the streams should + belong to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsResponse: + Response message for BatchCommitWriteStreams. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.BatchCommitWriteStreamsRequest): + request = storage.BatchCommitWriteStreamsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.batch_commit_write_streams] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def flush_rows(self, + request: Optional[Union[storage.FlushRowsRequest, dict]] = None, + *, + write_stream: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FlushRowsResponse: + r"""Flushes rows to a BUFFERED stream. + + If users are appending rows to BUFFERED stream, flush operation + is required in order for the rows to become available for + reading. A Flush operation flushes up to any previously flushed + offset in a BUFFERED stream, to the offset specified in the + request. + + Flush is not supported on the \_default stream, since it is not + BUFFERED. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + async def sample_flush_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.FlushRowsRequest( + write_stream="write_stream_value", + ) + + # Make the request + response = await client.flush_rows(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1.types.FlushRowsRequest, dict]]): + The request object. Request message for ``FlushRows``. + write_stream (:class:`str`): + Required. The stream that is the + target of the flush operation. + + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.FlushRowsResponse: + Respond message for FlushRows. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([write_stream]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.FlushRowsRequest): + request = storage.FlushRowsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.flush_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("write_stream", request.write_stream), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def __aenter__(self) -> "BigQueryWriteAsyncClient": + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.transport.close() + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +__all__ = ( + "BigQueryWriteAsyncClient", +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/client.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/client.py new file mode 100644 index 00000000..d1a99c46 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/client.py @@ -0,0 +1,1275 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import os +import re +from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, Iterable, Iterator, Sequence, Tuple, Type, Union, cast +import warnings + +from google.cloud.bigquery_storage_v1 import gapic_version as package_version + +from google.api_core import client_options as client_options_lib +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport import mtls # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.oauth2 import service_account # type: ignore + +try: + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.Retry, object, None] # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.cloud.bigquery_storage_v1.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +from .transports.base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .transports.grpc import BigQueryWriteGrpcTransport +from .transports.grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport + + +class BigQueryWriteClientMeta(type): + """Metaclass for the BigQueryWrite client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + _transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryWriteTransport]] + _transport_registry["grpc"] = BigQueryWriteGrpcTransport + _transport_registry["grpc_asyncio"] = BigQueryWriteGrpcAsyncIOTransport + + def get_transport_class(cls, + label: Optional[str] = None, + ) -> Type[BigQueryWriteTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class BigQueryWriteClient(metaclass=BigQueryWriteClientMeta): + """BigQuery Write API. + + The Write API can be used to write data to BigQuery. + + For supplementary information about the Write API, see: + + https://cloud.google.com/bigquery/docs/write-api + """ + + @staticmethod + def _get_default_mtls_endpoint(api_endpoint): + """Converts api endpoint to mTLS endpoint. + + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to + "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. + Args: + api_endpoint (Optional[str]): the api endpoint to convert. + Returns: + str: converted mTLS api endpoint. + """ + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" + DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_ENDPOINT + ) + + _DEFAULT_ENDPOINT_TEMPLATE = "bigquerystorage.{UNIVERSE_DOMAIN}" + _DEFAULT_UNIVERSE = "googleapis.com" + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file( + filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryWriteTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryWriteTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def table_path(project: str,dataset: str,table: str,) -> str: + """Returns a fully-qualified table string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + + @staticmethod + def parse_table_path(path: str) -> Dict[str,str]: + """Parses a table path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def write_stream_path(project: str,dataset: str,table: str,stream: str,) -> str: + """Returns a fully-qualified write_stream string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}".format(project=project, dataset=dataset, table=table, stream=stream, ) + + @staticmethod + def parse_write_stream_path(path: str) -> Dict[str,str]: + """Parses a write_stream path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)/streams/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_billing_account_path(billing_account: str, ) -> str: + """Returns a fully-qualified billing_account string.""" + return "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + + @staticmethod + def parse_common_billing_account_path(path: str) -> Dict[str,str]: + """Parse a billing_account path into its component segments.""" + m = re.match(r"^billingAccounts/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_folder_path(folder: str, ) -> str: + """Returns a fully-qualified folder string.""" + return "folders/{folder}".format(folder=folder, ) + + @staticmethod + def parse_common_folder_path(path: str) -> Dict[str,str]: + """Parse a folder path into its component segments.""" + m = re.match(r"^folders/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_organization_path(organization: str, ) -> str: + """Returns a fully-qualified organization string.""" + return "organizations/{organization}".format(organization=organization, ) + + @staticmethod + def parse_common_organization_path(path: str) -> Dict[str,str]: + """Parse a organization path into its component segments.""" + m = re.match(r"^organizations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_project_path(project: str, ) -> str: + """Returns a fully-qualified project string.""" + return "projects/{project}".format(project=project, ) + + @staticmethod + def parse_common_project_path(path: str) -> Dict[str,str]: + """Parse a project path into its component segments.""" + m = re.match(r"^projects/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_location_path(project: str, location: str, ) -> str: + """Returns a fully-qualified location string.""" + return "projects/{project}/locations/{location}".format(project=project, location=location, ) + + @staticmethod + def parse_common_location_path(path: str) -> Dict[str,str]: + """Parse a location path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @classmethod + def get_mtls_endpoint_and_cert_source(cls, client_options: Optional[client_options_lib.ClientOptions] = None): + """Deprecated. Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variable is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + + warnings.warn("get_mtls_endpoint_and_cert_source is deprecated. Use the api_endpoint property instead.", + DeprecationWarning) + if client_options is None: + client_options = client_options_lib.ClientOptions() + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_client_cert not in ("true", "false"): + raise ValueError("Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`") + + # Figure out the client cert source to use. + client_cert_source = None + if use_client_cert == "true": + if client_options.client_cert_source: + client_cert_source = client_options.client_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + api_endpoint = cls.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = cls.DEFAULT_ENDPOINT + + return api_endpoint, client_cert_source + + @staticmethod + def _read_environment_variables(): + """Returns the environment variables used by the client. + + Returns: + Tuple[bool, str, str]: returns the GOOGLE_API_USE_CLIENT_CERTIFICATE, + GOOGLE_API_USE_MTLS_ENDPOINT, and GOOGLE_CLOUD_UNIVERSE_DOMAIN environment variables. + + Raises: + ValueError: If GOOGLE_API_USE_CLIENT_CERTIFICATE is not + any of ["true", "false"]. + google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT + is not any of ["auto", "never", "always"]. + """ + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false").lower() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_client_cert not in ("true", "false"): + raise ValueError("Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`") + return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + + @staticmethod + def _get_client_cert_source(provided_cert_source, use_cert_flag): + """Return the client cert source to be used by the client. + + Args: + provided_cert_source (bytes): The client certificate source provided. + use_cert_flag (bool): A flag indicating whether to use the client certificate. + + Returns: + bytes or None: The client cert source to be used by the client. + """ + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + return client_cert_source + + @staticmethod + def _get_api_endpoint(api_override, client_cert_source, universe_domain, use_mtls_endpoint): + """Return the API endpoint used by the client. + + Args: + api_override (str): The API endpoint override. If specified, this is always + the return value of this function and the other arguments are not used. + client_cert_source (bytes): The client certificate source used by the client. + universe_domain (str): The universe domain used by the client. + use_mtls_endpoint (str): How to use the mTLS endpoint, which depends also on the other parameters. + Possible values are "always", "auto", or "never". + + Returns: + str: The API endpoint to be used by the client. + """ + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + _default_universe = BigQueryWriteClient._DEFAULT_UNIVERSE + if universe_domain != _default_universe: + raise MutualTLSChannelError(f"mTLS is not supported in any universe other than {_default_universe}.") + api_endpoint = BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=universe_domain) + return api_endpoint + + @staticmethod + def _get_universe_domain(client_universe_domain: Optional[str], universe_domain_env: Optional[str]) -> str: + """Return the universe domain used by the client. + + Args: + client_universe_domain (Optional[str]): The universe domain configured via the client options. + universe_domain_env (Optional[str]): The universe domain configured via the "GOOGLE_CLOUD_UNIVERSE_DOMAIN" environment variable. + + Returns: + str: The universe domain to be used by the client. + + Raises: + ValueError: If the universe domain is an empty string. + """ + universe_domain = BigQueryWriteClient._DEFAULT_UNIVERSE + if client_universe_domain is not None: + universe_domain = client_universe_domain + elif universe_domain_env is not None: + universe_domain = universe_domain_env + if len(universe_domain.strip()) == 0: + raise ValueError("Universe Domain cannot be an empty string.") + return universe_domain + + @staticmethod + def _compare_universes(client_universe: str, + credentials: ga_credentials.Credentials) -> bool: + """Returns True iff the universe domains used by the client and credentials match. + + Args: + client_universe (str): The universe domain configured via the client options. + credentials (ga_credentials.Credentials): The credentials being used in the client. + + Returns: + bool: True iff client_universe matches the universe in credentials. + + Raises: + ValueError: when client_universe does not match the universe in credentials. + """ + + default_universe = BigQueryWriteClient._DEFAULT_UNIVERSE + credentials_universe = getattr(credentials, "universe_domain", default_universe) + + if client_universe != credentials_universe: + raise ValueError("The configured universe domain " + f"({client_universe}) does not match the universe domain " + f"found in the credentials ({credentials_universe}). " + "If you haven't configured the universe domain explicitly, " + f"`{default_universe}` is the default.") + return True + + def _validate_universe_domain(self): + """Validates client's and credentials' universe domains are consistent. + + Returns: + bool: True iff the configured universe domain is valid. + + Raises: + ValueError: If the configured universe domain is not valid. + """ + self._is_universe_domain_valid = (self._is_universe_domain_valid or + BigQueryWriteClient._compare_universes(self.universe_domain, self.transport._credentials)) + return self._is_universe_domain_valid + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used by the client instance. + """ + return self._universe_domain + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Optional[Union[str, BigQueryWriteTransport, Callable[..., BigQueryWriteTransport]]] = None, + client_options: Optional[Union[client_options_lib.ClientOptions, dict]] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query write client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Optional[Union[str,BigQueryWriteTransport,Callable[..., BigQueryWriteTransport]]]): + The transport to use, or a Callable that constructs and returns a new transport. + If a Callable is given, it will be called with the same set of initialization + arguments as used in the BigQueryWriteTransport constructor. + If set to None, a transport is chosen automatically. + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide a client certificate for mTLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that the ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client_options = client_options + if isinstance(self._client_options, dict): + self._client_options = client_options_lib.from_dict(self._client_options) + if self._client_options is None: + self._client_options = client_options_lib.ClientOptions() + self._client_options = cast(client_options_lib.ClientOptions, self._client_options) + + universe_domain_opt = getattr(self._client_options, 'universe_domain', None) + + self._use_client_cert, self._use_mtls_endpoint, self._universe_domain_env = BigQueryWriteClient._read_environment_variables() + self._client_cert_source = BigQueryWriteClient._get_client_cert_source(self._client_options.client_cert_source, self._use_client_cert) + self._universe_domain = BigQueryWriteClient._get_universe_domain(universe_domain_opt, self._universe_domain_env) + self._api_endpoint = None # updated below, depending on `transport` + + # Initialize the universe domain validation. + self._is_universe_domain_valid = False + + api_key_value = getattr(self._client_options, "api_key", None) + if api_key_value and credentials: + raise ValueError("client_options.api_key and credentials are mutually exclusive") + + # Save or instantiate the transport. + # Ordinarily, we provide the transport, but allowing a custom transport + # instance provides an extensibility point for unusual situations. + transport_provided = isinstance(transport, BigQueryWriteTransport) + if transport_provided: + # transport is a BigQueryWriteTransport instance. + if credentials or self._client_options.credentials_file or api_key_value: + raise ValueError("When providing a transport instance, " + "provide its credentials directly.") + if self._client_options.scopes: + raise ValueError( + "When providing a transport instance, provide its scopes " + "directly." + ) + self._transport = cast(BigQueryWriteTransport, transport) + self._api_endpoint = self._transport.host + + self._api_endpoint = (self._api_endpoint or + BigQueryWriteClient._get_api_endpoint( + self._client_options.api_endpoint, + self._client_cert_source, + self._universe_domain, + self._use_mtls_endpoint)) + + if not transport_provided: + import google.auth._default # type: ignore + + if api_key_value and hasattr(google.auth._default, "get_api_key_credentials"): + credentials = google.auth._default.get_api_key_credentials(api_key_value) + + transport_init: Union[Type[BigQueryWriteTransport], Callable[..., BigQueryWriteTransport]] = ( + BigQueryWriteClient.get_transport_class(transport) + if isinstance(transport, str) or transport is None + else cast(Callable[..., BigQueryWriteTransport], transport) + ) + # initialize with the provided callable or the passed in class + self._transport = transport_init( + credentials=credentials, + credentials_file=self._client_options.credentials_file, + host=self._api_endpoint, + scopes=self._client_options.scopes, + client_cert_source_for_mtls=self._client_cert_source, + quota_project_id=self._client_options.quota_project_id, + client_info=client_info, + always_use_jwt_access=True, + api_audience=self._client_options.api_audience, + ) + + def create_write_stream(self, + request: Optional[Union[storage.CreateWriteStreamRequest, dict]] = None, + *, + parent: Optional[str] = None, + write_stream: Optional[stream.WriteStream] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Creates a write stream to the given table. Additionally, every + table has a special stream named '_default' to which data can be + written. This stream doesn't need to be created using + CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + def sample_create_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.CreateWriteStreamRequest( + parent="parent_value", + ) + + # Make the request + response = client.create_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1.types.CreateWriteStreamRequest, dict]): + The request object. Request message for ``CreateWriteStream``. + parent (str): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + write_stream (google.cloud.bigquery_storage_v1.types.WriteStream): + Required. Stream to be created. + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, write_stream]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.CreateWriteStreamRequest): + request = storage.CreateWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.create_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def append_rows(self, + requests: Optional[Iterator[storage.AppendRowsRequest]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Iterable[storage.AppendRowsResponse]: + r"""Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provides an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC connection. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains an optional offset at which the append + happened. No offset information will be returned for appends to + a default stream. + + Responses are received in the same order in which requests are + sent. There will be one response for each successful inserted + request. Responses may optionally embed error information if the + originating AppendRequest was not successfully processed. + + The specifics of when successfully appended data is made visible + to the table are governed by the type of stream: + + - For COMMITTED streams (which includes the default stream), + data is visible immediately upon successful append. + + - For BUFFERED streams, data is made visible via a subsequent + ``FlushRows`` rpc which advances a cursor to a newer offset + in the stream. + + - For PENDING streams, data is not made visible until the + stream itself is finalized (via the ``FinalizeWriteStream`` + rpc), and the stream is explicitly committed via the + ``BatchCommitWriteStreams`` rpc. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + def sample_append_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.AppendRowsRequest( + write_stream="write_stream_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1.AppendRowsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = client.append_rows(requests=request_generator()) + + # Handle the response + for response in stream: + print(response) + + Args: + requests (Iterator[google.cloud.bigquery_storage_v1.types.AppendRowsRequest]): + The request object iterator. Request message for ``AppendRows``. + + Because AppendRows is a bidirectional streaming RPC, + certain parts of the AppendRowsRequest need only be + specified for the first request before switching table + destinations. You can also switch table destinations + within the same connection for the default stream. + + The size of a single AppendRowsRequest must be less than + 10 MB in size. Requests larger than this return an + error, typically ``INVALID_ARGUMENT``. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + Iterable[google.cloud.bigquery_storage_v1.types.AppendRowsResponse]: + Response message for AppendRows. + """ + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.append_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + requests, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def get_write_stream(self, + request: Optional[Union[storage.GetWriteStreamRequest, dict]] = None, + *, + name: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Gets information about a write stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + def sample_get_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.GetWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = client.get_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1.types.GetWriteStreamRequest, dict]): + The request object. Request message for ``GetWriteStreamRequest``. + name (str): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.GetWriteStreamRequest): + request = storage.GetWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.get_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def finalize_write_stream(self, + request: Optional[Union[storage.FinalizeWriteStreamRequest, dict]] = None, + *, + name: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FinalizeWriteStreamResponse: + r"""Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + def sample_finalize_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.FinalizeWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = client.finalize_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamRequest, dict]): + The request object. Request message for invoking ``FinalizeWriteStream``. + name (str): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamResponse: + Response message for FinalizeWriteStream. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.FinalizeWriteStreamRequest): + request = storage.FinalizeWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.finalize_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def batch_commit_write_streams(self, + request: Optional[Union[storage.BatchCommitWriteStreamsRequest, dict]] = None, + *, + parent: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.BatchCommitWriteStreamsResponse: + r"""Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. + + Streams must be finalized before commit and cannot be committed + multiple times. Once a stream is committed, data in the stream + becomes available for read operations. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + def sample_batch_commit_write_streams(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.BatchCommitWriteStreamsRequest( + parent="parent_value", + write_streams=['write_streams_value1', 'write_streams_value2'], + ) + + # Make the request + response = client.batch_commit_write_streams(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsRequest, dict]): + The request object. Request message for ``BatchCommitWriteStreams``. + parent (str): + Required. Parent table that all the streams should + belong to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsResponse: + Response message for BatchCommitWriteStreams. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.BatchCommitWriteStreamsRequest): + request = storage.BatchCommitWriteStreamsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.batch_commit_write_streams] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def flush_rows(self, + request: Optional[Union[storage.FlushRowsRequest, dict]] = None, + *, + write_stream: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FlushRowsResponse: + r"""Flushes rows to a BUFFERED stream. + + If users are appending rows to BUFFERED stream, flush operation + is required in order for the rows to become available for + reading. A Flush operation flushes up to any previously flushed + offset in a BUFFERED stream, to the offset specified in the + request. + + Flush is not supported on the \_default stream, since it is not + BUFFERED. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1 + + def sample_flush_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.FlushRowsRequest( + write_stream="write_stream_value", + ) + + # Make the request + response = client.flush_rows(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1.types.FlushRowsRequest, dict]): + The request object. Request message for ``FlushRows``. + write_stream (str): + Required. The stream that is the + target of the flush operation. + + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1.types.FlushRowsResponse: + Respond message for FlushRows. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([write_stream]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.FlushRowsRequest): + request = storage.FlushRowsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.flush_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("write_stream", request.write_stream), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def __enter__(self) -> "BigQueryWriteClient": + return self + + def __exit__(self, type, value, traceback): + """Releases underlying transport's resources. + + .. warning:: + ONLY use as a context manager if the transport is NOT shared + with other clients! Exiting the with block will CLOSE the transport + and may cause errors in other clients! + """ + self.transport.close() + + + + + + + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +__all__ = ( + "BigQueryWriteClient", +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/README.rst b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/README.rst new file mode 100644 index 00000000..d2db1fed --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/README.rst @@ -0,0 +1,9 @@ + +transport inheritance structure +_______________________________ + +`BigQueryWriteTransport` is the ABC for all transports. +- public child `BigQueryWriteGrpcTransport` for sync gRPC transport (defined in `grpc.py`). +- public child `BigQueryWriteGrpcAsyncIOTransport` for async gRPC transport (defined in `grpc_asyncio.py`). +- private child `_BaseBigQueryWriteRestTransport` for base REST transport with inner classes `_BaseMETHOD` (defined in `rest_base.py`). +- public child `BigQueryWriteRestTransport` for sync REST transport with inner classes `METHOD` derived from the parent's corresponding `_BaseMETHOD` classes (defined in `rest.py`). diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/__init__.py new file mode 100644 index 00000000..56e33fc5 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from typing import Dict, Type + +from .base import BigQueryWriteTransport +from .grpc import BigQueryWriteGrpcTransport +from .grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport + + +# Compile a registry of transports. +_transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryWriteTransport]] +_transport_registry['grpc'] = BigQueryWriteGrpcTransport +_transport_registry['grpc_asyncio'] = BigQueryWriteGrpcAsyncIOTransport + +__all__ = ( + 'BigQueryWriteTransport', + 'BigQueryWriteGrpcTransport', + 'BigQueryWriteGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/base.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/base.py new file mode 100644 index 00000000..2e5d1e02 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/base.py @@ -0,0 +1,291 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union + +from google.cloud.bigquery_storage_v1 import gapic_version as package_version + +import google.auth # type: ignore +import google.api_core +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +class BigQueryWriteTransport(abc.ABC): + """Abstract transport class for BigQueryWrite.""" + + AUTH_SCOPES = ( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', + ) + + DEFAULT_HOST: str = 'bigquerystorage.googleapis.com' + def __init__( + self, *, + host: str = DEFAULT_HOST, + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + **kwargs, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A list of scopes. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + """ + + scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES} + + # Save the scopes. + self._scopes = scopes + if not hasattr(self, "_ignore_credentials"): + self._ignore_credentials: bool = False + + # If no credentials are provided, then determine the appropriate + # defaults. + if credentials and credentials_file: + raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive") + + if credentials_file is not None: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, + **scopes_kwargs, + quota_project_id=quota_project_id + ) + elif credentials is None and not self._ignore_credentials: + credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id) + # Don't apply audience if the credentials file passed from user. + if hasattr(credentials, "with_gdch_audience"): + credentials = credentials.with_gdch_audience(api_audience if api_audience else host) + + # If the credentials are service account credentials, then always try to use self signed JWT. + if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"): + credentials = credentials.with_always_use_jwt_access(True) + + # Save the credentials. + self._credentials = credentials + + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ':' not in host: + host += ':443' + self._host = host + + @property + def host(self): + return self._host + + def _prep_wrapped_messages(self, client_info): + # Precompute the wrapped methods. + self._wrapped_methods = { + self.create_write_stream: gapic_v1.method.wrap_method( + self.create_write_stream, + default_retry=retries.Retry( + initial=10.0, + maximum=120.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=1200.0, + ), + default_timeout=1200.0, + client_info=client_info, + ), + self.append_rows: gapic_v1.method.wrap_method( + self.append_rows, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.get_write_stream: gapic_v1.method.wrap_method( + self.get_write_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.finalize_write_stream: gapic_v1.method.wrap_method( + self.finalize_write_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.batch_commit_write_streams: gapic_v1.method.wrap_method( + self.batch_commit_write_streams, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.flush_rows: gapic_v1.method.wrap_method( + self.flush_rows, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + def close(self): + """Closes resources associated with the transport. + + .. warning:: + Only call this method if the transport is NOT shared + with other clients - this may cause errors in other clients! + """ + raise NotImplementedError() + + @property + def create_write_stream(self) -> Callable[ + [storage.CreateWriteStreamRequest], + Union[ + stream.WriteStream, + Awaitable[stream.WriteStream] + ]]: + raise NotImplementedError() + + @property + def append_rows(self) -> Callable[ + [storage.AppendRowsRequest], + Union[ + storage.AppendRowsResponse, + Awaitable[storage.AppendRowsResponse] + ]]: + raise NotImplementedError() + + @property + def get_write_stream(self) -> Callable[ + [storage.GetWriteStreamRequest], + Union[ + stream.WriteStream, + Awaitable[stream.WriteStream] + ]]: + raise NotImplementedError() + + @property + def finalize_write_stream(self) -> Callable[ + [storage.FinalizeWriteStreamRequest], + Union[ + storage.FinalizeWriteStreamResponse, + Awaitable[storage.FinalizeWriteStreamResponse] + ]]: + raise NotImplementedError() + + @property + def batch_commit_write_streams(self) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + Union[ + storage.BatchCommitWriteStreamsResponse, + Awaitable[storage.BatchCommitWriteStreamsResponse] + ]]: + raise NotImplementedError() + + @property + def flush_rows(self) -> Callable[ + [storage.FlushRowsRequest], + Union[ + storage.FlushRowsResponse, + Awaitable[storage.FlushRowsResponse] + ]]: + raise NotImplementedError() + + @property + def kind(self) -> str: + raise NotImplementedError() + + +__all__ = ( + 'BigQueryWriteTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc.py new file mode 100644 index 00000000..da608900 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc.py @@ -0,0 +1,461 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import grpc_helpers +from google.api_core import gapic_v1 +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from .base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO + + +class BigQueryWriteGrpcTransport(BigQueryWriteTransport): + """gRPC backend transport for BigQueryWrite. + + BigQuery Write API. + + The Write API can be used to write data to BigQuery. + + For supplementary information about the Write API, see: + + https://cloud.google.com/bigquery/docs/write-api + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + _stubs: Dict[str, Callable] + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]] = None, + api_mtls_endpoint: Optional[str] = None, + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + ssl_channel_credentials: Optional[grpc.ChannelCredentials] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if a ``channel`` instance is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if a ``channel`` instance is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if a ``channel`` instance is provided. + channel (Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]]): + A ``Channel`` instance through which to make calls, or a Callable + that constructs and returns one. If set to None, ``self.create_channel`` + is used to create the channel. If a Callable is given, it will be called + with the same arguments as used in ``self.create_channel``. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or application default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for the grpc channel. It is ignored if a ``channel`` instance is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if a ``channel`` instance or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if isinstance(channel, grpc.Channel): + # Ignore credentials if a channel was passed. + credentials = None + self._ignore_credentials = True + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, + ) + + if not self._grpc_channel: + # initialize with the provided callable or the default channel + channel_init = channel or type(self).create_channel + self._grpc_channel = channel_init( + self._host, + # use the credentials which are saved + credentials=self._credentials, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> grpc.Channel: + """Create and return a gRPC channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + grpc.Channel: A gRPC channel object. + + Raises: + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + + return grpc_helpers.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + @property + def grpc_channel(self) -> grpc.Channel: + """Return the channel designed to connect to this service. + """ + return self._grpc_channel + + @property + def create_write_stream(self) -> Callable[ + [storage.CreateWriteStreamRequest], + stream.WriteStream]: + r"""Return a callable for the create write stream method over gRPC. + + Creates a write stream to the given table. Additionally, every + table has a special stream named '_default' to which data can be + written. This stream doesn't need to be created using + CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Returns: + Callable[[~.CreateWriteStreamRequest], + ~.WriteStream]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_write_stream' not in self._stubs: + self._stubs['create_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/CreateWriteStream', + request_serializer=storage.CreateWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['create_write_stream'] + + @property + def append_rows(self) -> Callable[ + [storage.AppendRowsRequest], + storage.AppendRowsResponse]: + r"""Return a callable for the append rows method over gRPC. + + Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provides an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC connection. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains an optional offset at which the append + happened. No offset information will be returned for appends to + a default stream. + + Responses are received in the same order in which requests are + sent. There will be one response for each successful inserted + request. Responses may optionally embed error information if the + originating AppendRequest was not successfully processed. + + The specifics of when successfully appended data is made visible + to the table are governed by the type of stream: + + - For COMMITTED streams (which includes the default stream), + data is visible immediately upon successful append. + + - For BUFFERED streams, data is made visible via a subsequent + ``FlushRows`` rpc which advances a cursor to a newer offset + in the stream. + + - For PENDING streams, data is not made visible until the + stream itself is finalized (via the ``FinalizeWriteStream`` + rpc), and the stream is explicitly committed via the + ``BatchCommitWriteStreams`` rpc. + + Returns: + Callable[[~.AppendRowsRequest], + ~.AppendRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'append_rows' not in self._stubs: + self._stubs['append_rows'] = self.grpc_channel.stream_stream( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/AppendRows', + request_serializer=storage.AppendRowsRequest.serialize, + response_deserializer=storage.AppendRowsResponse.deserialize, + ) + return self._stubs['append_rows'] + + @property + def get_write_stream(self) -> Callable[ + [storage.GetWriteStreamRequest], + stream.WriteStream]: + r"""Return a callable for the get write stream method over gRPC. + + Gets information about a write stream. + + Returns: + Callable[[~.GetWriteStreamRequest], + ~.WriteStream]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'get_write_stream' not in self._stubs: + self._stubs['get_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/GetWriteStream', + request_serializer=storage.GetWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['get_write_stream'] + + @property + def finalize_write_stream(self) -> Callable[ + [storage.FinalizeWriteStreamRequest], + storage.FinalizeWriteStreamResponse]: + r"""Return a callable for the finalize write stream method over gRPC. + + Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Returns: + Callable[[~.FinalizeWriteStreamRequest], + ~.FinalizeWriteStreamResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'finalize_write_stream' not in self._stubs: + self._stubs['finalize_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/FinalizeWriteStream', + request_serializer=storage.FinalizeWriteStreamRequest.serialize, + response_deserializer=storage.FinalizeWriteStreamResponse.deserialize, + ) + return self._stubs['finalize_write_stream'] + + @property + def batch_commit_write_streams(self) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + storage.BatchCommitWriteStreamsResponse]: + r"""Return a callable for the batch commit write streams method over gRPC. + + Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. + + Streams must be finalized before commit and cannot be committed + multiple times. Once a stream is committed, data in the stream + becomes available for read operations. + + Returns: + Callable[[~.BatchCommitWriteStreamsRequest], + ~.BatchCommitWriteStreamsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_commit_write_streams' not in self._stubs: + self._stubs['batch_commit_write_streams'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/BatchCommitWriteStreams', + request_serializer=storage.BatchCommitWriteStreamsRequest.serialize, + response_deserializer=storage.BatchCommitWriteStreamsResponse.deserialize, + ) + return self._stubs['batch_commit_write_streams'] + + @property + def flush_rows(self) -> Callable[ + [storage.FlushRowsRequest], + storage.FlushRowsResponse]: + r"""Return a callable for the flush rows method over gRPC. + + Flushes rows to a BUFFERED stream. + + If users are appending rows to BUFFERED stream, flush operation + is required in order for the rows to become available for + reading. A Flush operation flushes up to any previously flushed + offset in a BUFFERED stream, to the offset specified in the + request. + + Flush is not supported on the \_default stream, since it is not + BUFFERED. + + Returns: + Callable[[~.FlushRowsRequest], + ~.FlushRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'flush_rows' not in self._stubs: + self._stubs['flush_rows'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/FlushRows', + request_serializer=storage.FlushRowsRequest.serialize, + response_deserializer=storage.FlushRowsResponse.deserialize, + ) + return self._stubs['flush_rows'] + + def close(self): + self.grpc_channel.close() + + @property + def kind(self) -> str: + return "grpc" + + +__all__ = ( + 'BigQueryWriteGrpcTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc_asyncio.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc_asyncio.py new file mode 100644 index 00000000..e077b652 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/services/big_query_write/transports/grpc_asyncio.py @@ -0,0 +1,571 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import inspect +import warnings +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers_async +from google.api_core import exceptions as core_exceptions +from google.api_core import retry_async as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore +from grpc.experimental import aio # type: ignore + +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from .base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .grpc import BigQueryWriteGrpcTransport + + +class BigQueryWriteGrpcAsyncIOTransport(BigQueryWriteTransport): + """gRPC AsyncIO backend transport for BigQueryWrite. + + BigQuery Write API. + + The Write API can be used to write data to BigQuery. + + For supplementary information about the Write API, see: + + https://cloud.google.com/bigquery/docs/write-api + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + + _grpc_channel: aio.Channel + _stubs: Dict[str, Callable] = {} + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> aio.Channel: + """Create and return a gRPC AsyncIO channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + aio.Channel: A gRPC AsyncIO channel object. + """ + + return grpc_helpers_async.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: Optional[Union[aio.Channel, Callable[..., aio.Channel]]] = None, + api_mtls_endpoint: Optional[str] = None, + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + ssl_channel_credentials: Optional[grpc.ChannelCredentials] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if a ``channel`` instance is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if a ``channel`` instance is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + channel (Optional[Union[aio.Channel, Callable[..., aio.Channel]]]): + A ``Channel`` instance through which to make calls, or a Callable + that constructs and returns one. If set to None, ``self.create_channel`` + is used to create the channel. If a Callable is given, it will be called + with the same arguments as used in ``self.create_channel``. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or application default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for the grpc channel. It is ignored if a ``channel`` instance is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if a ``channel`` instance or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if isinstance(channel, aio.Channel): + # Ignore credentials if a channel was passed. + credentials = None + self._ignore_credentials = True + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, + ) + + if not self._grpc_channel: + # initialize with the provided callable or the default channel + channel_init = channel or type(self).create_channel + self._grpc_channel = channel_init( + self._host, + # use the credentials which are saved + credentials=self._credentials, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._wrap_with_kind = "kind" in inspect.signature(gapic_v1.method_async.wrap_method).parameters + self._prep_wrapped_messages(client_info) + + @property + def grpc_channel(self) -> aio.Channel: + """Create the channel designed to connect to this service. + + This property caches on the instance; repeated calls return + the same channel. + """ + # Return the channel from cache. + return self._grpc_channel + + @property + def create_write_stream(self) -> Callable[ + [storage.CreateWriteStreamRequest], + Awaitable[stream.WriteStream]]: + r"""Return a callable for the create write stream method over gRPC. + + Creates a write stream to the given table. Additionally, every + table has a special stream named '_default' to which data can be + written. This stream doesn't need to be created using + CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Returns: + Callable[[~.CreateWriteStreamRequest], + Awaitable[~.WriteStream]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_write_stream' not in self._stubs: + self._stubs['create_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/CreateWriteStream', + request_serializer=storage.CreateWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['create_write_stream'] + + @property + def append_rows(self) -> Callable[ + [storage.AppendRowsRequest], + Awaitable[storage.AppendRowsResponse]]: + r"""Return a callable for the append rows method over gRPC. + + Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provides an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC connection. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains an optional offset at which the append + happened. No offset information will be returned for appends to + a default stream. + + Responses are received in the same order in which requests are + sent. There will be one response for each successful inserted + request. Responses may optionally embed error information if the + originating AppendRequest was not successfully processed. + + The specifics of when successfully appended data is made visible + to the table are governed by the type of stream: + + - For COMMITTED streams (which includes the default stream), + data is visible immediately upon successful append. + + - For BUFFERED streams, data is made visible via a subsequent + ``FlushRows`` rpc which advances a cursor to a newer offset + in the stream. + + - For PENDING streams, data is not made visible until the + stream itself is finalized (via the ``FinalizeWriteStream`` + rpc), and the stream is explicitly committed via the + ``BatchCommitWriteStreams`` rpc. + + Returns: + Callable[[~.AppendRowsRequest], + Awaitable[~.AppendRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'append_rows' not in self._stubs: + self._stubs['append_rows'] = self.grpc_channel.stream_stream( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/AppendRows', + request_serializer=storage.AppendRowsRequest.serialize, + response_deserializer=storage.AppendRowsResponse.deserialize, + ) + return self._stubs['append_rows'] + + @property + def get_write_stream(self) -> Callable[ + [storage.GetWriteStreamRequest], + Awaitable[stream.WriteStream]]: + r"""Return a callable for the get write stream method over gRPC. + + Gets information about a write stream. + + Returns: + Callable[[~.GetWriteStreamRequest], + Awaitable[~.WriteStream]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'get_write_stream' not in self._stubs: + self._stubs['get_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/GetWriteStream', + request_serializer=storage.GetWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['get_write_stream'] + + @property + def finalize_write_stream(self) -> Callable[ + [storage.FinalizeWriteStreamRequest], + Awaitable[storage.FinalizeWriteStreamResponse]]: + r"""Return a callable for the finalize write stream method over gRPC. + + Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Returns: + Callable[[~.FinalizeWriteStreamRequest], + Awaitable[~.FinalizeWriteStreamResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'finalize_write_stream' not in self._stubs: + self._stubs['finalize_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/FinalizeWriteStream', + request_serializer=storage.FinalizeWriteStreamRequest.serialize, + response_deserializer=storage.FinalizeWriteStreamResponse.deserialize, + ) + return self._stubs['finalize_write_stream'] + + @property + def batch_commit_write_streams(self) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + Awaitable[storage.BatchCommitWriteStreamsResponse]]: + r"""Return a callable for the batch commit write streams method over gRPC. + + Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. + + Streams must be finalized before commit and cannot be committed + multiple times. Once a stream is committed, data in the stream + becomes available for read operations. + + Returns: + Callable[[~.BatchCommitWriteStreamsRequest], + Awaitable[~.BatchCommitWriteStreamsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_commit_write_streams' not in self._stubs: + self._stubs['batch_commit_write_streams'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/BatchCommitWriteStreams', + request_serializer=storage.BatchCommitWriteStreamsRequest.serialize, + response_deserializer=storage.BatchCommitWriteStreamsResponse.deserialize, + ) + return self._stubs['batch_commit_write_streams'] + + @property + def flush_rows(self) -> Callable[ + [storage.FlushRowsRequest], + Awaitable[storage.FlushRowsResponse]]: + r"""Return a callable for the flush rows method over gRPC. + + Flushes rows to a BUFFERED stream. + + If users are appending rows to BUFFERED stream, flush operation + is required in order for the rows to become available for + reading. A Flush operation flushes up to any previously flushed + offset in a BUFFERED stream, to the offset specified in the + request. + + Flush is not supported on the \_default stream, since it is not + BUFFERED. + + Returns: + Callable[[~.FlushRowsRequest], + Awaitable[~.FlushRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'flush_rows' not in self._stubs: + self._stubs['flush_rows'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1.BigQueryWrite/FlushRows', + request_serializer=storage.FlushRowsRequest.serialize, + response_deserializer=storage.FlushRowsResponse.deserialize, + ) + return self._stubs['flush_rows'] + + def _prep_wrapped_messages(self, client_info): + """ Precompute the wrapped methods, overriding the base class method to use async wrappers.""" + self._wrapped_methods = { + self.create_write_stream: self._wrap_method( + self.create_write_stream, + default_retry=retries.AsyncRetry( + initial=10.0, + maximum=120.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=1200.0, + ), + default_timeout=1200.0, + client_info=client_info, + ), + self.append_rows: self._wrap_method( + self.append_rows, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.get_write_stream: self._wrap_method( + self.get_write_stream, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.finalize_write_stream: self._wrap_method( + self.finalize_write_stream, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.batch_commit_write_streams: self._wrap_method( + self.batch_commit_write_streams, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.flush_rows: self._wrap_method( + self.flush_rows, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + def _wrap_method(self, func, *args, **kwargs): + if self._wrap_with_kind: # pragma: NO COVER + kwargs["kind"] = self.kind + return gapic_v1.method_async.wrap_method(func, *args, **kwargs) + + def close(self): + return self.grpc_channel.close() + + @property + def kind(self) -> str: + return "grpc_asyncio" + + +__all__ = ( + 'BigQueryWriteGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/__init__.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/__init__.py new file mode 100644 index 00000000..ddf4cac7 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/__init__.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .arrow import ( + ArrowRecordBatch, + ArrowSchema, + ArrowSerializationOptions, +) +from .avro import ( + AvroRows, + AvroSchema, + AvroSerializationOptions, +) +from .protobuf import ( + ProtoRows, + ProtoSchema, +) +from .storage import ( + AppendRowsRequest, + AppendRowsResponse, + BatchCommitWriteStreamsRequest, + BatchCommitWriteStreamsResponse, + CreateReadSessionRequest, + CreateWriteStreamRequest, + FinalizeWriteStreamRequest, + FinalizeWriteStreamResponse, + FlushRowsRequest, + FlushRowsResponse, + GetWriteStreamRequest, + ReadRowsRequest, + ReadRowsResponse, + RowError, + SplitReadStreamRequest, + SplitReadStreamResponse, + StorageError, + StreamStats, + ThrottleState, +) +from .stream import ( + ReadSession, + ReadStream, + WriteStream, + DataFormat, + WriteStreamView, +) +from .table import ( + TableFieldSchema, + TableSchema, +) + +__all__ = ( + 'ArrowRecordBatch', + 'ArrowSchema', + 'ArrowSerializationOptions', + 'AvroRows', + 'AvroSchema', + 'AvroSerializationOptions', + 'ProtoRows', + 'ProtoSchema', + 'AppendRowsRequest', + 'AppendRowsResponse', + 'BatchCommitWriteStreamsRequest', + 'BatchCommitWriteStreamsResponse', + 'CreateReadSessionRequest', + 'CreateWriteStreamRequest', + 'FinalizeWriteStreamRequest', + 'FinalizeWriteStreamResponse', + 'FlushRowsRequest', + 'FlushRowsResponse', + 'GetWriteStreamRequest', + 'ReadRowsRequest', + 'ReadRowsResponse', + 'RowError', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + 'StorageError', + 'StreamStats', + 'ThrottleState', + 'ReadSession', + 'ReadStream', + 'WriteStream', + 'DataFormat', + 'WriteStreamView', + 'TableFieldSchema', + 'TableSchema', +) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/annotations.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/annotations.py new file mode 100644 index 00000000..c29eeed5 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/annotations.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1', + manifest={ + }, +) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/arrow.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/arrow.py new file mode 100644 index 00000000..ed013168 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/arrow.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1', + manifest={ + 'ArrowSchema', + 'ArrowRecordBatch', + 'ArrowSerializationOptions', + }, +) + + +class ArrowSchema(proto.Message): + r"""Arrow schema as specified in + https://arrow.apache.org/docs/python/api/datatypes.html and + serialized to bytes using IPC: + + https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc + + See code samples on how this message can be deserialized. + + Attributes: + serialized_schema (bytes): + IPC serialized Arrow schema. + """ + + serialized_schema: bytes = proto.Field( + proto.BYTES, + number=1, + ) + + +class ArrowRecordBatch(proto.Message): + r"""Arrow RecordBatch. + + Attributes: + serialized_record_batch (bytes): + IPC-serialized Arrow RecordBatch. + row_count (int): + [Deprecated] The count of rows in + ``serialized_record_batch``. Please use the + format-independent ReadRowsResponse.row_count instead. + """ + + serialized_record_batch: bytes = proto.Field( + proto.BYTES, + number=1, + ) + row_count: int = proto.Field( + proto.INT64, + number=2, + ) + + +class ArrowSerializationOptions(proto.Message): + r"""Contains options specific to Arrow Serialization. + + Attributes: + buffer_compression (google.cloud.bigquery_storage_v1.types.ArrowSerializationOptions.CompressionCodec): + The compression codec to use for Arrow + buffers in serialized record batches. + """ + class CompressionCodec(proto.Enum): + r"""Compression codec's supported by Arrow. + + Values: + COMPRESSION_UNSPECIFIED (0): + If unspecified no compression will be used. + LZ4_FRAME (1): + LZ4 Frame + (https://github.com/lz4/lz4/blob/dev/doc/lz4_Frame_format.md) + ZSTD (2): + Zstandard compression. + """ + COMPRESSION_UNSPECIFIED = 0 + LZ4_FRAME = 1 + ZSTD = 2 + + buffer_compression: CompressionCodec = proto.Field( + proto.ENUM, + number=2, + enum=CompressionCodec, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/avro.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/avro.py new file mode 100644 index 00000000..72cad0e7 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/avro.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1', + manifest={ + 'AvroSchema', + 'AvroRows', + 'AvroSerializationOptions', + }, +) + + +class AvroSchema(proto.Message): + r"""Avro schema. + + Attributes: + schema (str): + Json serialized schema, as described at + https://avro.apache.org/docs/1.8.1/spec.html. + """ + + schema: str = proto.Field( + proto.STRING, + number=1, + ) + + +class AvroRows(proto.Message): + r"""Avro rows. + + Attributes: + serialized_binary_rows (bytes): + Binary serialized rows in a block. + row_count (int): + [Deprecated] The count of rows in the returning block. + Please use the format-independent ReadRowsResponse.row_count + instead. + """ + + serialized_binary_rows: bytes = proto.Field( + proto.BYTES, + number=1, + ) + row_count: int = proto.Field( + proto.INT64, + number=2, + ) + + +class AvroSerializationOptions(proto.Message): + r"""Contains options specific to Avro Serialization. + + Attributes: + enable_display_name_attribute (bool): + Enable displayName attribute in Avro schema. + + The Avro specification requires field names to + be alphanumeric. By default, in cases when + column names do not conform to these + requirements (e.g. non-ascii unicode codepoints) + and Avro is requested as an output format, the + CreateReadSession call will fail. + + Setting this field to true, populates avro field + names with a placeholder value and populates a + "displayName" attribute for every avro field + with the original column name. + """ + + enable_display_name_attribute: bool = proto.Field( + proto.BOOL, + number=1, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/protobuf.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/protobuf.py new file mode 100644 index 00000000..2d5e24cb --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/protobuf.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + +from google.protobuf import descriptor_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1', + manifest={ + 'ProtoSchema', + 'ProtoRows', + }, +) + + +class ProtoSchema(proto.Message): + r"""ProtoSchema describes the schema of the serialized protocol + buffer data rows. + + Attributes: + proto_descriptor (google.protobuf.descriptor_pb2.DescriptorProto): + Descriptor for input message. The provided descriptor must + be self contained, such that data rows sent can be fully + decoded using only the single descriptor. For data rows that + are compositions of multiple independent messages, this + means the descriptor may need to be transformed to only use + nested types: + https://developers.google.com/protocol-buffers/docs/proto#nested + + For additional information for how proto types and values + map onto BigQuery see: + https://cloud.google.com/bigquery/docs/write-api#data_type_conversions + """ + + proto_descriptor: descriptor_pb2.DescriptorProto = proto.Field( + proto.MESSAGE, + number=1, + message=descriptor_pb2.DescriptorProto, + ) + + +class ProtoRows(proto.Message): + r""" + + Attributes: + serialized_rows (MutableSequence[bytes]): + A sequence of rows serialized as a Protocol + Buffer. + See + https://developers.google.com/protocol-buffers/docs/overview + for more information on deserializing this + field. + """ + + serialized_rows: MutableSequence[bytes] = proto.RepeatedField( + proto.BYTES, + number=1, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/storage.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/storage.py new file mode 100644 index 00000000..28f67326 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/storage.py @@ -0,0 +1,984 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + +from google.cloud.bigquery_storage_v1.types import arrow +from google.cloud.bigquery_storage_v1.types import avro +from google.cloud.bigquery_storage_v1.types import protobuf +from google.cloud.bigquery_storage_v1.types import stream +from google.cloud.bigquery_storage_v1.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.protobuf import wrappers_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1', + manifest={ + 'CreateReadSessionRequest', + 'ReadRowsRequest', + 'ThrottleState', + 'StreamStats', + 'ReadRowsResponse', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + 'CreateWriteStreamRequest', + 'AppendRowsRequest', + 'AppendRowsResponse', + 'GetWriteStreamRequest', + 'BatchCommitWriteStreamsRequest', + 'BatchCommitWriteStreamsResponse', + 'FinalizeWriteStreamRequest', + 'FinalizeWriteStreamResponse', + 'FlushRowsRequest', + 'FlushRowsResponse', + 'StorageError', + 'RowError', + }, +) + + +class CreateReadSessionRequest(proto.Message): + r"""Request message for ``CreateReadSession``. + + Attributes: + parent (str): + Required. The request project that owns the session, in the + form of ``projects/{project_id}``. + read_session (google.cloud.bigquery_storage_v1.types.ReadSession): + Required. Session to be created. + max_stream_count (int): + Max initial number of streams. If unset or zero, the server + will provide a value of streams so as to produce reasonable + throughput. Must be non-negative. The number of streams may + be lower than the requested number, depending on the amount + parallelism that is reasonable for the table. There is a + default system max limit of 1,000. + + This must be greater than or equal to + preferred_min_stream_count. Typically, clients should either + leave this unset to let the system to determine an upper + bound OR set this a size for the maximum "units of work" it + can gracefully handle. + preferred_min_stream_count (int): + The minimum preferred stream count. This + parameter can be used to inform the service that + there is a desired lower bound on the number of + streams. This is typically a target parallelism + of the client (e.g. a Spark cluster with + N-workers would set this to a low multiple of N + to ensure good cluster utilization). + + The system will make a best effort to provide at + least this number of streams, but in some cases + might provide less. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + read_session: stream.ReadSession = proto.Field( + proto.MESSAGE, + number=2, + message=stream.ReadSession, + ) + max_stream_count: int = proto.Field( + proto.INT32, + number=3, + ) + preferred_min_stream_count: int = proto.Field( + proto.INT32, + number=4, + ) + + +class ReadRowsRequest(proto.Message): + r"""Request message for ``ReadRows``. + + Attributes: + read_stream (str): + Required. Stream to read rows from. + offset (int): + The offset requested must be less than the + last row read from Read. Requesting a larger + offset is undefined. If not specified, start + reading from offset zero. + """ + + read_stream: str = proto.Field( + proto.STRING, + number=1, + ) + offset: int = proto.Field( + proto.INT64, + number=2, + ) + + +class ThrottleState(proto.Message): + r"""Information on if the current connection is being throttled. + + Attributes: + throttle_percent (int): + How much this connection is being throttled. + Zero means no throttling, 100 means fully + throttled. + """ + + throttle_percent: int = proto.Field( + proto.INT32, + number=1, + ) + + +class StreamStats(proto.Message): + r"""Estimated stream statistics for a given read Stream. + + Attributes: + progress (google.cloud.bigquery_storage_v1.types.StreamStats.Progress): + Represents the progress of the current + stream. + """ + + class Progress(proto.Message): + r""" + + Attributes: + at_response_start (float): + The fraction of rows assigned to the stream that have been + processed by the server so far, not including the rows in + the current response message. + + This value, along with ``at_response_end``, can be used to + interpolate the progress made as the rows in the message are + being processed using the following formula: + ``at_response_start + (at_response_end - at_response_start) * rows_processed_from_response / rows_in_response``. + + Note that if a filter is provided, the ``at_response_end`` + value of the previous response may not necessarily be equal + to the ``at_response_start`` value of the current response. + at_response_end (float): + Similar to ``at_response_start``, except that this value + includes the rows in the current response. + """ + + at_response_start: float = proto.Field( + proto.DOUBLE, + number=1, + ) + at_response_end: float = proto.Field( + proto.DOUBLE, + number=2, + ) + + progress: Progress = proto.Field( + proto.MESSAGE, + number=2, + message=Progress, + ) + + +class ReadRowsResponse(proto.Message): + r"""Response from calling ``ReadRows`` may include row data, progress + and throttling information. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + avro_rows (google.cloud.bigquery_storage_v1.types.AvroRows): + Serialized row data in AVRO format. + + This field is a member of `oneof`_ ``rows``. + arrow_record_batch (google.cloud.bigquery_storage_v1.types.ArrowRecordBatch): + Serialized row data in Arrow RecordBatch + format. + + This field is a member of `oneof`_ ``rows``. + row_count (int): + Number of serialized rows in the rows block. + stats (google.cloud.bigquery_storage_v1.types.StreamStats): + Statistics for the stream. + throttle_state (google.cloud.bigquery_storage_v1.types.ThrottleState): + Throttling state. If unset, the latest + response still describes the current throttling + status. + avro_schema (google.cloud.bigquery_storage_v1.types.AvroSchema): + Output only. Avro schema. + + This field is a member of `oneof`_ ``schema``. + arrow_schema (google.cloud.bigquery_storage_v1.types.ArrowSchema): + Output only. Arrow schema. + + This field is a member of `oneof`_ ``schema``. + uncompressed_byte_size (int): + Optional. If the row data in this ReadRowsResponse is + compressed, then uncompressed byte size is the original size + of the uncompressed row data. If it is set to a value + greater than 0, then decompress into a buffer of size + uncompressed_byte_size using the compression codec that was + requested during session creation time and which is + specified in TableReadOptions.response_compression_codec in + ReadSession. This value is not set if no + response_compression_codec was not requested and it is -1 if + the requested compression would not have reduced the size of + this ReadRowsResponse's row data. This attempts to match + Apache Arrow's behavior described here + https://github.com/apache/arrow/issues/15102 where the + uncompressed length may be set to -1 to indicate that the + data that follows is not compressed, which can be useful for + cases where compression does not yield appreciable savings. + When uncompressed_byte_size is not greater than 0, the + client should skip decompression. + + This field is a member of `oneof`_ ``_uncompressed_byte_size``. + """ + + avro_rows: avro.AvroRows = proto.Field( + proto.MESSAGE, + number=3, + oneof='rows', + message=avro.AvroRows, + ) + arrow_record_batch: arrow.ArrowRecordBatch = proto.Field( + proto.MESSAGE, + number=4, + oneof='rows', + message=arrow.ArrowRecordBatch, + ) + row_count: int = proto.Field( + proto.INT64, + number=6, + ) + stats: 'StreamStats' = proto.Field( + proto.MESSAGE, + number=2, + message='StreamStats', + ) + throttle_state: 'ThrottleState' = proto.Field( + proto.MESSAGE, + number=5, + message='ThrottleState', + ) + avro_schema: avro.AvroSchema = proto.Field( + proto.MESSAGE, + number=7, + oneof='schema', + message=avro.AvroSchema, + ) + arrow_schema: arrow.ArrowSchema = proto.Field( + proto.MESSAGE, + number=8, + oneof='schema', + message=arrow.ArrowSchema, + ) + uncompressed_byte_size: int = proto.Field( + proto.INT64, + number=9, + optional=True, + ) + + +class SplitReadStreamRequest(proto.Message): + r"""Request message for ``SplitReadStream``. + + Attributes: + name (str): + Required. Name of the stream to split. + fraction (float): + A value in the range (0.0, 1.0) that + specifies the fractional point at which the + original stream should be split. The actual + split point is evaluated on pre-filtered rows, + so if a filter is provided, then there is no + guarantee that the division of the rows between + the new child streams will be proportional to + this fractional value. Additionally, because the + server-side unit for assigning data is + collections of rows, this fraction will always + map to a data storage boundary on the server + side. + """ + + name: str = proto.Field( + proto.STRING, + number=1, + ) + fraction: float = proto.Field( + proto.DOUBLE, + number=2, + ) + + +class SplitReadStreamResponse(proto.Message): + r"""Response message for ``SplitReadStream``. + + Attributes: + primary_stream (google.cloud.bigquery_storage_v1.types.ReadStream): + Primary stream, which contains the beginning portion of + \|original_stream|. An empty value indicates that the + original stream can no longer be split. + remainder_stream (google.cloud.bigquery_storage_v1.types.ReadStream): + Remainder stream, which contains the tail of + \|original_stream|. An empty value indicates that the + original stream can no longer be split. + """ + + primary_stream: stream.ReadStream = proto.Field( + proto.MESSAGE, + number=1, + message=stream.ReadStream, + ) + remainder_stream: stream.ReadStream = proto.Field( + proto.MESSAGE, + number=2, + message=stream.ReadStream, + ) + + +class CreateWriteStreamRequest(proto.Message): + r"""Request message for ``CreateWriteStream``. + + Attributes: + parent (str): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + write_stream (google.cloud.bigquery_storage_v1.types.WriteStream): + Required. Stream to be created. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + write_stream: stream.WriteStream = proto.Field( + proto.MESSAGE, + number=2, + message=stream.WriteStream, + ) + + +class AppendRowsRequest(proto.Message): + r"""Request message for ``AppendRows``. + + Because AppendRows is a bidirectional streaming RPC, certain parts + of the AppendRowsRequest need only be specified for the first + request before switching table destinations. You can also switch + table destinations within the same connection for the default + stream. + + The size of a single AppendRowsRequest must be less than 10 MB in + size. Requests larger than this return an error, typically + ``INVALID_ARGUMENT``. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + write_stream (str): + Required. The write_stream identifies the append operation. + It must be provided in the following scenarios: + + - In the first request to an AppendRows connection. + + - In all subsequent requests to an AppendRows connection, + if you use the same connection to write to multiple + tables or change the input schema for default streams. + + For explicitly created write streams, the format is: + + - ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{id}`` + + For the special default stream, the format is: + + - ``projects/{project}/datasets/{dataset}/tables/{table}/streams/_default``. + + An example of a possible sequence of requests with + write_stream fields within a single connection: + + - r1: {write_stream: stream_name_1} + + - r2: {write_stream: /*omit*/} + + - r3: {write_stream: /*omit*/} + + - r4: {write_stream: stream_name_2} + + - r5: {write_stream: stream_name_2} + + The destination changed in request_4, so the write_stream + field must be populated in all subsequent requests in this + stream. + offset (google.protobuf.wrappers_pb2.Int64Value): + If present, the write is only performed if the next append + offset is same as the provided value. If not present, the + write is performed at the current end of stream. Specifying + a value for this field is not allowed when calling + AppendRows for the '_default' stream. + proto_rows (google.cloud.bigquery_storage_v1.types.AppendRowsRequest.ProtoData): + Rows in proto format. + + This field is a member of `oneof`_ ``rows``. + arrow_rows (google.cloud.bigquery_storage_v1.types.AppendRowsRequest.ArrowData): + Rows in arrow format. This is an experimental + feature only selected for allowlisted customers. + + This field is a member of `oneof`_ ``rows``. + trace_id (str): + Id set by client to annotate its identity. + Only initial request setting is respected. + missing_value_interpretations (MutableMapping[str, google.cloud.bigquery_storage_v1.types.AppendRowsRequest.MissingValueInterpretation]): + A map to indicate how to interpret missing value for some + fields. Missing values are fields present in user schema but + missing in rows. The key is the field name. The value is the + interpretation of missing values for the field. + + For example, a map {'foo': NULL_VALUE, 'bar': DEFAULT_VALUE} + means all missing values in field foo are interpreted as + NULL, all missing values in field bar are interpreted as the + default value of field bar in table schema. + + If a field is not in this map and has missing values, the + missing values in this field are interpreted as NULL. + + This field only applies to the current request, it won't + affect other requests on the connection. + + Currently, field name can only be top-level column name, + can't be a struct field path like 'foo.bar'. + default_missing_value_interpretation (google.cloud.bigquery_storage_v1.types.AppendRowsRequest.MissingValueInterpretation): + Optional. Default missing value interpretation for all + columns in the table. When a value is specified on an + ``AppendRowsRequest``, it is applied to all requests on the + connection from that point forward, until a subsequent + ``AppendRowsRequest`` sets it to a different value. + ``missing_value_interpretation`` can override + ``default_missing_value_interpretation``. For example, if + you want to write ``NULL`` instead of using default values + for some columns, you can set + ``default_missing_value_interpretation`` to + ``DEFAULT_VALUE`` and at the same time, set + ``missing_value_interpretations`` to ``NULL_VALUE`` on those + columns. + """ + class MissingValueInterpretation(proto.Enum): + r"""An enum to indicate how to interpret missing values of fields + that are present in user schema but missing in rows. A missing + value can represent a NULL or a column default value defined in + BigQuery table schema. + + Values: + MISSING_VALUE_INTERPRETATION_UNSPECIFIED (0): + Invalid missing value interpretation. + Requests with this value will be rejected. + NULL_VALUE (1): + Missing value is interpreted as NULL. + DEFAULT_VALUE (2): + Missing value is interpreted as column + default value if declared in the table schema, + NULL otherwise. + """ + MISSING_VALUE_INTERPRETATION_UNSPECIFIED = 0 + NULL_VALUE = 1 + DEFAULT_VALUE = 2 + + class ArrowData(proto.Message): + r"""Arrow schema and data. + Arrow format is an experimental feature only selected for + allowlisted customers. + + Attributes: + writer_schema (google.cloud.bigquery_storage_v1.types.ArrowSchema): + Optional. Arrow Schema used to serialize the + data. + rows (google.cloud.bigquery_storage_v1.types.ArrowRecordBatch): + Required. Serialized row data in Arrow + format. + """ + + writer_schema: arrow.ArrowSchema = proto.Field( + proto.MESSAGE, + number=1, + message=arrow.ArrowSchema, + ) + rows: arrow.ArrowRecordBatch = proto.Field( + proto.MESSAGE, + number=2, + message=arrow.ArrowRecordBatch, + ) + + class ProtoData(proto.Message): + r"""ProtoData contains the data rows and schema when constructing + append requests. + + Attributes: + writer_schema (google.cloud.bigquery_storage_v1.types.ProtoSchema): + The protocol buffer schema used to serialize the data. + Provide this value whenever: + + - You send the first request of an RPC connection. + + - You change the input schema. + + - You specify a new destination table. + rows (google.cloud.bigquery_storage_v1.types.ProtoRows): + Serialized row data in protobuf message + format. Currently, the backend expects the + serialized rows to adhere to proto2 semantics + when appending rows, particularly with respect + to how default values are encoded. + """ + + writer_schema: protobuf.ProtoSchema = proto.Field( + proto.MESSAGE, + number=1, + message=protobuf.ProtoSchema, + ) + rows: protobuf.ProtoRows = proto.Field( + proto.MESSAGE, + number=2, + message=protobuf.ProtoRows, + ) + + write_stream: str = proto.Field( + proto.STRING, + number=1, + ) + offset: wrappers_pb2.Int64Value = proto.Field( + proto.MESSAGE, + number=2, + message=wrappers_pb2.Int64Value, + ) + proto_rows: ProtoData = proto.Field( + proto.MESSAGE, + number=4, + oneof='rows', + message=ProtoData, + ) + arrow_rows: ArrowData = proto.Field( + proto.MESSAGE, + number=5, + oneof='rows', + message=ArrowData, + ) + trace_id: str = proto.Field( + proto.STRING, + number=6, + ) + missing_value_interpretations: MutableMapping[str, MissingValueInterpretation] = proto.MapField( + proto.STRING, + proto.ENUM, + number=7, + enum=MissingValueInterpretation, + ) + default_missing_value_interpretation: MissingValueInterpretation = proto.Field( + proto.ENUM, + number=8, + enum=MissingValueInterpretation, + ) + + +class AppendRowsResponse(proto.Message): + r"""Response message for ``AppendRows``. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + append_result (google.cloud.bigquery_storage_v1.types.AppendRowsResponse.AppendResult): + Result if the append is successful. + + This field is a member of `oneof`_ ``response``. + error (google.rpc.status_pb2.Status): + Error returned when problems were encountered. If present, + it indicates rows were not accepted into the system. Users + can retry or continue with other append requests within the + same connection. + + Additional information about error signalling: + + ALREADY_EXISTS: Happens when an append specified an offset, + and the backend already has received data at this offset. + Typically encountered in retry scenarios, and can be + ignored. + + OUT_OF_RANGE: Returned when the specified offset in the + stream is beyond the current end of the stream. + + INVALID_ARGUMENT: Indicates a malformed request or data. + + ABORTED: Request processing is aborted because of prior + failures. The request can be retried if previous failure is + addressed. + + INTERNAL: Indicates server side error(s) that can be + retried. + + This field is a member of `oneof`_ ``response``. + updated_schema (google.cloud.bigquery_storage_v1.types.TableSchema): + If backend detects a schema update, pass it + to user so that user can use it to input new + type of message. It will be empty when no schema + updates have occurred. + row_errors (MutableSequence[google.cloud.bigquery_storage_v1.types.RowError]): + If a request failed due to corrupted rows, no + rows in the batch will be appended. The API will + return row level error info, so that the caller + can remove the bad rows and retry the request. + write_stream (str): + The target of the append operation. Matches the write_stream + in the corresponding request. + """ + + class AppendResult(proto.Message): + r"""AppendResult is returned for successful append requests. + + Attributes: + offset (google.protobuf.wrappers_pb2.Int64Value): + The row offset at which the last append + occurred. The offset will not be set if + appending using default streams. + """ + + offset: wrappers_pb2.Int64Value = proto.Field( + proto.MESSAGE, + number=1, + message=wrappers_pb2.Int64Value, + ) + + append_result: AppendResult = proto.Field( + proto.MESSAGE, + number=1, + oneof='response', + message=AppendResult, + ) + error: status_pb2.Status = proto.Field( + proto.MESSAGE, + number=2, + oneof='response', + message=status_pb2.Status, + ) + updated_schema: table.TableSchema = proto.Field( + proto.MESSAGE, + number=3, + message=table.TableSchema, + ) + row_errors: MutableSequence['RowError'] = proto.RepeatedField( + proto.MESSAGE, + number=4, + message='RowError', + ) + write_stream: str = proto.Field( + proto.STRING, + number=5, + ) + + +class GetWriteStreamRequest(proto.Message): + r"""Request message for ``GetWriteStreamRequest``. + + Attributes: + name (str): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + view (google.cloud.bigquery_storage_v1.types.WriteStreamView): + Indicates whether to get full or partial view + of the WriteStream. If not set, view returned + will be basic. + """ + + name: str = proto.Field( + proto.STRING, + number=1, + ) + view: stream.WriteStreamView = proto.Field( + proto.ENUM, + number=3, + enum=stream.WriteStreamView, + ) + + +class BatchCommitWriteStreamsRequest(proto.Message): + r"""Request message for ``BatchCommitWriteStreams``. + + Attributes: + parent (str): + Required. Parent table that all the streams should belong + to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + write_streams (MutableSequence[str]): + Required. The group of streams that will be + committed atomically. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + write_streams: MutableSequence[str] = proto.RepeatedField( + proto.STRING, + number=2, + ) + + +class BatchCommitWriteStreamsResponse(proto.Message): + r"""Response message for ``BatchCommitWriteStreams``. + + Attributes: + commit_time (google.protobuf.timestamp_pb2.Timestamp): + The time at which streams were committed in microseconds + granularity. This field will only exist when there are no + stream errors. **Note** if this field is not set, it means + the commit was not successful. + stream_errors (MutableSequence[google.cloud.bigquery_storage_v1.types.StorageError]): + Stream level error if commit failed. Only + streams with error will be in the list. + If empty, there is no error and all streams are + committed successfully. If non empty, certain + streams have errors and ZERO stream is committed + due to atomicity guarantee. + """ + + commit_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=1, + message=timestamp_pb2.Timestamp, + ) + stream_errors: MutableSequence['StorageError'] = proto.RepeatedField( + proto.MESSAGE, + number=2, + message='StorageError', + ) + + +class FinalizeWriteStreamRequest(proto.Message): + r"""Request message for invoking ``FinalizeWriteStream``. + + Attributes: + name (str): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + """ + + name: str = proto.Field( + proto.STRING, + number=1, + ) + + +class FinalizeWriteStreamResponse(proto.Message): + r"""Response message for ``FinalizeWriteStream``. + + Attributes: + row_count (int): + Number of rows in the finalized stream. + """ + + row_count: int = proto.Field( + proto.INT64, + number=1, + ) + + +class FlushRowsRequest(proto.Message): + r"""Request message for ``FlushRows``. + + Attributes: + write_stream (str): + Required. The stream that is the target of + the flush operation. + offset (google.protobuf.wrappers_pb2.Int64Value): + Ending offset of the flush operation. Rows + before this offset(including this offset) will + be flushed. + """ + + write_stream: str = proto.Field( + proto.STRING, + number=1, + ) + offset: wrappers_pb2.Int64Value = proto.Field( + proto.MESSAGE, + number=2, + message=wrappers_pb2.Int64Value, + ) + + +class FlushRowsResponse(proto.Message): + r"""Respond message for ``FlushRows``. + + Attributes: + offset (int): + The rows before this offset (including this + offset) are flushed. + """ + + offset: int = proto.Field( + proto.INT64, + number=1, + ) + + +class StorageError(proto.Message): + r"""Structured custom BigQuery Storage error message. The error + can be attached as error details in the returned rpc Status. In + particular, the use of error codes allows more structured error + handling, and reduces the need to evaluate unstructured error + text strings. + + Attributes: + code (google.cloud.bigquery_storage_v1.types.StorageError.StorageErrorCode): + BigQuery Storage specific error code. + entity (str): + Name of the failed entity. + error_message (str): + Message that describes the error. + """ + class StorageErrorCode(proto.Enum): + r"""Error code for ``StorageError``. + + Values: + STORAGE_ERROR_CODE_UNSPECIFIED (0): + Default error. + TABLE_NOT_FOUND (1): + Table is not found in the system. + STREAM_ALREADY_COMMITTED (2): + Stream is already committed. + STREAM_NOT_FOUND (3): + Stream is not found. + INVALID_STREAM_TYPE (4): + Invalid Stream type. + For example, you try to commit a stream that is + not pending. + INVALID_STREAM_STATE (5): + Invalid Stream state. + For example, you try to commit a stream that is + not finalized or is garbaged. + STREAM_FINALIZED (6): + Stream is finalized. + SCHEMA_MISMATCH_EXTRA_FIELDS (7): + There is a schema mismatch and it is caused + by user schema has extra field than bigquery + schema. + OFFSET_ALREADY_EXISTS (8): + Offset already exists. + OFFSET_OUT_OF_RANGE (9): + Offset out of range. + CMEK_NOT_PROVIDED (10): + Customer-managed encryption key (CMEK) not + provided for CMEK-enabled data. + INVALID_CMEK_PROVIDED (11): + Customer-managed encryption key (CMEK) was + incorrectly provided. + CMEK_ENCRYPTION_ERROR (12): + There is an encryption error while using + customer-managed encryption key. + KMS_SERVICE_ERROR (13): + Key Management Service (KMS) service returned + an error, which can be retried. + KMS_PERMISSION_DENIED (14): + Permission denied while using + customer-managed encryption key. + """ + STORAGE_ERROR_CODE_UNSPECIFIED = 0 + TABLE_NOT_FOUND = 1 + STREAM_ALREADY_COMMITTED = 2 + STREAM_NOT_FOUND = 3 + INVALID_STREAM_TYPE = 4 + INVALID_STREAM_STATE = 5 + STREAM_FINALIZED = 6 + SCHEMA_MISMATCH_EXTRA_FIELDS = 7 + OFFSET_ALREADY_EXISTS = 8 + OFFSET_OUT_OF_RANGE = 9 + CMEK_NOT_PROVIDED = 10 + INVALID_CMEK_PROVIDED = 11 + CMEK_ENCRYPTION_ERROR = 12 + KMS_SERVICE_ERROR = 13 + KMS_PERMISSION_DENIED = 14 + + code: StorageErrorCode = proto.Field( + proto.ENUM, + number=1, + enum=StorageErrorCode, + ) + entity: str = proto.Field( + proto.STRING, + number=2, + ) + error_message: str = proto.Field( + proto.STRING, + number=3, + ) + + +class RowError(proto.Message): + r"""The message that presents row level error info in a request. + + Attributes: + index (int): + Index of the malformed row in the request. + code (google.cloud.bigquery_storage_v1.types.RowError.RowErrorCode): + Structured error reason for a row error. + message (str): + Description of the issue encountered when + processing the row. + """ + class RowErrorCode(proto.Enum): + r"""Error code for ``RowError``. + + Values: + ROW_ERROR_CODE_UNSPECIFIED (0): + Default error. + FIELDS_ERROR (1): + One or more fields in the row has errors. + """ + ROW_ERROR_CODE_UNSPECIFIED = 0 + FIELDS_ERROR = 1 + + index: int = proto.Field( + proto.INT64, + number=1, + ) + code: RowErrorCode = proto.Field( + proto.ENUM, + number=2, + enum=RowErrorCode, + ) + message: str = proto.Field( + proto.STRING, + number=3, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/stream.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/stream.py new file mode 100644 index 00000000..e629e327 --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/stream.py @@ -0,0 +1,492 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + +from google.cloud.bigquery_storage_v1.types import arrow +from google.cloud.bigquery_storage_v1.types import avro +from google.cloud.bigquery_storage_v1.types import table as gcbs_table +from google.protobuf import timestamp_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1', + manifest={ + 'DataFormat', + 'WriteStreamView', + 'ReadSession', + 'ReadStream', + 'WriteStream', + }, +) + + +class DataFormat(proto.Enum): + r"""Data format for input or output data. + + Values: + DATA_FORMAT_UNSPECIFIED (0): + Data format is unspecified. + AVRO (1): + Avro is a standard open source row based file + format. See https://avro.apache.org/ for more + details. + ARROW (2): + Arrow is a standard open source column-based + message format. See https://arrow.apache.org/ + for more details. + """ + DATA_FORMAT_UNSPECIFIED = 0 + AVRO = 1 + ARROW = 2 + + +class WriteStreamView(proto.Enum): + r"""WriteStreamView is a view enum that controls what details + about a write stream should be returned. + + Values: + WRITE_STREAM_VIEW_UNSPECIFIED (0): + The default / unset value. + BASIC (1): + The BASIC projection returns basic metadata + about a write stream. The basic view does not + include schema information. This is the default + view returned by GetWriteStream. + FULL (2): + The FULL projection returns all available + write stream metadata, including the schema. + CreateWriteStream returns the full projection of + write stream metadata. + """ + WRITE_STREAM_VIEW_UNSPECIFIED = 0 + BASIC = 1 + FULL = 2 + + +class ReadSession(proto.Message): + r"""Information about the ReadSession. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + name (str): + Output only. Unique identifier for the session, in the form + ``projects/{project_id}/locations/{location}/sessions/{session_id}``. + expire_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Time at which the session becomes invalid. + After this time, subsequent requests to read this Session + will return errors. The expire_time is automatically + assigned and currently cannot be specified or updated. + data_format (google.cloud.bigquery_storage_v1.types.DataFormat): + Immutable. Data format of the output data. + DATA_FORMAT_UNSPECIFIED not supported. + avro_schema (google.cloud.bigquery_storage_v1.types.AvroSchema): + Output only. Avro schema. + + This field is a member of `oneof`_ ``schema``. + arrow_schema (google.cloud.bigquery_storage_v1.types.ArrowSchema): + Output only. Arrow schema. + + This field is a member of `oneof`_ ``schema``. + table (str): + Immutable. Table that this ReadSession is reading from, in + the form + ``projects/{project_id}/datasets/{dataset_id}/tables/{table_id}`` + table_modifiers (google.cloud.bigquery_storage_v1.types.ReadSession.TableModifiers): + Optional. Any modifiers which are applied + when reading from the specified table. + read_options (google.cloud.bigquery_storage_v1.types.ReadSession.TableReadOptions): + Optional. Read options for this session (e.g. + column selection, filters). + streams (MutableSequence[google.cloud.bigquery_storage_v1.types.ReadStream]): + Output only. A list of streams created with the session. + + At least one stream is created with the session. In the + future, larger request_stream_count values *may* result in + this list being unpopulated, in that case, the user will + need to use a List method to get the streams instead, which + is not yet available. + estimated_total_bytes_scanned (int): + Output only. An estimate on the number of + bytes this session will scan when all streams + are completely consumed. This estimate is based + on metadata from the table which might be + incomplete or stale. + estimated_total_physical_file_size (int): + Output only. A pre-projected estimate of the + total physical size of files (in bytes) that + this session will scan when all streams are + consumed. This estimate is independent of the + selected columns and can be based on incomplete + or stale metadata from the table. This field is + only set for BigLake tables. + estimated_row_count (int): + Output only. An estimate on the number of + rows present in this session's streams. This + estimate is based on metadata from the table + which might be incomplete or stale. + trace_id (str): + Optional. ID set by client to annotate a + session identity. This does not need to be + strictly unique, but instead the same ID should + be used to group logically connected sessions + (e.g. All using the same ID for all sessions + needed to complete a Spark SQL query is + reasonable). + + Maximum length is 256 bytes. + """ + + class TableModifiers(proto.Message): + r"""Additional attributes when reading a table. + + Attributes: + snapshot_time (google.protobuf.timestamp_pb2.Timestamp): + The snapshot time of the table. If not set, + interpreted as now. + """ + + snapshot_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=1, + message=timestamp_pb2.Timestamp, + ) + + class TableReadOptions(proto.Message): + r"""Options dictating how we read a table. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + selected_fields (MutableSequence[str]): + Optional. The names of the fields in the table to be + returned. If no field names are specified, then all fields + in the table are returned. + + Nested fields -- the child elements of a STRUCT field -- can + be selected individually using their fully-qualified names, + and will be returned as record fields containing only the + selected nested fields. If a STRUCT field is specified in + the selected fields list, all of the child elements will be + returned. + + As an example, consider a table with the following schema: + + { "name": "struct_field", "type": "RECORD", "mode": + "NULLABLE", "fields": [ { "name": "string_field1", "type": + "STRING", . "mode": "NULLABLE" }, { "name": "string_field2", + "type": "STRING", "mode": "NULLABLE" } ] } + + Specifying "struct_field" in the selected fields list will + result in a read session schema with the following logical + structure: + + struct_field { string_field1 string_field2 } + + Specifying "struct_field.string_field1" in the selected + fields list will result in a read session schema with the + following logical structure: + + struct_field { string_field1 } + + The order of the fields in the read session schema is + derived from the table schema and does not correspond to the + order in which the fields are specified in this list. + row_restriction (str): + SQL text filtering statement, similar to a WHERE clause in a + query. Aggregates are not supported. + + Examples: "int_field > 5" "date_field = CAST('2014-9-27' as + DATE)" "nullable_field is not NULL" "st_equals(geo_field, + st_geofromtext("POINT(2, 2)"))" "numeric_field BETWEEN 1.0 + AND 5.0" + + Restricted to a maximum length for 1 MB. + arrow_serialization_options (google.cloud.bigquery_storage_v1.types.ArrowSerializationOptions): + Optional. Options specific to the Apache + Arrow output format. + + This field is a member of `oneof`_ ``output_format_serialization_options``. + avro_serialization_options (google.cloud.bigquery_storage_v1.types.AvroSerializationOptions): + Optional. Options specific to the Apache Avro + output format + + This field is a member of `oneof`_ ``output_format_serialization_options``. + sample_percentage (float): + Optional. Specifies a table sampling percentage. + Specifically, the query planner will use TABLESAMPLE SYSTEM + (sample_percentage PERCENT). The sampling percentage is + applied at the data block granularity. It will randomly + choose for each data block whether to read the rows in that + data block. For more details, see + https://cloud.google.com/bigquery/docs/table-sampling) + + This field is a member of `oneof`_ ``_sample_percentage``. + response_compression_codec (google.cloud.bigquery_storage_v1.types.ReadSession.TableReadOptions.ResponseCompressionCodec): + Optional. Set response_compression_codec when creating a + read session to enable application-level compression of + ReadRows responses. + + This field is a member of `oneof`_ ``_response_compression_codec``. + """ + class ResponseCompressionCodec(proto.Enum): + r"""Specifies which compression codec to attempt on the entire + serialized response payload (either Arrow record batch or Avro + rows). This is not to be confused with the Apache Arrow native + compression codecs specified in ArrowSerializationOptions. For + performance reasons, when creating a read session requesting + Arrow responses, setting both native Arrow compression and + application-level response compression will not be allowed - + choose, at most, one kind of compression. + + Values: + RESPONSE_COMPRESSION_CODEC_UNSPECIFIED (0): + Default is no compression. + RESPONSE_COMPRESSION_CODEC_LZ4 (2): + Use raw LZ4 compression. + """ + RESPONSE_COMPRESSION_CODEC_UNSPECIFIED = 0 + RESPONSE_COMPRESSION_CODEC_LZ4 = 2 + + selected_fields: MutableSequence[str] = proto.RepeatedField( + proto.STRING, + number=1, + ) + row_restriction: str = proto.Field( + proto.STRING, + number=2, + ) + arrow_serialization_options: arrow.ArrowSerializationOptions = proto.Field( + proto.MESSAGE, + number=3, + oneof='output_format_serialization_options', + message=arrow.ArrowSerializationOptions, + ) + avro_serialization_options: avro.AvroSerializationOptions = proto.Field( + proto.MESSAGE, + number=4, + oneof='output_format_serialization_options', + message=avro.AvroSerializationOptions, + ) + sample_percentage: float = proto.Field( + proto.DOUBLE, + number=5, + optional=True, + ) + response_compression_codec: 'ReadSession.TableReadOptions.ResponseCompressionCodec' = proto.Field( + proto.ENUM, + number=6, + optional=True, + enum='ReadSession.TableReadOptions.ResponseCompressionCodec', + ) + + name: str = proto.Field( + proto.STRING, + number=1, + ) + expire_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=2, + message=timestamp_pb2.Timestamp, + ) + data_format: 'DataFormat' = proto.Field( + proto.ENUM, + number=3, + enum='DataFormat', + ) + avro_schema: avro.AvroSchema = proto.Field( + proto.MESSAGE, + number=4, + oneof='schema', + message=avro.AvroSchema, + ) + arrow_schema: arrow.ArrowSchema = proto.Field( + proto.MESSAGE, + number=5, + oneof='schema', + message=arrow.ArrowSchema, + ) + table: str = proto.Field( + proto.STRING, + number=6, + ) + table_modifiers: TableModifiers = proto.Field( + proto.MESSAGE, + number=7, + message=TableModifiers, + ) + read_options: TableReadOptions = proto.Field( + proto.MESSAGE, + number=8, + message=TableReadOptions, + ) + streams: MutableSequence['ReadStream'] = proto.RepeatedField( + proto.MESSAGE, + number=10, + message='ReadStream', + ) + estimated_total_bytes_scanned: int = proto.Field( + proto.INT64, + number=12, + ) + estimated_total_physical_file_size: int = proto.Field( + proto.INT64, + number=15, + ) + estimated_row_count: int = proto.Field( + proto.INT64, + number=14, + ) + trace_id: str = proto.Field( + proto.STRING, + number=13, + ) + + +class ReadStream(proto.Message): + r"""Information about a single stream that gets data out of the storage + system. Most of the information about ``ReadStream`` instances is + aggregated, making ``ReadStream`` lightweight. + + Attributes: + name (str): + Output only. Name of the stream, in the form + ``projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}``. + """ + + name: str = proto.Field( + proto.STRING, + number=1, + ) + + +class WriteStream(proto.Message): + r"""Information about a single stream that gets data inside the + storage system. + + Attributes: + name (str): + Output only. Name of the stream, in the form + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + type_ (google.cloud.bigquery_storage_v1.types.WriteStream.Type): + Immutable. Type of the stream. + create_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Create time of the stream. For the \_default + stream, this is the creation_time of the table. + commit_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Commit time of the stream. If a stream is of + ``COMMITTED`` type, then it will have a commit_time same as + ``create_time``. If the stream is of ``PENDING`` type, empty + commit_time means it is not committed. + table_schema (google.cloud.bigquery_storage_v1.types.TableSchema): + Output only. The schema of the destination table. It is only + returned in ``CreateWriteStream`` response. Caller should + generate data that's compatible with this schema to send in + initial ``AppendRowsRequest``. The table schema could go out + of date during the life time of the stream. + write_mode (google.cloud.bigquery_storage_v1.types.WriteStream.WriteMode): + Immutable. Mode of the stream. + location (str): + Immutable. The geographic location where the + stream's dataset resides. See + https://cloud.google.com/bigquery/docs/locations + for supported locations. + """ + class Type(proto.Enum): + r"""Type enum of the stream. + + Values: + TYPE_UNSPECIFIED (0): + Unknown type. + COMMITTED (1): + Data will commit automatically and appear as + soon as the write is acknowledged. + PENDING (2): + Data is invisible until the stream is + committed. + BUFFERED (3): + Data is only visible up to the offset to + which it was flushed. + """ + TYPE_UNSPECIFIED = 0 + COMMITTED = 1 + PENDING = 2 + BUFFERED = 3 + + class WriteMode(proto.Enum): + r"""Mode enum of the stream. + + Values: + WRITE_MODE_UNSPECIFIED (0): + Unknown type. + INSERT (1): + Insert new records into the table. + It is the default value if customers do not + specify it. + """ + WRITE_MODE_UNSPECIFIED = 0 + INSERT = 1 + + name: str = proto.Field( + proto.STRING, + number=1, + ) + type_: Type = proto.Field( + proto.ENUM, + number=2, + enum=Type, + ) + create_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=3, + message=timestamp_pb2.Timestamp, + ) + commit_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=4, + message=timestamp_pb2.Timestamp, + ) + table_schema: gcbs_table.TableSchema = proto.Field( + proto.MESSAGE, + number=5, + message=gcbs_table.TableSchema, + ) + write_mode: WriteMode = proto.Field( + proto.ENUM, + number=7, + enum=WriteMode, + ) + location: str = proto.Field( + proto.STRING, + number=8, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/table.py b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/table.py new file mode 100644 index 00000000..7eb58c4f --- /dev/null +++ b/owl-bot-staging/v1/google/cloud/bigquery_storage_v1/types/table.py @@ -0,0 +1,275 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1', + manifest={ + 'TableSchema', + 'TableFieldSchema', + }, +) + + +class TableSchema(proto.Message): + r"""Schema of a table. This schema is a subset of + google.cloud.bigquery.v2.TableSchema containing information + necessary to generate valid message to write to BigQuery. + + Attributes: + fields (MutableSequence[google.cloud.bigquery_storage_v1.types.TableFieldSchema]): + Describes the fields in a table. + """ + + fields: MutableSequence['TableFieldSchema'] = proto.RepeatedField( + proto.MESSAGE, + number=1, + message='TableFieldSchema', + ) + + +class TableFieldSchema(proto.Message): + r"""TableFieldSchema defines a single field/column within a table + schema. + + Attributes: + name (str): + Required. The field name. The name must contain only letters + (a-z, A-Z), numbers (0-9), or underscores (_), and must + start with a letter or underscore. The maximum length is 128 + characters. + type_ (google.cloud.bigquery_storage_v1.types.TableFieldSchema.Type): + Required. The field data type. + mode (google.cloud.bigquery_storage_v1.types.TableFieldSchema.Mode): + Optional. The field mode. The default value + is NULLABLE. + fields (MutableSequence[google.cloud.bigquery_storage_v1.types.TableFieldSchema]): + Optional. Describes the nested schema fields + if the type property is set to STRUCT. + description (str): + Optional. The field description. The maximum + length is 1,024 characters. + max_length (int): + Optional. Maximum length of values of this field for STRINGS + or BYTES. + + If max_length is not specified, no maximum length constraint + is imposed on this field. + + If type = "STRING", then max_length represents the maximum + UTF-8 length of strings in this field. + + If type = "BYTES", then max_length represents the maximum + number of bytes in this field. + + It is invalid to set this field if type is not "STRING" or + "BYTES". + precision (int): + Optional. Precision (maximum number of total digits in base + 10) and scale (maximum number of digits in the fractional + part in base 10) constraints for values of this field for + NUMERIC or BIGNUMERIC. + + It is invalid to set precision or scale if type is not + "NUMERIC" or "BIGNUMERIC". + + If precision and scale are not specified, no value range + constraint is imposed on this field insofar as values are + permitted by the type. + + Values of this NUMERIC or BIGNUMERIC field must be in this + range when: + + - Precision (P) and scale (S) are specified: [-10^(P-S) + + 10^(-S), 10^(P-S) - 10^(-S)] + - Precision (P) is specified but not scale (and thus scale + is interpreted to be equal to zero): [-10^P + 1, 10^P - + 1]. + + Acceptable values for precision and scale if both are + specified: + + - If type = "NUMERIC": 1 <= precision - scale <= 29 and 0 + <= scale <= 9. + - If type = "BIGNUMERIC": 1 <= precision - scale <= 38 and + 0 <= scale <= 38. + + Acceptable values for precision if only precision is + specified but not scale (and thus scale is interpreted to be + equal to zero): + + - If type = "NUMERIC": 1 <= precision <= 29. + - If type = "BIGNUMERIC": 1 <= precision <= 38. + + If scale is specified but not precision, then it is invalid. + scale (int): + Optional. See documentation for precision. + default_value_expression (str): + Optional. A SQL expression to specify the [default value] + (https://cloud.google.com/bigquery/docs/default-values) for + this field. + range_element_type (google.cloud.bigquery_storage_v1.types.TableFieldSchema.FieldElementType): + Optional. The subtype of the RANGE, if the type of this + field is RANGE. If the type is RANGE, this field is + required. Possible values for the field element type of a + RANGE include: + + - DATE + - DATETIME + - TIMESTAMP + """ + class Type(proto.Enum): + r""" + + Values: + TYPE_UNSPECIFIED (0): + Illegal value + STRING (1): + 64K, UTF8 + INT64 (2): + 64-bit signed + DOUBLE (3): + 64-bit IEEE floating point + STRUCT (4): + Aggregate type + BYTES (5): + 64K, Binary + BOOL (6): + 2-valued + TIMESTAMP (7): + 64-bit signed usec since UTC epoch + DATE (8): + Civil date - Year, Month, Day + TIME (9): + Civil time - Hour, Minute, Second, + Microseconds + DATETIME (10): + Combination of civil date and civil time + GEOGRAPHY (11): + Geography object + NUMERIC (12): + Numeric value + BIGNUMERIC (13): + BigNumeric value + INTERVAL (14): + Interval + JSON (15): + JSON, String + RANGE (16): + RANGE + """ + TYPE_UNSPECIFIED = 0 + STRING = 1 + INT64 = 2 + DOUBLE = 3 + STRUCT = 4 + BYTES = 5 + BOOL = 6 + TIMESTAMP = 7 + DATE = 8 + TIME = 9 + DATETIME = 10 + GEOGRAPHY = 11 + NUMERIC = 12 + BIGNUMERIC = 13 + INTERVAL = 14 + JSON = 15 + RANGE = 16 + + class Mode(proto.Enum): + r""" + + Values: + MODE_UNSPECIFIED (0): + Illegal value + NULLABLE (1): + No description available. + REQUIRED (2): + No description available. + REPEATED (3): + No description available. + """ + MODE_UNSPECIFIED = 0 + NULLABLE = 1 + REQUIRED = 2 + REPEATED = 3 + + class FieldElementType(proto.Message): + r"""Represents the type of a field element. + + Attributes: + type_ (google.cloud.bigquery_storage_v1.types.TableFieldSchema.Type): + Required. The type of a field element. + """ + + type_: 'TableFieldSchema.Type' = proto.Field( + proto.ENUM, + number=1, + enum='TableFieldSchema.Type', + ) + + name: str = proto.Field( + proto.STRING, + number=1, + ) + type_: Type = proto.Field( + proto.ENUM, + number=2, + enum=Type, + ) + mode: Mode = proto.Field( + proto.ENUM, + number=3, + enum=Mode, + ) + fields: MutableSequence['TableFieldSchema'] = proto.RepeatedField( + proto.MESSAGE, + number=4, + message='TableFieldSchema', + ) + description: str = proto.Field( + proto.STRING, + number=6, + ) + max_length: int = proto.Field( + proto.INT64, + number=7, + ) + precision: int = proto.Field( + proto.INT64, + number=8, + ) + scale: int = proto.Field( + proto.INT64, + number=9, + ) + default_value_expression: str = proto.Field( + proto.STRING, + number=10, + ) + range_element_type: FieldElementType = proto.Field( + proto.MESSAGE, + number=11, + message=FieldElementType, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1/mypy.ini b/owl-bot-staging/v1/mypy.ini new file mode 100644 index 00000000..574c5aed --- /dev/null +++ b/owl-bot-staging/v1/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +python_version = 3.7 +namespace_packages = True diff --git a/owl-bot-staging/v1/noxfile.py b/owl-bot-staging/v1/noxfile.py new file mode 100644 index 00000000..ae78e7f6 --- /dev/null +++ b/owl-bot-staging/v1/noxfile.py @@ -0,0 +1,278 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import pathlib +import re +import shutil +import subprocess +import sys + + +import nox # type: ignore + +ALL_PYTHON = [ + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", + "3.12" +] + +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt" +PACKAGE_NAME = 'google-cloud-bigquery-storage' + +BLACK_VERSION = "black==22.3.0" +BLACK_PATHS = ["docs", "google", "tests", "samples", "noxfile.py", "setup.py"] +DEFAULT_PYTHON_VERSION = "3.12" + +nox.sessions = [ + "unit", + "cover", + "mypy", + "check_lower_bounds" + # exclude update_lower_bounds from default + "docs", + "blacken", + "lint", + "prerelease_deps", +] + +@nox.session(python=ALL_PYTHON) +@nox.parametrize( + "protobuf_implementation", + [ "python", "upb", "cpp" ], +) +def unit(session, protobuf_implementation): + """Run the unit test suite.""" + + if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"): + session.skip("cpp implementation is not supported in python 3.11+") + + session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"') + session.install('-e', '.', "-c", f"testing/constraints-{session.python}.txt") + + # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. + # The 'cpp' implementation requires Protobuf<4. + if protobuf_implementation == "cpp": + session.install("protobuf<4") + + session.run( + 'py.test', + '--quiet', + '--cov=google/cloud/bigquery_storage_v1/', + '--cov=tests/', + '--cov-config=.coveragerc', + '--cov-report=term', + '--cov-report=html', + os.path.join('tests', 'unit', ''.join(session.posargs)), + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + +@nox.session(python=ALL_PYTHON[-1]) +@nox.parametrize( + "protobuf_implementation", + [ "python", "upb", "cpp" ], +) +def prerelease_deps(session, protobuf_implementation): + """Run the unit test suite against pre-release versions of dependencies.""" + + if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"): + session.skip("cpp implementation is not supported in python 3.11+") + + # Install test environment dependencies + session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"') + + # Install the package without dependencies + session.install('-e', '.', '--no-deps') + + # We test the minimum dependency versions using the minimum Python + # version so the lowest python runtime that we test has a corresponding constraints + # file, located at `testing/constraints--.txt`, which contains all of the + # dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{ALL_PYTHON[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + constraints_deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + session.install(*constraints_deps) + + prerel_deps = [ + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpcio", + "grpcio-status", + "protobuf", + "proto-plus", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = [ + "requests", + ] + session.install(*other_deps) + + # Print out prerelease package versions + + session.run("python", "-c", "import google.api_core; print(google.api_core.__version__)") + session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + session.run("python", "-c", "import grpc; print(grpc.__version__)") + session.run( + "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" + ) + session.run( + "python", "-c", "import proto; print(proto.__version__)" + ) + + session.run( + 'py.test', + '--quiet', + '--cov=google/cloud/bigquery_storage_v1/', + '--cov=tests/', + '--cov-config=.coveragerc', + '--cov-report=term', + '--cov-report=html', + os.path.join('tests', 'unit', ''.join(session.posargs)), + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def cover(session): + """Run the final coverage report. + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=100") + + session.run("coverage", "erase") + + +@nox.session(python=ALL_PYTHON) +def mypy(session): + """Run the type checker.""" + session.install( + 'mypy', + 'types-requests', + 'types-protobuf' + ) + session.install('.') + session.run( + 'mypy', + '-p', + 'google', + ) + + +@nox.session +def update_lower_bounds(session): + """Update lower bounds in constraints.txt to match setup.py""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'update', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + + +@nox.session +def check_lower_bounds(session): + """Check lower bounds in setup.py are reflected in constraints file""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'check', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docs(session): + """Build the docs for this library.""" + + session.install("-e", ".") + session.install("sphinx==7.0.1", "alabaster", "recommonmark") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint(session): + """Run linters. + + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + session.install("flake8", BLACK_VERSION) + session.run( + "black", + "--check", + *BLACK_PATHS, + ) + session.run("flake8", "google", "tests", "samples") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def blacken(session): + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + session.run( + "black", + *BLACK_PATHS, + ) diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_create_read_session_async.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_create_read_session_async.py new file mode 100644 index 00000000..a1ac8598 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_create_read_session_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for CreateReadSession +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryRead_CreateReadSession_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +async def sample_create_read_session(): + # Create a client + client = bigquery_storage_v1.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.CreateReadSessionRequest( + parent="parent_value", + ) + + # Make the request + response = await client.create_read_session(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryRead_CreateReadSession_async] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_create_read_session_sync.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_create_read_session_sync.py new file mode 100644 index 00000000..c93df92d --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_create_read_session_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for CreateReadSession +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryRead_CreateReadSession_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +def sample_create_read_session(): + # Create a client + client = bigquery_storage_v1.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.CreateReadSessionRequest( + parent="parent_value", + ) + + # Make the request + response = client.create_read_session(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryRead_CreateReadSession_sync] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_read_rows_async.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_read_rows_async.py new file mode 100644 index 00000000..ead55f79 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_read_rows_async.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for ReadRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryRead_ReadRows_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +async def sample_read_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.ReadRowsRequest( + read_stream="read_stream_value", + ) + + # Make the request + stream = await client.read_rows(request=request) + + # Handle the response + async for response in stream: + print(response) + +# [END bigquerystorage_v1_generated_BigQueryRead_ReadRows_async] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_read_rows_sync.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_read_rows_sync.py new file mode 100644 index 00000000..7de5afa0 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_read_rows_sync.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for ReadRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryRead_ReadRows_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +def sample_read_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.ReadRowsRequest( + read_stream="read_stream_value", + ) + + # Make the request + stream = client.read_rows(request=request) + + # Handle the response + for response in stream: + print(response) + +# [END bigquerystorage_v1_generated_BigQueryRead_ReadRows_sync] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_split_read_stream_async.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_split_read_stream_async.py new file mode 100644 index 00000000..e84d54de --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_split_read_stream_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for SplitReadStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryRead_SplitReadStream_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +async def sample_split_read_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.SplitReadStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.split_read_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryRead_SplitReadStream_async] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_split_read_stream_sync.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_split_read_stream_sync.py new file mode 100644 index 00000000..c48ae91a --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_read_split_read_stream_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for SplitReadStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryRead_SplitReadStream_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +def sample_split_read_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.SplitReadStreamRequest( + name="name_value", + ) + + # Make the request + response = client.split_read_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryRead_SplitReadStream_sync] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_append_rows_async.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_append_rows_async.py new file mode 100644 index 00000000..ef6f8ba8 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_append_rows_async.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for AppendRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_AppendRows_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +async def sample_append_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.AppendRowsRequest( + write_stream="write_stream_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1.AppendRowsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = await client.append_rows(requests=request_generator()) + + # Handle the response + async for response in stream: + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_AppendRows_async] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_append_rows_sync.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_append_rows_sync.py new file mode 100644 index 00000000..a28b8ef9 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_append_rows_sync.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for AppendRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_AppendRows_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +def sample_append_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.AppendRowsRequest( + write_stream="write_stream_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1.AppendRowsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = client.append_rows(requests=request_generator()) + + # Handle the response + for response in stream: + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_AppendRows_sync] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_batch_commit_write_streams_async.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_batch_commit_write_streams_async.py new file mode 100644 index 00000000..0163ab17 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_batch_commit_write_streams_async.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for BatchCommitWriteStreams +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_BatchCommitWriteStreams_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +async def sample_batch_commit_write_streams(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.BatchCommitWriteStreamsRequest( + parent="parent_value", + write_streams=['write_streams_value1', 'write_streams_value2'], + ) + + # Make the request + response = await client.batch_commit_write_streams(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_BatchCommitWriteStreams_async] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_batch_commit_write_streams_sync.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_batch_commit_write_streams_sync.py new file mode 100644 index 00000000..71704d2c --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_batch_commit_write_streams_sync.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for BatchCommitWriteStreams +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_BatchCommitWriteStreams_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +def sample_batch_commit_write_streams(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.BatchCommitWriteStreamsRequest( + parent="parent_value", + write_streams=['write_streams_value1', 'write_streams_value2'], + ) + + # Make the request + response = client.batch_commit_write_streams(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_BatchCommitWriteStreams_sync] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_create_write_stream_async.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_create_write_stream_async.py new file mode 100644 index 00000000..1c8b6c4d --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_create_write_stream_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for CreateWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_CreateWriteStream_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +async def sample_create_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.CreateWriteStreamRequest( + parent="parent_value", + ) + + # Make the request + response = await client.create_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_CreateWriteStream_async] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_create_write_stream_sync.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_create_write_stream_sync.py new file mode 100644 index 00000000..27ea7cec --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_create_write_stream_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for CreateWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_CreateWriteStream_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +def sample_create_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.CreateWriteStreamRequest( + parent="parent_value", + ) + + # Make the request + response = client.create_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_CreateWriteStream_sync] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_finalize_write_stream_async.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_finalize_write_stream_async.py new file mode 100644 index 00000000..0fcd2a7f --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_finalize_write_stream_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for FinalizeWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_FinalizeWriteStream_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +async def sample_finalize_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.FinalizeWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.finalize_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_FinalizeWriteStream_async] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_finalize_write_stream_sync.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_finalize_write_stream_sync.py new file mode 100644 index 00000000..b5b89fb2 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_finalize_write_stream_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for FinalizeWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_FinalizeWriteStream_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +def sample_finalize_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.FinalizeWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = client.finalize_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_FinalizeWriteStream_sync] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_flush_rows_async.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_flush_rows_async.py new file mode 100644 index 00000000..10ec9829 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_flush_rows_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for FlushRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_FlushRows_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +async def sample_flush_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.FlushRowsRequest( + write_stream="write_stream_value", + ) + + # Make the request + response = await client.flush_rows(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_FlushRows_async] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_flush_rows_sync.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_flush_rows_sync.py new file mode 100644 index 00000000..41ab6473 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_flush_rows_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for FlushRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_FlushRows_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +def sample_flush_rows(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.FlushRowsRequest( + write_stream="write_stream_value", + ) + + # Make the request + response = client.flush_rows(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_FlushRows_sync] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_get_write_stream_async.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_get_write_stream_async.py new file mode 100644 index 00000000..a560cb47 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_get_write_stream_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for GetWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_GetWriteStream_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +async def sample_get_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.GetWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.get_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_GetWriteStream_async] diff --git a/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_get_write_stream_sync.py b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_get_write_stream_sync.py new file mode 100644 index 00000000..20c12674 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/bigquerystorage_v1_generated_big_query_write_get_write_stream_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for GetWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1_generated_BigQueryWrite_GetWriteStream_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1 + + +def sample_get_write_stream(): + # Create a client + client = bigquery_storage_v1.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1.GetWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = client.get_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1_generated_BigQueryWrite_GetWriteStream_sync] diff --git a/owl-bot-staging/v1/samples/generated_samples/snippet_metadata_google.cloud.bigquery.storage.v1.json b/owl-bot-staging/v1/samples/generated_samples/snippet_metadata_google.cloud.bigquery.storage.v1.json new file mode 100644 index 00000000..c1d15715 --- /dev/null +++ b/owl-bot-staging/v1/samples/generated_samples/snippet_metadata_google.cloud.bigquery.storage.v1.json @@ -0,0 +1,1480 @@ +{ + "clientLibrary": { + "apis": [ + { + "id": "google.cloud.bigquery.storage.v1", + "version": "v1" + } + ], + "language": "PYTHON", + "name": "google-cloud-bigquery-storage", + "version": "0.1.0" + }, + "snippets": [ + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadAsyncClient", + "shortName": "BigQueryReadAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadAsyncClient.create_read_session", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead.CreateReadSession", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "CreateReadSession" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.CreateReadSessionRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "read_session", + "type": "google.cloud.bigquery_storage_v1.types.ReadSession" + }, + { + "name": "max_stream_count", + "type": "int" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.ReadSession", + "shortName": "create_read_session" + }, + "description": "Sample for CreateReadSession", + "file": "bigquerystorage_v1_generated_big_query_read_create_read_session_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryRead_CreateReadSession_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_read_create_read_session_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadClient", + "shortName": "BigQueryReadClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadClient.create_read_session", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead.CreateReadSession", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "CreateReadSession" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.CreateReadSessionRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "read_session", + "type": "google.cloud.bigquery_storage_v1.types.ReadSession" + }, + { + "name": "max_stream_count", + "type": "int" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.ReadSession", + "shortName": "create_read_session" + }, + "description": "Sample for CreateReadSession", + "file": "bigquerystorage_v1_generated_big_query_read_create_read_session_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryRead_CreateReadSession_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_read_create_read_session_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadAsyncClient", + "shortName": "BigQueryReadAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadAsyncClient.read_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead.ReadRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "ReadRows" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.ReadRowsRequest" + }, + { + "name": "read_stream", + "type": "str" + }, + { + "name": "offset", + "type": "int" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "Iterable[google.cloud.bigquery_storage_v1.types.ReadRowsResponse]", + "shortName": "read_rows" + }, + "description": "Sample for ReadRows", + "file": "bigquerystorage_v1_generated_big_query_read_read_rows_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryRead_ReadRows_async", + "segments": [ + { + "end": 52, + "start": 27, + "type": "FULL" + }, + { + "end": 52, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 53, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_read_read_rows_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadClient", + "shortName": "BigQueryReadClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadClient.read_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead.ReadRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "ReadRows" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.ReadRowsRequest" + }, + { + "name": "read_stream", + "type": "str" + }, + { + "name": "offset", + "type": "int" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "Iterable[google.cloud.bigquery_storage_v1.types.ReadRowsResponse]", + "shortName": "read_rows" + }, + "description": "Sample for ReadRows", + "file": "bigquerystorage_v1_generated_big_query_read_read_rows_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryRead_ReadRows_sync", + "segments": [ + { + "end": 52, + "start": 27, + "type": "FULL" + }, + { + "end": 52, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 53, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_read_read_rows_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadAsyncClient", + "shortName": "BigQueryReadAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadAsyncClient.split_read_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead.SplitReadStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "SplitReadStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.SplitReadStreamRequest" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.SplitReadStreamResponse", + "shortName": "split_read_stream" + }, + "description": "Sample for SplitReadStream", + "file": "bigquerystorage_v1_generated_big_query_read_split_read_stream_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryRead_SplitReadStream_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_read_split_read_stream_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadClient", + "shortName": "BigQueryReadClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryReadClient.split_read_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead.SplitReadStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "SplitReadStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.SplitReadStreamRequest" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.SplitReadStreamResponse", + "shortName": "split_read_stream" + }, + "description": "Sample for SplitReadStream", + "file": "bigquerystorage_v1_generated_big_query_read_split_read_stream_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryRead_SplitReadStream_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_read_split_read_stream_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient.append_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.AppendRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "AppendRows" + }, + "parameters": [ + { + "name": "requests", + "type": "Iterator[google.cloud.bigquery_storage_v1.types.AppendRowsRequest]" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "Iterable[google.cloud.bigquery_storage_v1.types.AppendRowsResponse]", + "shortName": "append_rows" + }, + "description": "Sample for AppendRows", + "file": "bigquerystorage_v1_generated_big_query_write_append_rows_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_AppendRows_async", + "segments": [ + { + "end": 62, + "start": 27, + "type": "FULL" + }, + { + "end": 62, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 55, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 58, + "start": 56, + "type": "REQUEST_EXECUTION" + }, + { + "end": 63, + "start": 59, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_append_rows_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient.append_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.AppendRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "AppendRows" + }, + "parameters": [ + { + "name": "requests", + "type": "Iterator[google.cloud.bigquery_storage_v1.types.AppendRowsRequest]" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "Iterable[google.cloud.bigquery_storage_v1.types.AppendRowsResponse]", + "shortName": "append_rows" + }, + "description": "Sample for AppendRows", + "file": "bigquerystorage_v1_generated_big_query_write_append_rows_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_AppendRows_sync", + "segments": [ + { + "end": 62, + "start": 27, + "type": "FULL" + }, + { + "end": 62, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 55, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 58, + "start": 56, + "type": "REQUEST_EXECUTION" + }, + { + "end": 63, + "start": 59, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_append_rows_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient.batch_commit_write_streams", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.BatchCommitWriteStreams", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "BatchCommitWriteStreams" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsResponse", + "shortName": "batch_commit_write_streams" + }, + "description": "Sample for BatchCommitWriteStreams", + "file": "bigquerystorage_v1_generated_big_query_write_batch_commit_write_streams_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_BatchCommitWriteStreams_async", + "segments": [ + { + "end": 52, + "start": 27, + "type": "FULL" + }, + { + "end": 52, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 46, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 49, + "start": 47, + "type": "REQUEST_EXECUTION" + }, + { + "end": 53, + "start": 50, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_batch_commit_write_streams_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient.batch_commit_write_streams", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.BatchCommitWriteStreams", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "BatchCommitWriteStreams" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.BatchCommitWriteStreamsResponse", + "shortName": "batch_commit_write_streams" + }, + "description": "Sample for BatchCommitWriteStreams", + "file": "bigquerystorage_v1_generated_big_query_write_batch_commit_write_streams_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_BatchCommitWriteStreams_sync", + "segments": [ + { + "end": 52, + "start": 27, + "type": "FULL" + }, + { + "end": 52, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 46, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 49, + "start": 47, + "type": "REQUEST_EXECUTION" + }, + { + "end": 53, + "start": 50, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_batch_commit_write_streams_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient.create_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.CreateWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "CreateWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.CreateWriteStreamRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "write_stream", + "type": "google.cloud.bigquery_storage_v1.types.WriteStream" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.WriteStream", + "shortName": "create_write_stream" + }, + "description": "Sample for CreateWriteStream", + "file": "bigquerystorage_v1_generated_big_query_write_create_write_stream_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_CreateWriteStream_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_create_write_stream_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient.create_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.CreateWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "CreateWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.CreateWriteStreamRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "write_stream", + "type": "google.cloud.bigquery_storage_v1.types.WriteStream" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.WriteStream", + "shortName": "create_write_stream" + }, + "description": "Sample for CreateWriteStream", + "file": "bigquerystorage_v1_generated_big_query_write_create_write_stream_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_CreateWriteStream_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_create_write_stream_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient.finalize_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.FinalizeWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "FinalizeWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamRequest" + }, + { + "name": "name", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamResponse", + "shortName": "finalize_write_stream" + }, + "description": "Sample for FinalizeWriteStream", + "file": "bigquerystorage_v1_generated_big_query_write_finalize_write_stream_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_FinalizeWriteStream_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_finalize_write_stream_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient.finalize_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.FinalizeWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "FinalizeWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamRequest" + }, + { + "name": "name", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.FinalizeWriteStreamResponse", + "shortName": "finalize_write_stream" + }, + "description": "Sample for FinalizeWriteStream", + "file": "bigquerystorage_v1_generated_big_query_write_finalize_write_stream_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_FinalizeWriteStream_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_finalize_write_stream_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient.flush_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.FlushRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "FlushRows" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.FlushRowsRequest" + }, + { + "name": "write_stream", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.FlushRowsResponse", + "shortName": "flush_rows" + }, + "description": "Sample for FlushRows", + "file": "bigquerystorage_v1_generated_big_query_write_flush_rows_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_FlushRows_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_flush_rows_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient.flush_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.FlushRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "FlushRows" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.FlushRowsRequest" + }, + { + "name": "write_stream", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.FlushRowsResponse", + "shortName": "flush_rows" + }, + "description": "Sample for FlushRows", + "file": "bigquerystorage_v1_generated_big_query_write_flush_rows_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_FlushRows_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_flush_rows_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteAsyncClient.get_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.GetWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "GetWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.GetWriteStreamRequest" + }, + { + "name": "name", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.WriteStream", + "shortName": "get_write_stream" + }, + "description": "Sample for GetWriteStream", + "file": "bigquerystorage_v1_generated_big_query_write_get_write_stream_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_GetWriteStream_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_get_write_stream_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1.BigQueryWriteClient.get_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite.GetWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "GetWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1.types.GetWriteStreamRequest" + }, + { + "name": "name", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1.types.WriteStream", + "shortName": "get_write_stream" + }, + "description": "Sample for GetWriteStream", + "file": "bigquerystorage_v1_generated_big_query_write_get_write_stream_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1_generated_BigQueryWrite_GetWriteStream_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1_generated_big_query_write_get_write_stream_sync.py" + } + ] +} diff --git a/owl-bot-staging/v1/scripts/fixup_bigquery_storage_v1_keywords.py b/owl-bot-staging/v1/scripts/fixup_bigquery_storage_v1_keywords.py new file mode 100644 index 00000000..e6c2750a --- /dev/null +++ b/owl-bot-staging/v1/scripts/fixup_bigquery_storage_v1_keywords.py @@ -0,0 +1,184 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import os +import libcst as cst +import pathlib +import sys +from typing import (Any, Callable, Dict, List, Sequence, Tuple) + + +def partition( + predicate: Callable[[Any], bool], + iterator: Sequence[Any] +) -> Tuple[List[Any], List[Any]]: + """A stable, out-of-place partition.""" + results = ([], []) + + for i in iterator: + results[int(predicate(i))].append(i) + + # Returns trueList, falseList + return results[1], results[0] + + +class bigquery_storageCallTransformer(cst.CSTTransformer): + CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') + METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { + 'append_rows': ('write_stream', 'offset', 'proto_rows', 'arrow_rows', 'trace_id', 'missing_value_interpretations', 'default_missing_value_interpretation', ), + 'batch_commit_write_streams': ('parent', 'write_streams', ), + 'create_read_session': ('parent', 'read_session', 'max_stream_count', 'preferred_min_stream_count', ), + 'create_write_stream': ('parent', 'write_stream', ), + 'finalize_write_stream': ('name', ), + 'flush_rows': ('write_stream', 'offset', ), + 'get_write_stream': ('name', 'view', ), + 'read_rows': ('read_stream', 'offset', ), + 'split_read_stream': ('name', 'fraction', ), + } + + def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: + try: + key = original.func.attr.value + kword_params = self.METHOD_TO_PARAMS[key] + except (AttributeError, KeyError): + # Either not a method from the API or too convoluted to be sure. + return updated + + # If the existing code is valid, keyword args come after positional args. + # Therefore, all positional args must map to the first parameters. + args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) + if any(k.keyword.value == "request" for k in kwargs): + # We've already fixed this file, don't fix it again. + return updated + + kwargs, ctrl_kwargs = partition( + lambda a: a.keyword.value not in self.CTRL_PARAMS, + kwargs + ) + + args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] + ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) + for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) + + request_arg = cst.Arg( + value=cst.Dict([ + cst.DictElement( + cst.SimpleString("'{}'".format(name)), +cst.Element(value=arg.value) + ) + # Note: the args + kwargs looks silly, but keep in mind that + # the control parameters had to be stripped out, and that + # those could have been passed positionally or by keyword. + for name, arg in zip(kword_params, args + kwargs)]), + keyword=cst.Name("request") + ) + + return updated.with_changes( + args=[request_arg] + ctrl_kwargs + ) + + +def fix_files( + in_dir: pathlib.Path, + out_dir: pathlib.Path, + *, + transformer=bigquery_storageCallTransformer(), +): + """Duplicate the input dir to the output dir, fixing file method calls. + + Preconditions: + * in_dir is a real directory + * out_dir is a real, empty directory + """ + pyfile_gen = ( + pathlib.Path(os.path.join(root, f)) + for root, _, files in os.walk(in_dir) + for f in files if os.path.splitext(f)[1] == ".py" + ) + + for fpath in pyfile_gen: + with open(fpath, 'r') as f: + src = f.read() + + # Parse the code and insert method call fixes. + tree = cst.parse_module(src) + updated = tree.visit(transformer) + + # Create the path and directory structure for the new file. + updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) + updated_path.parent.mkdir(parents=True, exist_ok=True) + + # Generate the updated source file at the corresponding path. + with open(updated_path, 'w') as f: + f.write(updated.code) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="""Fix up source that uses the bigquery_storage client library. + +The existing sources are NOT overwritten but are copied to output_dir with changes made. + +Note: This tool operates at a best-effort level at converting positional + parameters in client method calls to keyword based parameters. + Cases where it WILL FAIL include + A) * or ** expansion in a method call. + B) Calls via function or method alias (includes free function calls) + C) Indirect or dispatched calls (e.g. the method is looked up dynamically) + + These all constitute false negatives. The tool will also detect false + positives when an API method shares a name with another method. +""") + parser.add_argument( + '-d', + '--input-directory', + required=True, + dest='input_dir', + help='the input directory to walk for python files to fix up', + ) + parser.add_argument( + '-o', + '--output-directory', + required=True, + dest='output_dir', + help='the directory to output files fixed via un-flattening', + ) + args = parser.parse_args() + input_dir = pathlib.Path(args.input_dir) + output_dir = pathlib.Path(args.output_dir) + if not input_dir.is_dir(): + print( + f"input directory '{input_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if not output_dir.is_dir(): + print( + f"output directory '{output_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if os.listdir(output_dir): + print( + f"output directory '{output_dir}' is not empty", + file=sys.stderr, + ) + sys.exit(-1) + + fix_files(input_dir, output_dir) diff --git a/owl-bot-staging/v1/setup.py b/owl-bot-staging/v1/setup.py new file mode 100644 index 00000000..b688ea84 --- /dev/null +++ b/owl-bot-staging/v1/setup.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import io +import os +import re + +import setuptools # type: ignore + +package_root = os.path.abspath(os.path.dirname(__file__)) + +name = 'google-cloud-bigquery-storage' + + +description = "Google Cloud Bigquery Storage API client library" + +version = None + +with open(os.path.join(package_root, 'google/cloud/bigquery_storage/gapic_version.py')) as fp: + version_candidates = re.findall(r"(?<=\")\d+.\d+.\d+(?=\")", fp.read()) + assert (len(version_candidates) == 1) + version = version_candidates[0] + +if version[0] == "0": + release_status = "Development Status :: 4 - Beta" +else: + release_status = "Development Status :: 5 - Production/Stable" + +dependencies = [ + "google-api-core[grpc] >= 1.34.1, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", + # Exclude incompatible versions of `google-auth` + # See https://github.com/googleapis/google-cloud-python/issues/12364 + "google-auth >= 2.14.1, <3.0.0dev,!=2.24.0,!=2.25.0", + "proto-plus >= 1.22.3, <2.0.0dev", + "protobuf>=3.20.2,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", +] +extras = { +} +url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-cloud-bigquery-storage" + +package_root = os.path.abspath(os.path.dirname(__file__)) + +readme_filename = os.path.join(package_root, "README.rst") +with io.open(readme_filename, encoding="utf-8") as readme_file: + readme = readme_file.read() + +packages = [ + package + for package in setuptools.find_namespace_packages() + if package.startswith("google") +] + +setuptools.setup( + name=name, + version=version, + description=description, + long_description=readme, + author="Google LLC", + author_email="googleapis-packages@google.com", + license="Apache 2.0", + url=url, + classifiers=[ + release_status, + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Topic :: Internet", + ], + platforms="Posix; MacOS X; Windows", + packages=packages, + python_requires=">=3.7", + install_requires=dependencies, + extras_require=extras, + include_package_data=True, + zip_safe=False, +) diff --git a/owl-bot-staging/v1/testing/constraints-3.10.txt b/owl-bot-staging/v1/testing/constraints-3.10.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1/testing/constraints-3.10.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1/testing/constraints-3.11.txt b/owl-bot-staging/v1/testing/constraints-3.11.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1/testing/constraints-3.11.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1/testing/constraints-3.12.txt b/owl-bot-staging/v1/testing/constraints-3.12.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1/testing/constraints-3.12.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1/testing/constraints-3.7.txt b/owl-bot-staging/v1/testing/constraints-3.7.txt new file mode 100644 index 00000000..fc812592 --- /dev/null +++ b/owl-bot-staging/v1/testing/constraints-3.7.txt @@ -0,0 +1,10 @@ +# This constraints file is used to check that lower bounds +# are correct in setup.py +# List all library dependencies and extras in this file. +# Pin the version to the lower bound. +# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0dev", +# Then this file should have google-cloud-foo==1.14.0 +google-api-core==1.34.1 +google-auth==2.14.1 +proto-plus==1.22.3 +protobuf==3.20.2 diff --git a/owl-bot-staging/v1/testing/constraints-3.8.txt b/owl-bot-staging/v1/testing/constraints-3.8.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1/testing/constraints-3.8.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1/testing/constraints-3.9.txt b/owl-bot-staging/v1/testing/constraints-3.9.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1/testing/constraints-3.9.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1/tests/__init__.py b/owl-bot-staging/v1/tests/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1/tests/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1/tests/unit/__init__.py b/owl-bot-staging/v1/tests/unit/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1/tests/unit/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1/tests/unit/gapic/__init__.py b/owl-bot-staging/v1/tests/unit/gapic/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1/tests/unit/gapic/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/__init__.py b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py new file mode 100644 index 00000000..fdf971e9 --- /dev/null +++ b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_read.py @@ -0,0 +1,2514 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +# try/except added for compatibility with python < 3.8 +try: + from unittest import mock + from unittest.mock import AsyncMock # pragma: NO COVER +except ImportError: # pragma: NO COVER + import mock + +import grpc +from grpc.experimental import aio +import math +import pytest +from google.api_core import api_core_version +from proto.marshal.rules.dates import DurationRule, TimestampRule +from proto.marshal.rules import wrappers + +try: + from google.auth.aio import credentials as ga_credentials_async + HAS_GOOGLE_AUTH_AIO = True +except ImportError: # pragma: NO COVER + HAS_GOOGLE_AUTH_AIO = False + +from google.api_core import client_options +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers +from google.api_core import grpc_helpers_async +from google.api_core import path_template +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials +from google.auth.exceptions import MutualTLSChannelError +from google.cloud.bigquery_storage_v1.services.big_query_read import BigQueryReadAsyncClient +from google.cloud.bigquery_storage_v1.services.big_query_read import BigQueryReadClient +from google.cloud.bigquery_storage_v1.services.big_query_read import transports +from google.cloud.bigquery_storage_v1.types import arrow +from google.cloud.bigquery_storage_v1.types import avro +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.oauth2 import service_account +from google.protobuf import timestamp_pb2 # type: ignore +import google.auth + + +async def mock_async_gen(data, chunk_size=1): + for i in range(0, len(data)): # pragma: NO COVER + chunk = data[i : i + chunk_size] + yield chunk.encode("utf-8") + +def client_cert_source_callback(): + return b"cert bytes", b"key bytes" + +# TODO: use async auth anon credentials by default once the minimum version of google-auth is upgraded. +# See related issue: https://github.com/googleapis/gapic-generator-python/issues/2107. +def async_anonymous_credentials(): + if HAS_GOOGLE_AUTH_AIO: + return ga_credentials_async.AnonymousCredentials() + return ga_credentials.AnonymousCredentials() + +# If default endpoint is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint(client): + return "foo.googleapis.com" if ("localhost" in client.DEFAULT_ENDPOINT) else client.DEFAULT_ENDPOINT + +# If default endpoint template is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint template so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint_template(client): + return "test.{UNIVERSE_DOMAIN}" if ("localhost" in client._DEFAULT_ENDPOINT_TEMPLATE) else client._DEFAULT_ENDPOINT_TEMPLATE + + +def test__get_default_mtls_endpoint(): + api_endpoint = "example.googleapis.com" + api_mtls_endpoint = "example.mtls.googleapis.com" + sandbox_endpoint = "example.sandbox.googleapis.com" + sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" + non_googleapi = "api.example.com" + + assert BigQueryReadClient._get_default_mtls_endpoint(None) is None + assert BigQueryReadClient._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi + +def test__read_environment_variables(): + assert BigQueryReadClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert BigQueryReadClient._read_environment_variables() == (True, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + assert BigQueryReadClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + BigQueryReadClient._read_environment_variables() + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + assert BigQueryReadClient._read_environment_variables() == (False, "never", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + assert BigQueryReadClient._read_environment_variables() == (False, "always", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}): + assert BigQueryReadClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + BigQueryReadClient._read_environment_variables() + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + with mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}): + assert BigQueryReadClient._read_environment_variables() == (False, "auto", "foo.com") + +def test__get_client_cert_source(): + mock_provided_cert_source = mock.Mock() + mock_default_cert_source = mock.Mock() + + assert BigQueryReadClient._get_client_cert_source(None, False) is None + assert BigQueryReadClient._get_client_cert_source(mock_provided_cert_source, False) is None + assert BigQueryReadClient._get_client_cert_source(mock_provided_cert_source, True) == mock_provided_cert_source + + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=mock_default_cert_source): + assert BigQueryReadClient._get_client_cert_source(None, True) is mock_default_cert_source + assert BigQueryReadClient._get_client_cert_source(mock_provided_cert_source, "true") is mock_provided_cert_source + +@mock.patch.object(BigQueryReadClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadAsyncClient)) +def test__get_api_endpoint(): + api_override = "foo.com" + mock_client_cert_source = mock.Mock() + default_universe = BigQueryReadClient._DEFAULT_UNIVERSE + default_endpoint = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=default_universe) + mock_universe = "bar.com" + mock_endpoint = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=mock_universe) + + assert BigQueryReadClient._get_api_endpoint(api_override, mock_client_cert_source, default_universe, "always") == api_override + assert BigQueryReadClient._get_api_endpoint(None, mock_client_cert_source, default_universe, "auto") == BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryReadClient._get_api_endpoint(None, None, default_universe, "auto") == default_endpoint + assert BigQueryReadClient._get_api_endpoint(None, None, default_universe, "always") == BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryReadClient._get_api_endpoint(None, mock_client_cert_source, default_universe, "always") == BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryReadClient._get_api_endpoint(None, None, mock_universe, "never") == mock_endpoint + assert BigQueryReadClient._get_api_endpoint(None, None, default_universe, "never") == default_endpoint + + with pytest.raises(MutualTLSChannelError) as excinfo: + BigQueryReadClient._get_api_endpoint(None, mock_client_cert_source, mock_universe, "auto") + assert str(excinfo.value) == "mTLS is not supported in any universe other than googleapis.com." + + +def test__get_universe_domain(): + client_universe_domain = "foo.com" + universe_domain_env = "bar.com" + + assert BigQueryReadClient._get_universe_domain(client_universe_domain, universe_domain_env) == client_universe_domain + assert BigQueryReadClient._get_universe_domain(None, universe_domain_env) == universe_domain_env + assert BigQueryReadClient._get_universe_domain(None, None) == BigQueryReadClient._DEFAULT_UNIVERSE + + with pytest.raises(ValueError) as excinfo: + BigQueryReadClient._get_universe_domain("", None) + assert str(excinfo.value) == "Universe Domain cannot be an empty string." + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), +]) +def test__validate_universe_domain(client_class, transport_class, transport_name): + client = client_class( + transport=transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + ) + assert client._validate_universe_domain() == True + + # Test the case when universe is already validated. + assert client._validate_universe_domain() == True + + if transport_name == "grpc": + # Test the case where credentials are provided by the + # `local_channel_credentials`. The default universes in both match. + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + client = client_class(transport=transport_class(channel=channel)) + assert client._validate_universe_domain() == True + + # Test the case where credentials do not exist: e.g. a transport is provided + # with no credentials. Validation should still succeed because there is no + # mismatch with non-existent credentials. + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + transport=transport_class(channel=channel) + transport._credentials = None + client = client_class(transport=transport) + assert client._validate_universe_domain() == True + + # TODO: This is needed to cater for older versions of google-auth + # Make this test unconditional once the minimum supported version of + # google-auth becomes 2.23.0 or higher. + google_auth_major, google_auth_minor = [int(part) for part in google.auth.__version__.split(".")[0:2]] + if google_auth_major > 2 or (google_auth_major == 2 and google_auth_minor >= 23): + credentials = ga_credentials.AnonymousCredentials() + credentials._universe_domain = "foo.com" + # Test the case when there is a universe mismatch from the credentials. + client = client_class( + transport=transport_class(credentials=credentials) + ) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert str(excinfo.value) == "The configured universe domain (googleapis.com) does not match the universe domain found in the credentials (foo.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + + # Test the case when there is a universe mismatch from the client. + # + # TODO: Make this test unconditional once the minimum supported version of + # google-api-core becomes 2.15.0 or higher. + api_core_major, api_core_minor = [int(part) for part in api_core_version.__version__.split(".")[0:2]] + if api_core_major > 2 or (api_core_major == 2 and api_core_minor >= 15): + client = client_class(client_options={"universe_domain": "bar.com"}, transport=transport_class(credentials=ga_credentials.AnonymousCredentials(),)) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert str(excinfo.value) == "The configured universe domain (bar.com) does not match the universe domain found in the credentials (googleapis.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + + # Test that ValueError is raised if universe_domain is provided via client options and credentials is None + with pytest.raises(ValueError): + client._compare_universes("foo.bar", None) + + +@pytest.mark.parametrize("client_class,transport_name", [ + (BigQueryReadClient, "grpc"), + (BigQueryReadAsyncClient, "grpc_asyncio"), +]) +def test_big_query_read_client_from_service_account_info(client_class, transport_name): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_info') as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info, transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + + +@pytest.mark.parametrize("transport_class,transport_name", [ + (transports.BigQueryReadGrpcTransport, "grpc"), + (transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_read_client_service_account_always_use_jwt(transport_class, transport_name): + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize("client_class,transport_name", [ + (BigQueryReadClient, "grpc"), + (BigQueryReadAsyncClient, "grpc_asyncio"), +]) +def test_big_query_read_client_from_service_account_file(client_class, transport_name): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_file') as factory: + factory.return_value = creds + client = client_class.from_service_account_file("dummy/file/path.json", transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + client = client_class.from_service_account_json("dummy/file/path.json", transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + + +def test_big_query_read_client_get_transport_class(): + transport = BigQueryReadClient.get_transport_class() + available_transports = [ + transports.BigQueryReadGrpcTransport, + ] + assert transport in available_transports + + transport = BigQueryReadClient.get_transport_class("grpc") + assert transport == transports.BigQueryReadGrpcTransport + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +@mock.patch.object(BigQueryReadClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadAsyncClient)) +def test_big_query_read_client_client_options(client_class, transport_class, transport_name): + # Check that if channel is provided we won't create a new one. + with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + client = client_class(transport=transport) + gtc.assert_not_called() + + # Check that if channel is provided via str we will create a new one. + with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: + client = client_class(transport=transport_name) + gtc.assert_called() + + # Check the case api_endpoint is provided. + options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name, client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_MTLS_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client = client_class(transport=transport_name) + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + client = client_class(transport=transport_name) + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + # Check the case api_endpoint is provided + options = client_options.ClientOptions(api_audience="https://language.googleapis.com") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience="https://language.googleapis.com" + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,use_client_cert_env", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "true"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "true"), + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "false"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "false"), +]) +@mock.patch.object(BigQueryReadClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadAsyncClient)) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_big_query_read_client_mtls_env_auto(client_class, transport_class, transport_name, use_client_cert_env): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + options = client_options.ClientOptions(client_cert_source=client_cert_source_callback) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE) + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT + + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=client_cert_source_callback): + if use_client_cert_env == "false": + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE) + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback + + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=False): + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + +@pytest.mark.parametrize("client_class", [ + BigQueryReadClient, BigQueryReadAsyncClient +]) +@mock.patch.object(BigQueryReadClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadAsyncClient)) +def test_big_query_read_client_get_mtls_endpoint_and_cert_source(client_class): + mock_client_cert_source = mock.Mock() + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "true". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + mock_api_endpoint = "foo" + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source == mock_client_cert_source + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "false". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert doesn't exist. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=False): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert exists. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=mock_client_cert_source): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source == mock_client_cert_source + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + +@pytest.mark.parametrize("client_class", [ + BigQueryReadClient, BigQueryReadAsyncClient +]) +@mock.patch.object(BigQueryReadClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadAsyncClient)) +def test_big_query_read_client_client_api_endpoint(client_class): + mock_client_cert_source = client_cert_source_callback + api_override = "foo.com" + default_universe = BigQueryReadClient._DEFAULT_UNIVERSE + default_endpoint = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=default_universe) + mock_universe = "bar.com" + mock_endpoint = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=mock_universe) + + # If ClientOptions.api_endpoint is set and GOOGLE_API_USE_CLIENT_CERTIFICATE="true", + # use ClientOptions.api_endpoint as the api endpoint regardless. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch("google.auth.transport.requests.AuthorizedSession.configure_mtls_channel"): + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=api_override) + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == api_override + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="always", + # use the DEFAULT_MTLS_ENDPOINT as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + + # If ClientOptions.api_endpoint is not set, GOOGLE_API_USE_MTLS_ENDPOINT="auto" (default), + # GOOGLE_API_USE_CLIENT_CERTIFICATE="false" (default), default cert source doesn't exist, + # and ClientOptions.universe_domain="bar.com", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with universe domain as the api endpoint. + options = client_options.ClientOptions() + universe_exists = hasattr(options, "universe_domain") + if universe_exists: + options = client_options.ClientOptions(universe_domain=mock_universe) + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + else: + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == (mock_endpoint if universe_exists else default_endpoint) + assert client.universe_domain == (mock_universe if universe_exists else default_universe) + + # If ClientOptions does not have a universe domain attribute and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + options = client_options.ClientOptions() + if hasattr(options, "universe_domain"): + delattr(options, "universe_domain") + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_read_client_client_options_scopes(client_class, transport_class, transport_name): + # Check the case scopes are provided. + options = client_options.ClientOptions( + scopes=["1", "2"], + ) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=["1", "2"], + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,grpc_helpers", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", grpc_helpers), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", grpc_helpers_async), +]) +def test_big_query_read_client_client_options_credentials_file(client_class, transport_class, transport_name, grpc_helpers): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + +def test_big_query_read_client_client_options_from_dict(): + with mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadGrpcTransport.__init__') as grpc_transport: + grpc_transport.return_value = None + client = BigQueryReadClient( + client_options={'api_endpoint': 'squid.clam.whelk'} + ) + grpc_transport.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + +@pytest.mark.parametrize("client_class,transport_class,transport_name,grpc_helpers", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", grpc_helpers), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", grpc_helpers_async), +]) +def test_big_query_read_client_create_channel_credentials_file(client_class, transport_class, transport_name, grpc_helpers): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=None, + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("request_type", [ + storage.CreateReadSessionRequest, + dict, +]) +def test_create_read_session(request_type, transport: str = 'grpc'): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession( + name='name_value', + data_format=stream.DataFormat.AVRO, + table='table_value', + estimated_total_bytes_scanned=3076, + estimated_total_physical_file_size=3608, + estimated_row_count=2047, + trace_id='trace_id_value', + ) + response = client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.CreateReadSessionRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.ReadSession) + assert response.name == 'name_value' + assert response.data_format == stream.DataFormat.AVRO + assert response.table == 'table_value' + assert response.estimated_total_bytes_scanned == 3076 + assert response.estimated_total_physical_file_size == 3608 + assert response.estimated_row_count == 2047 + assert response.trace_id == 'trace_id_value' + + +def test_create_read_session_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.CreateReadSessionRequest( + parent='parent_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.create_read_session(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateReadSessionRequest( + parent='parent_value', + ) + +def test_create_read_session_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.create_read_session in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.create_read_session] = mock_rpc + request = {} + client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.create_read_session(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_create_read_session_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.create_read_session in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.create_read_session] = mock_rpc + + request = {} + await client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.create_read_session(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_create_read_session_async(transport: str = 'grpc_asyncio', request_type=storage.CreateReadSessionRequest): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession( + name='name_value', + data_format=stream.DataFormat.AVRO, + table='table_value', + estimated_total_bytes_scanned=3076, + estimated_total_physical_file_size=3608, + estimated_row_count=2047, + trace_id='trace_id_value', + )) + response = await client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.CreateReadSessionRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.ReadSession) + assert response.name == 'name_value' + assert response.data_format == stream.DataFormat.AVRO + assert response.table == 'table_value' + assert response.estimated_total_bytes_scanned == 3076 + assert response.estimated_total_physical_file_size == 3608 + assert response.estimated_row_count == 2047 + assert response.trace_id == 'trace_id_value' + + +@pytest.mark.asyncio +async def test_create_read_session_async_from_dict(): + await test_create_read_session_async(request_type=dict) + +def test_create_read_session_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateReadSessionRequest() + + request.read_session.table = 'table_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value = stream.ReadSession() + client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_session.table=table_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_create_read_session_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateReadSessionRequest() + + request.read_session.table = 'table_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) + await client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_session.table=table_value', + ) in kw['metadata'] + + +def test_create_read_session_flattened(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.create_read_session( + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + arg = args[0].read_session + mock_val = stream.ReadSession(name='name_value') + assert arg == mock_val + arg = args[0].max_stream_count + mock_val = 1721 + assert arg == mock_val + + +def test_create_read_session_flattened_error(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.create_read_session( + storage.CreateReadSessionRequest(), + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + +@pytest.mark.asyncio +async def test_create_read_session_flattened_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.create_read_session( + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + arg = args[0].read_session + mock_val = stream.ReadSession(name='name_value') + assert arg == mock_val + arg = args[0].max_stream_count + mock_val = 1721 + assert arg == mock_val + +@pytest.mark.asyncio +async def test_create_read_session_flattened_error_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.create_read_session( + storage.CreateReadSessionRequest(), + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + +@pytest.mark.parametrize("request_type", [ + storage.ReadRowsRequest, + dict, +]) +def test_read_rows(request_type, transport: str = 'grpc'): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + response = client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.ReadRowsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + for message in response: + assert isinstance(message, storage.ReadRowsResponse) + + +def test_read_rows_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.ReadRowsRequest( + read_stream='read_stream_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.read_rows(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.ReadRowsRequest( + read_stream='read_stream_value', + ) + +def test_read_rows_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.read_rows in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.read_rows] = mock_rpc + request = {} + client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.read_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_read_rows_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.read_rows in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.read_rows] = mock_rpc + + request = {} + await client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.read_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_read_rows_async(transport: str = 'grpc_asyncio', request_type=storage.ReadRowsRequest): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) + response = await client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.ReadRowsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + message = await response.read() + assert isinstance(message, storage.ReadRowsResponse) + + +@pytest.mark.asyncio +async def test_read_rows_async_from_dict(): + await test_read_rows_async(request_type=dict) + +def test_read_rows_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.ReadRowsRequest() + + request.read_stream = 'read_stream_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value = iter([storage.ReadRowsResponse()]) + client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_stream=read_stream_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_read_rows_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.ReadRowsRequest() + + request.read_stream = 'read_stream_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) + await client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_stream=read_stream_value', + ) in kw['metadata'] + + +def test_read_rows_flattened(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.read_rows( + read_stream='read_stream_value', + offset=647, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].read_stream + mock_val = 'read_stream_value' + assert arg == mock_val + arg = args[0].offset + mock_val = 647 + assert arg == mock_val + + +def test_read_rows_flattened_error(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.read_rows( + storage.ReadRowsRequest(), + read_stream='read_stream_value', + offset=647, + ) + +@pytest.mark.asyncio +async def test_read_rows_flattened_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.read_rows( + read_stream='read_stream_value', + offset=647, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].read_stream + mock_val = 'read_stream_value' + assert arg == mock_val + arg = args[0].offset + mock_val = 647 + assert arg == mock_val + +@pytest.mark.asyncio +async def test_read_rows_flattened_error_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.read_rows( + storage.ReadRowsRequest(), + read_stream='read_stream_value', + offset=647, + ) + + +@pytest.mark.parametrize("request_type", [ + storage.SplitReadStreamRequest, + dict, +]) +def test_split_read_stream(request_type, transport: str = 'grpc'): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.SplitReadStreamResponse( + ) + response = client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.SplitReadStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.SplitReadStreamResponse) + + +def test_split_read_stream_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.SplitReadStreamRequest( + name='name_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.split_read_stream(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.SplitReadStreamRequest( + name='name_value', + ) + +def test_split_read_stream_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.split_read_stream in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.split_read_stream] = mock_rpc + request = {} + client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.split_read_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_split_read_stream_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.split_read_stream in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.split_read_stream] = mock_rpc + + request = {} + await client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.split_read_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_split_read_stream_async(transport: str = 'grpc_asyncio', request_type=storage.SplitReadStreamRequest): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse( + )) + response = await client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.SplitReadStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.SplitReadStreamResponse) + + +@pytest.mark.asyncio +async def test_split_read_stream_async_from_dict(): + await test_split_read_stream_async(request_type=dict) + +def test_split_read_stream_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.SplitReadStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value = storage.SplitReadStreamResponse() + client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_split_read_stream_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.SplitReadStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse()) + await client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide an api_key and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options=options, + transport=transport, + ) + + # It is an error to provide an api_key and a credential. + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options=options, + credentials=ga_credentials.AnonymousCredentials() + ) + + # It is an error to provide scopes and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options={"scopes": ["1", "2"]}, + transport=transport, + ) + + +def test_transport_instance(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + client = BigQueryReadClient(transport=transport) + assert client.transport is transport + +def test_transport_get_channel(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + + transport = transports.BigQueryReadGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + +@pytest.mark.parametrize("transport_class", [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, +]) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + +def test_transport_kind_grpc(): + transport = BigQueryReadClient.get_transport_class("grpc")( + credentials=ga_credentials.AnonymousCredentials() + ) + assert transport.kind == "grpc" + + +def test_initialize_client_w_grpc(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc" + ) + assert client is not None + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_create_read_session_empty_call_grpc(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value = stream.ReadSession() + client.create_read_session(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.CreateReadSessionRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_read_rows_empty_call_grpc(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value = iter([storage.ReadRowsResponse()]) + client.read_rows(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.ReadRowsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_split_read_stream_empty_call_grpc(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value = storage.SplitReadStreamResponse() + client.split_read_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.SplitReadStreamRequest() + + assert args[0] == request_msg + + +def test_transport_kind_grpc_asyncio(): + transport = BigQueryReadAsyncClient.get_transport_class("grpc_asyncio")( + credentials=async_anonymous_credentials() + ) + assert transport.kind == "grpc_asyncio" + + +def test_initialize_client_w_grpc_asyncio(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio" + ) + assert client is not None + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_create_read_session_empty_call_grpc_asyncio(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession( + name='name_value', + data_format=stream.DataFormat.AVRO, + table='table_value', + estimated_total_bytes_scanned=3076, + estimated_total_physical_file_size=3608, + estimated_row_count=2047, + trace_id='trace_id_value', + )) + await client.create_read_session(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.CreateReadSessionRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_read_rows_empty_call_grpc_asyncio(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) + await client.read_rows(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.ReadRowsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_split_read_stream_empty_call_grpc_asyncio(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse( + )) + await client.split_read_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.SplitReadStreamRequest() + + assert args[0] == request_msg + + +def test_transport_grpc_default(): + # A client should use the gRPC transport by default. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert isinstance( + client.transport, + transports.BigQueryReadGrpcTransport, + ) + +def test_big_query_read_base_transport_error(): + # Passing both a credentials object and credentials_file should raise an error + with pytest.raises(core_exceptions.DuplicateCredentialArgs): + transport = transports.BigQueryReadTransport( + credentials=ga_credentials.AnonymousCredentials(), + credentials_file="credentials.json" + ) + + +def test_big_query_read_base_transport(): + # Instantiate the base transport. + with mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadTransport.__init__') as Transport: + Transport.return_value = None + transport = transports.BigQueryReadTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Every method on the transport should just blindly + # raise NotImplementedError. + methods = ( + 'create_read_session', + 'read_rows', + 'split_read_stream', + ) + for method in methods: + with pytest.raises(NotImplementedError): + getattr(transport, method)(request=object()) + + with pytest.raises(NotImplementedError): + transport.close() + + # Catch all for all remaining methods and properties + remainder = [ + 'kind', + ] + for r in remainder: + with pytest.raises(NotImplementedError): + getattr(transport, r)() + + +def test_big_query_read_base_transport_with_credentials_file(): + # Instantiate the base transport with a credentials file + with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryReadTransport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id="octopus", + ) + + +def test_big_query_read_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(google.auth, 'default', autospec=True) as adc, mock.patch('google.cloud.bigquery_storage_v1.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryReadTransport() + adc.assert_called_once() + + +def test_big_query_read_auth_adc(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryReadClient() + adc.assert_called_once_with( + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id=None, + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, + ], +) +def test_big_query_read_transport_auth_adc(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + adc.assert_called_once_with( + scopes=["1", "2"], + default_scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/cloud-platform',), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, + ], +) +def test_big_query_read_transport_auth_gdch_credentials(transport_class): + host = 'https://language.com' + api_audience_tests = [None, 'https://language2.com'] + api_audience_expect = [host, 'https://language2.com'] + for t, e in zip(api_audience_tests, api_audience_expect): + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + gdch_mock = mock.MagicMock() + type(gdch_mock).with_gdch_audience = mock.PropertyMock(return_value=gdch_mock) + adc.return_value = (gdch_mock, None) + transport_class(host=host, api_audience=t) + gdch_mock.with_gdch_audience.assert_called_once_with( + e + ) + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.BigQueryReadGrpcTransport, grpc_helpers), + (transports.BigQueryReadGrpcAsyncIOTransport, grpc_helpers_async) + ], +) +def test_big_query_read_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class( + quota_project_id="octopus", + scopes=["1", "2"] + ) + + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=["1", "2"], + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_grpc_transport_client_cert_source_for_mtls( + transport_class +): + cred = ga_credentials.AnonymousCredentials() + + # Check ssl_channel_credentials is used if provided. + with mock.patch.object(transport_class, "create_channel") as mock_create_channel: + mock_ssl_channel_creds = mock.Mock() + transport_class( + host="squid.clam.whelk", + credentials=cred, + ssl_channel_credentials=mock_ssl_channel_creds + ) + mock_create_channel.assert_called_once_with( + "squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_channel_creds, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls + # is used. + with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): + with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: + transport_class( + credentials=cred, + client_cert_source_for_mtls=client_cert_source_callback + ) + expected_cert, expected_key = client_cert_source_callback() + mock_ssl_cred.assert_called_once_with( + certificate_chain=expected_cert, + private_key=expected_key + ) + + +@pytest.mark.parametrize("transport_name", [ + "grpc", + "grpc_asyncio", +]) +def test_big_query_read_host_no_port(transport_name): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com'), + transport=transport_name, + ) + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + +@pytest.mark.parametrize("transport_name", [ + "grpc", + "grpc_asyncio", +]) +def test_big_query_read_host_with_port(transport_name): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com:8000'), + transport=transport_name, + ) + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:8000' + ) + +def test_big_query_read_grpc_transport_channel(): + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryReadGrpcTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +def test_big_query_read_grpc_asyncio_transport_channel(): + channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryReadGrpcAsyncIOTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_transport_channel_mtls_with_client_cert_source( + transport_class +): + with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred: + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_ssl_cred = mock.Mock() + grpc_ssl_channel_cred.return_value = mock_ssl_cred + + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + + cred = ga_credentials.AnonymousCredentials() + with pytest.warns(DeprecationWarning): + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (cred, None) + transport = transport_class( + host="squid.clam.whelk", + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=client_cert_source_callback, + ) + adc.assert_called_once() + + grpc_ssl_channel_cred.assert_called_once_with( + certificate_chain=b"cert bytes", private_key=b"key bytes" + ) + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + assert transport._ssl_channel_credentials == mock_ssl_cred + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_transport_channel_mtls_with_adc( + transport_class +): + mock_ssl_cred = mock.Mock() + with mock.patch.multiple( + "google.auth.transport.grpc.SslCredentials", + __init__=mock.Mock(return_value=None), + ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), + ): + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + mock_cred = mock.Mock() + + with pytest.warns(DeprecationWarning): + transport = transport_class( + host="squid.clam.whelk", + credentials=mock_cred, + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=None, + ) + + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=mock_cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + + +def test_read_session_path(): + project = "squid" + location = "clam" + session = "whelk" + expected = "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) + actual = BigQueryReadClient.read_session_path(project, location, session) + assert expected == actual + + +def test_parse_read_session_path(): + expected = { + "project": "octopus", + "location": "oyster", + "session": "nudibranch", + } + path = BigQueryReadClient.read_session_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_read_session_path(path) + assert expected == actual + +def test_read_stream_path(): + project = "cuttlefish" + location = "mussel" + session = "winkle" + stream = "nautilus" + expected = "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) + actual = BigQueryReadClient.read_stream_path(project, location, session, stream) + assert expected == actual + + +def test_parse_read_stream_path(): + expected = { + "project": "scallop", + "location": "abalone", + "session": "squid", + "stream": "clam", + } + path = BigQueryReadClient.read_stream_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_read_stream_path(path) + assert expected == actual + +def test_table_path(): + project = "whelk" + dataset = "octopus" + table = "oyster" + expected = "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + actual = BigQueryReadClient.table_path(project, dataset, table) + assert expected == actual + + +def test_parse_table_path(): + expected = { + "project": "nudibranch", + "dataset": "cuttlefish", + "table": "mussel", + } + path = BigQueryReadClient.table_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_table_path(path) + assert expected == actual + +def test_common_billing_account_path(): + billing_account = "winkle" + expected = "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + actual = BigQueryReadClient.common_billing_account_path(billing_account) + assert expected == actual + + +def test_parse_common_billing_account_path(): + expected = { + "billing_account": "nautilus", + } + path = BigQueryReadClient.common_billing_account_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_billing_account_path(path) + assert expected == actual + +def test_common_folder_path(): + folder = "scallop" + expected = "folders/{folder}".format(folder=folder, ) + actual = BigQueryReadClient.common_folder_path(folder) + assert expected == actual + + +def test_parse_common_folder_path(): + expected = { + "folder": "abalone", + } + path = BigQueryReadClient.common_folder_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_folder_path(path) + assert expected == actual + +def test_common_organization_path(): + organization = "squid" + expected = "organizations/{organization}".format(organization=organization, ) + actual = BigQueryReadClient.common_organization_path(organization) + assert expected == actual + + +def test_parse_common_organization_path(): + expected = { + "organization": "clam", + } + path = BigQueryReadClient.common_organization_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_organization_path(path) + assert expected == actual + +def test_common_project_path(): + project = "whelk" + expected = "projects/{project}".format(project=project, ) + actual = BigQueryReadClient.common_project_path(project) + assert expected == actual + + +def test_parse_common_project_path(): + expected = { + "project": "octopus", + } + path = BigQueryReadClient.common_project_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_project_path(path) + assert expected == actual + +def test_common_location_path(): + project = "oyster" + location = "nudibranch" + expected = "projects/{project}/locations/{location}".format(project=project, location=location, ) + actual = BigQueryReadClient.common_location_path(project, location) + assert expected == actual + + +def test_parse_common_location_path(): + expected = { + "project": "cuttlefish", + "location": "mussel", + } + path = BigQueryReadClient.common_location_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_location_path(path) + assert expected == actual + + +def test_client_with_default_client_info(): + client_info = gapic_v1.client_info.ClientInfo() + + with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: + transport_class = BigQueryReadClient.get_transport_class() + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + +def test_transport_close_grpc(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc" + ) + with mock.patch.object(type(getattr(client.transport, "_grpc_channel")), "close") as close: + with client: + close.assert_not_called() + close.assert_called_once() + + +@pytest.mark.asyncio +async def test_transport_close_grpc_asyncio(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio" + ) + with mock.patch.object(type(getattr(client.transport, "_grpc_channel")), "close") as close: + async with client: + close.assert_not_called() + close.assert_called_once() + + +def test_client_ctx(): + transports = [ + 'grpc', + ] + for transport in transports: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport + ) + # Test client calls underlying transport. + with mock.patch.object(type(client.transport), "close") as close: + close.assert_not_called() + with client: + pass + close.assert_called() + +@pytest.mark.parametrize("client_class,transport_class", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport), +]) +def test_api_key_credentials(client_class, transport_class): + with mock.patch.object( + google.auth._default, "get_api_key_credentials", create=True + ) as get_api_key_credentials: + mock_cred = mock.Mock() + get_api_key_credentials.return_value = mock_cred + options = client_options.ClientOptions() + options.api_key = "api_key" + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=mock_cred, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) diff --git a/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_write.py b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_write.py new file mode 100644 index 00000000..6b4ca866 --- /dev/null +++ b/owl-bot-staging/v1/tests/unit/gapic/bigquery_storage_v1/test_big_query_write.py @@ -0,0 +1,3422 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +# try/except added for compatibility with python < 3.8 +try: + from unittest import mock + from unittest.mock import AsyncMock # pragma: NO COVER +except ImportError: # pragma: NO COVER + import mock + +import grpc +from grpc.experimental import aio +import math +import pytest +from google.api_core import api_core_version +from proto.marshal.rules.dates import DurationRule, TimestampRule +from proto.marshal.rules import wrappers + +try: + from google.auth.aio import credentials as ga_credentials_async + HAS_GOOGLE_AUTH_AIO = True +except ImportError: # pragma: NO COVER + HAS_GOOGLE_AUTH_AIO = False + +from google.api_core import client_options +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers +from google.api_core import grpc_helpers_async +from google.api_core import path_template +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials +from google.auth.exceptions import MutualTLSChannelError +from google.cloud.bigquery_storage_v1.services.big_query_write import BigQueryWriteAsyncClient +from google.cloud.bigquery_storage_v1.services.big_query_write import BigQueryWriteClient +from google.cloud.bigquery_storage_v1.services.big_query_write import transports +from google.cloud.bigquery_storage_v1.types import arrow +from google.cloud.bigquery_storage_v1.types import protobuf +from google.cloud.bigquery_storage_v1.types import storage +from google.cloud.bigquery_storage_v1.types import stream +from google.cloud.bigquery_storage_v1.types import table +from google.oauth2 import service_account +from google.protobuf import descriptor_pb2 # type: ignore +from google.protobuf import timestamp_pb2 # type: ignore +from google.protobuf import wrappers_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +import google.auth + + +async def mock_async_gen(data, chunk_size=1): + for i in range(0, len(data)): # pragma: NO COVER + chunk = data[i : i + chunk_size] + yield chunk.encode("utf-8") + +def client_cert_source_callback(): + return b"cert bytes", b"key bytes" + +# TODO: use async auth anon credentials by default once the minimum version of google-auth is upgraded. +# See related issue: https://github.com/googleapis/gapic-generator-python/issues/2107. +def async_anonymous_credentials(): + if HAS_GOOGLE_AUTH_AIO: + return ga_credentials_async.AnonymousCredentials() + return ga_credentials.AnonymousCredentials() + +# If default endpoint is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint(client): + return "foo.googleapis.com" if ("localhost" in client.DEFAULT_ENDPOINT) else client.DEFAULT_ENDPOINT + +# If default endpoint template is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint template so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint_template(client): + return "test.{UNIVERSE_DOMAIN}" if ("localhost" in client._DEFAULT_ENDPOINT_TEMPLATE) else client._DEFAULT_ENDPOINT_TEMPLATE + + +def test__get_default_mtls_endpoint(): + api_endpoint = "example.googleapis.com" + api_mtls_endpoint = "example.mtls.googleapis.com" + sandbox_endpoint = "example.sandbox.googleapis.com" + sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" + non_googleapi = "api.example.com" + + assert BigQueryWriteClient._get_default_mtls_endpoint(None) is None + assert BigQueryWriteClient._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi + +def test__read_environment_variables(): + assert BigQueryWriteClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert BigQueryWriteClient._read_environment_variables() == (True, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + assert BigQueryWriteClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + BigQueryWriteClient._read_environment_variables() + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + assert BigQueryWriteClient._read_environment_variables() == (False, "never", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + assert BigQueryWriteClient._read_environment_variables() == (False, "always", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}): + assert BigQueryWriteClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + BigQueryWriteClient._read_environment_variables() + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + with mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}): + assert BigQueryWriteClient._read_environment_variables() == (False, "auto", "foo.com") + +def test__get_client_cert_source(): + mock_provided_cert_source = mock.Mock() + mock_default_cert_source = mock.Mock() + + assert BigQueryWriteClient._get_client_cert_source(None, False) is None + assert BigQueryWriteClient._get_client_cert_source(mock_provided_cert_source, False) is None + assert BigQueryWriteClient._get_client_cert_source(mock_provided_cert_source, True) == mock_provided_cert_source + + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=mock_default_cert_source): + assert BigQueryWriteClient._get_client_cert_source(None, True) is mock_default_cert_source + assert BigQueryWriteClient._get_client_cert_source(mock_provided_cert_source, "true") is mock_provided_cert_source + +@mock.patch.object(BigQueryWriteClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteAsyncClient)) +def test__get_api_endpoint(): + api_override = "foo.com" + mock_client_cert_source = mock.Mock() + default_universe = BigQueryWriteClient._DEFAULT_UNIVERSE + default_endpoint = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=default_universe) + mock_universe = "bar.com" + mock_endpoint = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=mock_universe) + + assert BigQueryWriteClient._get_api_endpoint(api_override, mock_client_cert_source, default_universe, "always") == api_override + assert BigQueryWriteClient._get_api_endpoint(None, mock_client_cert_source, default_universe, "auto") == BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryWriteClient._get_api_endpoint(None, None, default_universe, "auto") == default_endpoint + assert BigQueryWriteClient._get_api_endpoint(None, None, default_universe, "always") == BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryWriteClient._get_api_endpoint(None, mock_client_cert_source, default_universe, "always") == BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryWriteClient._get_api_endpoint(None, None, mock_universe, "never") == mock_endpoint + assert BigQueryWriteClient._get_api_endpoint(None, None, default_universe, "never") == default_endpoint + + with pytest.raises(MutualTLSChannelError) as excinfo: + BigQueryWriteClient._get_api_endpoint(None, mock_client_cert_source, mock_universe, "auto") + assert str(excinfo.value) == "mTLS is not supported in any universe other than googleapis.com." + + +def test__get_universe_domain(): + client_universe_domain = "foo.com" + universe_domain_env = "bar.com" + + assert BigQueryWriteClient._get_universe_domain(client_universe_domain, universe_domain_env) == client_universe_domain + assert BigQueryWriteClient._get_universe_domain(None, universe_domain_env) == universe_domain_env + assert BigQueryWriteClient._get_universe_domain(None, None) == BigQueryWriteClient._DEFAULT_UNIVERSE + + with pytest.raises(ValueError) as excinfo: + BigQueryWriteClient._get_universe_domain("", None) + assert str(excinfo.value) == "Universe Domain cannot be an empty string." + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), +]) +def test__validate_universe_domain(client_class, transport_class, transport_name): + client = client_class( + transport=transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + ) + assert client._validate_universe_domain() == True + + # Test the case when universe is already validated. + assert client._validate_universe_domain() == True + + if transport_name == "grpc": + # Test the case where credentials are provided by the + # `local_channel_credentials`. The default universes in both match. + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + client = client_class(transport=transport_class(channel=channel)) + assert client._validate_universe_domain() == True + + # Test the case where credentials do not exist: e.g. a transport is provided + # with no credentials. Validation should still succeed because there is no + # mismatch with non-existent credentials. + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + transport=transport_class(channel=channel) + transport._credentials = None + client = client_class(transport=transport) + assert client._validate_universe_domain() == True + + # TODO: This is needed to cater for older versions of google-auth + # Make this test unconditional once the minimum supported version of + # google-auth becomes 2.23.0 or higher. + google_auth_major, google_auth_minor = [int(part) for part in google.auth.__version__.split(".")[0:2]] + if google_auth_major > 2 or (google_auth_major == 2 and google_auth_minor >= 23): + credentials = ga_credentials.AnonymousCredentials() + credentials._universe_domain = "foo.com" + # Test the case when there is a universe mismatch from the credentials. + client = client_class( + transport=transport_class(credentials=credentials) + ) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert str(excinfo.value) == "The configured universe domain (googleapis.com) does not match the universe domain found in the credentials (foo.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + + # Test the case when there is a universe mismatch from the client. + # + # TODO: Make this test unconditional once the minimum supported version of + # google-api-core becomes 2.15.0 or higher. + api_core_major, api_core_minor = [int(part) for part in api_core_version.__version__.split(".")[0:2]] + if api_core_major > 2 or (api_core_major == 2 and api_core_minor >= 15): + client = client_class(client_options={"universe_domain": "bar.com"}, transport=transport_class(credentials=ga_credentials.AnonymousCredentials(),)) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert str(excinfo.value) == "The configured universe domain (bar.com) does not match the universe domain found in the credentials (googleapis.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + + # Test that ValueError is raised if universe_domain is provided via client options and credentials is None + with pytest.raises(ValueError): + client._compare_universes("foo.bar", None) + + +@pytest.mark.parametrize("client_class,transport_name", [ + (BigQueryWriteClient, "grpc"), + (BigQueryWriteAsyncClient, "grpc_asyncio"), +]) +def test_big_query_write_client_from_service_account_info(client_class, transport_name): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_info') as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info, transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + + +@pytest.mark.parametrize("transport_class,transport_name", [ + (transports.BigQueryWriteGrpcTransport, "grpc"), + (transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_write_client_service_account_always_use_jwt(transport_class, transport_name): + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize("client_class,transport_name", [ + (BigQueryWriteClient, "grpc"), + (BigQueryWriteAsyncClient, "grpc_asyncio"), +]) +def test_big_query_write_client_from_service_account_file(client_class, transport_name): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_file') as factory: + factory.return_value = creds + client = client_class.from_service_account_file("dummy/file/path.json", transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + client = client_class.from_service_account_json("dummy/file/path.json", transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + + +def test_big_query_write_client_get_transport_class(): + transport = BigQueryWriteClient.get_transport_class() + available_transports = [ + transports.BigQueryWriteGrpcTransport, + ] + assert transport in available_transports + + transport = BigQueryWriteClient.get_transport_class("grpc") + assert transport == transports.BigQueryWriteGrpcTransport + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), +]) +@mock.patch.object(BigQueryWriteClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteAsyncClient)) +def test_big_query_write_client_client_options(client_class, transport_class, transport_name): + # Check that if channel is provided we won't create a new one. + with mock.patch.object(BigQueryWriteClient, 'get_transport_class') as gtc: + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + client = client_class(transport=transport) + gtc.assert_not_called() + + # Check that if channel is provided via str we will create a new one. + with mock.patch.object(BigQueryWriteClient, 'get_transport_class') as gtc: + client = client_class(transport=transport_name) + gtc.assert_called() + + # Check the case api_endpoint is provided. + options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name, client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_MTLS_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client = client_class(transport=transport_name) + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + client = client_class(transport=transport_name) + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + # Check the case api_endpoint is provided + options = client_options.ClientOptions(api_audience="https://language.googleapis.com") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience="https://language.googleapis.com" + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,use_client_cert_env", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", "true"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", "true"), + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", "false"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", "false"), +]) +@mock.patch.object(BigQueryWriteClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteAsyncClient)) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_big_query_write_client_mtls_env_auto(client_class, transport_class, transport_name, use_client_cert_env): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + options = client_options.ClientOptions(client_cert_source=client_cert_source_callback) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE) + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT + + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=client_cert_source_callback): + if use_client_cert_env == "false": + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE) + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback + + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=False): + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + +@pytest.mark.parametrize("client_class", [ + BigQueryWriteClient, BigQueryWriteAsyncClient +]) +@mock.patch.object(BigQueryWriteClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteAsyncClient)) +def test_big_query_write_client_get_mtls_endpoint_and_cert_source(client_class): + mock_client_cert_source = mock.Mock() + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "true". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + mock_api_endpoint = "foo" + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source == mock_client_cert_source + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "false". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert doesn't exist. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=False): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert exists. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=mock_client_cert_source): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source == mock_client_cert_source + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + +@pytest.mark.parametrize("client_class", [ + BigQueryWriteClient, BigQueryWriteAsyncClient +]) +@mock.patch.object(BigQueryWriteClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteAsyncClient)) +def test_big_query_write_client_client_api_endpoint(client_class): + mock_client_cert_source = client_cert_source_callback + api_override = "foo.com" + default_universe = BigQueryWriteClient._DEFAULT_UNIVERSE + default_endpoint = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=default_universe) + mock_universe = "bar.com" + mock_endpoint = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=mock_universe) + + # If ClientOptions.api_endpoint is set and GOOGLE_API_USE_CLIENT_CERTIFICATE="true", + # use ClientOptions.api_endpoint as the api endpoint regardless. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch("google.auth.transport.requests.AuthorizedSession.configure_mtls_channel"): + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=api_override) + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == api_override + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="always", + # use the DEFAULT_MTLS_ENDPOINT as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + + # If ClientOptions.api_endpoint is not set, GOOGLE_API_USE_MTLS_ENDPOINT="auto" (default), + # GOOGLE_API_USE_CLIENT_CERTIFICATE="false" (default), default cert source doesn't exist, + # and ClientOptions.universe_domain="bar.com", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with universe domain as the api endpoint. + options = client_options.ClientOptions() + universe_exists = hasattr(options, "universe_domain") + if universe_exists: + options = client_options.ClientOptions(universe_domain=mock_universe) + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + else: + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == (mock_endpoint if universe_exists else default_endpoint) + assert client.universe_domain == (mock_universe if universe_exists else default_universe) + + # If ClientOptions does not have a universe domain attribute and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + options = client_options.ClientOptions() + if hasattr(options, "universe_domain"): + delattr(options, "universe_domain") + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_write_client_client_options_scopes(client_class, transport_class, transport_name): + # Check the case scopes are provided. + options = client_options.ClientOptions( + scopes=["1", "2"], + ) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=["1", "2"], + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,grpc_helpers", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", grpc_helpers), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", grpc_helpers_async), +]) +def test_big_query_write_client_client_options_credentials_file(client_class, transport_class, transport_name, grpc_helpers): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + +def test_big_query_write_client_client_options_from_dict(): + with mock.patch('google.cloud.bigquery_storage_v1.services.big_query_write.transports.BigQueryWriteGrpcTransport.__init__') as grpc_transport: + grpc_transport.return_value = None + client = BigQueryWriteClient( + client_options={'api_endpoint': 'squid.clam.whelk'} + ) + grpc_transport.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + +@pytest.mark.parametrize("client_class,transport_class,transport_name,grpc_helpers", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", grpc_helpers), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", grpc_helpers_async), +]) +def test_big_query_write_client_create_channel_credentials_file(client_class, transport_class, transport_name, grpc_helpers): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=None, + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("request_type", [ + storage.CreateWriteStreamRequest, + dict, +]) +def test_create_write_stream(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + write_mode=stream.WriteStream.WriteMode.INSERT, + location='location_value', + ) + response = client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.CreateWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + assert response.write_mode == stream.WriteStream.WriteMode.INSERT + assert response.location == 'location_value' + + +def test_create_write_stream_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.CreateWriteStreamRequest( + parent='parent_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.create_write_stream(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateWriteStreamRequest( + parent='parent_value', + ) + +def test_create_write_stream_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.create_write_stream in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.create_write_stream] = mock_rpc + request = {} + client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.create_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_create_write_stream_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.create_write_stream in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.create_write_stream] = mock_rpc + + request = {} + await client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.create_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_create_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.CreateWriteStreamRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + write_mode=stream.WriteStream.WriteMode.INSERT, + location='location_value', + )) + response = await client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.CreateWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + assert response.write_mode == stream.WriteStream.WriteMode.INSERT + assert response.location == 'location_value' + + +@pytest.mark.asyncio +async def test_create_write_stream_async_from_dict(): + await test_create_write_stream_async(request_type=dict) + +def test_create_write_stream_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateWriteStreamRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + call.return_value = stream.WriteStream() + client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_create_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateWriteStreamRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + await client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +def test_create_write_stream_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.create_write_stream( + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + arg = args[0].write_stream + mock_val = stream.WriteStream(name='name_value') + assert arg == mock_val + + +def test_create_write_stream_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.create_write_stream( + storage.CreateWriteStreamRequest(), + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + +@pytest.mark.asyncio +async def test_create_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.create_write_stream( + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + arg = args[0].write_stream + mock_val = stream.WriteStream(name='name_value') + assert arg == mock_val + +@pytest.mark.asyncio +async def test_create_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.create_write_stream( + storage.CreateWriteStreamRequest(), + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + + +@pytest.mark.parametrize("request_type", [ + storage.AppendRowsRequest, + dict, +]) +def test_append_rows(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + requests = [request] + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.append_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.AppendRowsResponse()]) + response = client.append_rows(iter(requests)) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert next(args[0]) == request + + # Establish that the response is the type that we expect. + for message in response: + assert isinstance(message, storage.AppendRowsResponse) + + +def test_append_rows_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.append_rows in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.append_rows] = mock_rpc + request = [{}] + client.append_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.append_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_append_rows_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.append_rows in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.append_rows] = mock_rpc + + request = [{}] + await client.append_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.append_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_append_rows_async(transport: str = 'grpc_asyncio', request_type=storage.AppendRowsRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + requests = [request] + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.append_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = mock.Mock(aio.StreamStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.AppendRowsResponse()]) + response = await client.append_rows(iter(requests)) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert next(args[0]) == request + + # Establish that the response is the type that we expect. + message = await response.read() + assert isinstance(message, storage.AppendRowsResponse) + + +@pytest.mark.asyncio +async def test_append_rows_async_from_dict(): + await test_append_rows_async(request_type=dict) + + +@pytest.mark.parametrize("request_type", [ + storage.GetWriteStreamRequest, + dict, +]) +def test_get_write_stream(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + write_mode=stream.WriteStream.WriteMode.INSERT, + location='location_value', + ) + response = client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.GetWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + assert response.write_mode == stream.WriteStream.WriteMode.INSERT + assert response.location == 'location_value' + + +def test_get_write_stream_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.GetWriteStreamRequest( + name='name_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.get_write_stream(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.GetWriteStreamRequest( + name='name_value', + ) + +def test_get_write_stream_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.get_write_stream in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.get_write_stream] = mock_rpc + request = {} + client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.get_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_get_write_stream_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.get_write_stream in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.get_write_stream] = mock_rpc + + request = {} + await client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.get_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_get_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.GetWriteStreamRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + write_mode=stream.WriteStream.WriteMode.INSERT, + location='location_value', + )) + response = await client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.GetWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + assert response.write_mode == stream.WriteStream.WriteMode.INSERT + assert response.location == 'location_value' + + +@pytest.mark.asyncio +async def test_get_write_stream_async_from_dict(): + await test_get_write_stream_async(request_type=dict) + +def test_get_write_stream_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.GetWriteStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + call.return_value = stream.WriteStream() + client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_get_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.GetWriteStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + await client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +def test_get_write_stream_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.get_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].name + mock_val = 'name_value' + assert arg == mock_val + + +def test_get_write_stream_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.get_write_stream( + storage.GetWriteStreamRequest(), + name='name_value', + ) + +@pytest.mark.asyncio +async def test_get_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.get_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].name + mock_val = 'name_value' + assert arg == mock_val + +@pytest.mark.asyncio +async def test_get_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.get_write_stream( + storage.GetWriteStreamRequest(), + name='name_value', + ) + + +@pytest.mark.parametrize("request_type", [ + storage.FinalizeWriteStreamRequest, + dict, +]) +def test_finalize_write_stream(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse( + row_count=992, + ) + response = client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.FinalizeWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FinalizeWriteStreamResponse) + assert response.row_count == 992 + + +def test_finalize_write_stream_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.FinalizeWriteStreamRequest( + name='name_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.finalize_write_stream(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FinalizeWriteStreamRequest( + name='name_value', + ) + +def test_finalize_write_stream_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.finalize_write_stream in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.finalize_write_stream] = mock_rpc + request = {} + client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.finalize_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_finalize_write_stream_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.finalize_write_stream in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.finalize_write_stream] = mock_rpc + + request = {} + await client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.finalize_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_finalize_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.FinalizeWriteStreamRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse( + row_count=992, + )) + response = await client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.FinalizeWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FinalizeWriteStreamResponse) + assert response.row_count == 992 + + +@pytest.mark.asyncio +async def test_finalize_write_stream_async_from_dict(): + await test_finalize_write_stream_async(request_type=dict) + +def test_finalize_write_stream_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FinalizeWriteStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + call.return_value = storage.FinalizeWriteStreamResponse() + client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_finalize_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FinalizeWriteStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse()) + await client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +def test_finalize_write_stream_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.finalize_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].name + mock_val = 'name_value' + assert arg == mock_val + + +def test_finalize_write_stream_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.finalize_write_stream( + storage.FinalizeWriteStreamRequest(), + name='name_value', + ) + +@pytest.mark.asyncio +async def test_finalize_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.finalize_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].name + mock_val = 'name_value' + assert arg == mock_val + +@pytest.mark.asyncio +async def test_finalize_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.finalize_write_stream( + storage.FinalizeWriteStreamRequest(), + name='name_value', + ) + + +@pytest.mark.parametrize("request_type", [ + storage.BatchCommitWriteStreamsRequest, + dict, +]) +def test_batch_commit_write_streams(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse( + ) + response = client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.BatchCommitWriteStreamsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.BatchCommitWriteStreamsResponse) + + +def test_batch_commit_write_streams_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.BatchCommitWriteStreamsRequest( + parent='parent_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.batch_commit_write_streams(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.BatchCommitWriteStreamsRequest( + parent='parent_value', + ) + +def test_batch_commit_write_streams_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.batch_commit_write_streams in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.batch_commit_write_streams] = mock_rpc + request = {} + client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.batch_commit_write_streams(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.batch_commit_write_streams in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.batch_commit_write_streams] = mock_rpc + + request = {} + await client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.batch_commit_write_streams(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_async(transport: str = 'grpc_asyncio', request_type=storage.BatchCommitWriteStreamsRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse( + )) + response = await client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.BatchCommitWriteStreamsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.BatchCommitWriteStreamsResponse) + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_async_from_dict(): + await test_batch_commit_write_streams_async(request_type=dict) + +def test_batch_commit_write_streams_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.BatchCommitWriteStreamsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + call.return_value = storage.BatchCommitWriteStreamsResponse() + client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.BatchCommitWriteStreamsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse()) + await client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +def test_batch_commit_write_streams_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.batch_commit_write_streams( + parent='parent_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + + +def test_batch_commit_write_streams_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.batch_commit_write_streams( + storage.BatchCommitWriteStreamsRequest(), + parent='parent_value', + ) + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.batch_commit_write_streams( + parent='parent_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.batch_commit_write_streams( + storage.BatchCommitWriteStreamsRequest(), + parent='parent_value', + ) + + +@pytest.mark.parametrize("request_type", [ + storage.FlushRowsRequest, + dict, +]) +def test_flush_rows(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse( + offset=647, + ) + response = client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.FlushRowsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FlushRowsResponse) + assert response.offset == 647 + + +def test_flush_rows_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.FlushRowsRequest( + write_stream='write_stream_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.flush_rows(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FlushRowsRequest( + write_stream='write_stream_value', + ) + +def test_flush_rows_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.flush_rows in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.flush_rows] = mock_rpc + request = {} + client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.flush_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_flush_rows_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.flush_rows in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.flush_rows] = mock_rpc + + request = {} + await client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.flush_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_flush_rows_async(transport: str = 'grpc_asyncio', request_type=storage.FlushRowsRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse( + offset=647, + )) + response = await client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.FlushRowsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FlushRowsResponse) + assert response.offset == 647 + + +@pytest.mark.asyncio +async def test_flush_rows_async_from_dict(): + await test_flush_rows_async(request_type=dict) + +def test_flush_rows_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FlushRowsRequest() + + request.write_stream = 'write_stream_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + call.return_value = storage.FlushRowsResponse() + client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'write_stream=write_stream_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_flush_rows_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FlushRowsRequest() + + request.write_stream = 'write_stream_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse()) + await client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'write_stream=write_stream_value', + ) in kw['metadata'] + + +def test_flush_rows_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.flush_rows( + write_stream='write_stream_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].write_stream + mock_val = 'write_stream_value' + assert arg == mock_val + + +def test_flush_rows_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.flush_rows( + storage.FlushRowsRequest(), + write_stream='write_stream_value', + ) + +@pytest.mark.asyncio +async def test_flush_rows_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.flush_rows( + write_stream='write_stream_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].write_stream + mock_val = 'write_stream_value' + assert arg == mock_val + +@pytest.mark.asyncio +async def test_flush_rows_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.flush_rows( + storage.FlushRowsRequest(), + write_stream='write_stream_value', + ) + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide an api_key and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options=options, + transport=transport, + ) + + # It is an error to provide an api_key and a credential. + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options=options, + credentials=ga_credentials.AnonymousCredentials() + ) + + # It is an error to provide scopes and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options={"scopes": ["1", "2"]}, + transport=transport, + ) + + +def test_transport_instance(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + client = BigQueryWriteClient(transport=transport) + assert client.transport is transport + +def test_transport_get_channel(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + + transport = transports.BigQueryWriteGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + +@pytest.mark.parametrize("transport_class", [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, +]) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + +def test_transport_kind_grpc(): + transport = BigQueryWriteClient.get_transport_class("grpc")( + credentials=ga_credentials.AnonymousCredentials() + ) + assert transport.kind == "grpc" + + +def test_initialize_client_w_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc" + ) + assert client is not None + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_create_write_stream_empty_call_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + call.return_value = stream.WriteStream() + client.create_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.CreateWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_get_write_stream_empty_call_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + call.return_value = stream.WriteStream() + client.get_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.GetWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_finalize_write_stream_empty_call_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + call.return_value = storage.FinalizeWriteStreamResponse() + client.finalize_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.FinalizeWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_batch_commit_write_streams_empty_call_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + call.return_value = storage.BatchCommitWriteStreamsResponse() + client.batch_commit_write_streams(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.BatchCommitWriteStreamsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_flush_rows_empty_call_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + call.return_value = storage.FlushRowsResponse() + client.flush_rows(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.FlushRowsRequest() + + assert args[0] == request_msg + + +def test_transport_kind_grpc_asyncio(): + transport = BigQueryWriteAsyncClient.get_transport_class("grpc_asyncio")( + credentials=async_anonymous_credentials() + ) + assert transport.kind == "grpc_asyncio" + + +def test_initialize_client_w_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio" + ) + assert client is not None + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_create_write_stream_empty_call_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + write_mode=stream.WriteStream.WriteMode.INSERT, + location='location_value', + )) + await client.create_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.CreateWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_get_write_stream_empty_call_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + write_mode=stream.WriteStream.WriteMode.INSERT, + location='location_value', + )) + await client.get_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.GetWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_finalize_write_stream_empty_call_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse( + row_count=992, + )) + await client.finalize_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.FinalizeWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_batch_commit_write_streams_empty_call_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse( + )) + await client.batch_commit_write_streams(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.BatchCommitWriteStreamsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_flush_rows_empty_call_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse( + offset=647, + )) + await client.flush_rows(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.FlushRowsRequest() + + assert args[0] == request_msg + + +def test_transport_grpc_default(): + # A client should use the gRPC transport by default. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert isinstance( + client.transport, + transports.BigQueryWriteGrpcTransport, + ) + +def test_big_query_write_base_transport_error(): + # Passing both a credentials object and credentials_file should raise an error + with pytest.raises(core_exceptions.DuplicateCredentialArgs): + transport = transports.BigQueryWriteTransport( + credentials=ga_credentials.AnonymousCredentials(), + credentials_file="credentials.json" + ) + + +def test_big_query_write_base_transport(): + # Instantiate the base transport. + with mock.patch('google.cloud.bigquery_storage_v1.services.big_query_write.transports.BigQueryWriteTransport.__init__') as Transport: + Transport.return_value = None + transport = transports.BigQueryWriteTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Every method on the transport should just blindly + # raise NotImplementedError. + methods = ( + 'create_write_stream', + 'append_rows', + 'get_write_stream', + 'finalize_write_stream', + 'batch_commit_write_streams', + 'flush_rows', + ) + for method in methods: + with pytest.raises(NotImplementedError): + getattr(transport, method)(request=object()) + + with pytest.raises(NotImplementedError): + transport.close() + + # Catch all for all remaining methods and properties + remainder = [ + 'kind', + ] + for r in remainder: + with pytest.raises(NotImplementedError): + getattr(transport, r)() + + +def test_big_query_write_base_transport_with_credentials_file(): + # Instantiate the base transport with a credentials file + with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryWriteTransport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id="octopus", + ) + + +def test_big_query_write_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(google.auth, 'default', autospec=True) as adc, mock.patch('google.cloud.bigquery_storage_v1.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryWriteTransport() + adc.assert_called_once() + + +def test_big_query_write_auth_adc(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryWriteClient() + adc.assert_called_once_with( + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id=None, + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +def test_big_query_write_transport_auth_adc(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + adc.assert_called_once_with( + scopes=["1", "2"], + default_scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.insertdata', 'https://www.googleapis.com/auth/cloud-platform',), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +def test_big_query_write_transport_auth_gdch_credentials(transport_class): + host = 'https://language.com' + api_audience_tests = [None, 'https://language2.com'] + api_audience_expect = [host, 'https://language2.com'] + for t, e in zip(api_audience_tests, api_audience_expect): + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + gdch_mock = mock.MagicMock() + type(gdch_mock).with_gdch_audience = mock.PropertyMock(return_value=gdch_mock) + adc.return_value = (gdch_mock, None) + transport_class(host=host, api_audience=t) + gdch_mock.with_gdch_audience.assert_called_once_with( + e + ) + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.BigQueryWriteGrpcTransport, grpc_helpers), + (transports.BigQueryWriteGrpcAsyncIOTransport, grpc_helpers_async) + ], +) +def test_big_query_write_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class( + quota_project_id="octopus", + scopes=["1", "2"] + ) + + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=["1", "2"], + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) +def test_big_query_write_grpc_transport_client_cert_source_for_mtls( + transport_class +): + cred = ga_credentials.AnonymousCredentials() + + # Check ssl_channel_credentials is used if provided. + with mock.patch.object(transport_class, "create_channel") as mock_create_channel: + mock_ssl_channel_creds = mock.Mock() + transport_class( + host="squid.clam.whelk", + credentials=cred, + ssl_channel_credentials=mock_ssl_channel_creds + ) + mock_create_channel.assert_called_once_with( + "squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_channel_creds, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls + # is used. + with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): + with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: + transport_class( + credentials=cred, + client_cert_source_for_mtls=client_cert_source_callback + ) + expected_cert, expected_key = client_cert_source_callback() + mock_ssl_cred.assert_called_once_with( + certificate_chain=expected_cert, + private_key=expected_key + ) + + +@pytest.mark.parametrize("transport_name", [ + "grpc", + "grpc_asyncio", +]) +def test_big_query_write_host_no_port(transport_name): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com'), + transport=transport_name, + ) + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + +@pytest.mark.parametrize("transport_name", [ + "grpc", + "grpc_asyncio", +]) +def test_big_query_write_host_with_port(transport_name): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com:8000'), + transport=transport_name, + ) + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:8000' + ) + +def test_big_query_write_grpc_transport_channel(): + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryWriteGrpcTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +def test_big_query_write_grpc_asyncio_transport_channel(): + channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryWriteGrpcAsyncIOTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) +def test_big_query_write_transport_channel_mtls_with_client_cert_source( + transport_class +): + with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred: + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_ssl_cred = mock.Mock() + grpc_ssl_channel_cred.return_value = mock_ssl_cred + + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + + cred = ga_credentials.AnonymousCredentials() + with pytest.warns(DeprecationWarning): + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (cred, None) + transport = transport_class( + host="squid.clam.whelk", + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=client_cert_source_callback, + ) + adc.assert_called_once() + + grpc_ssl_channel_cred.assert_called_once_with( + certificate_chain=b"cert bytes", private_key=b"key bytes" + ) + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + assert transport._ssl_channel_credentials == mock_ssl_cred + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) +def test_big_query_write_transport_channel_mtls_with_adc( + transport_class +): + mock_ssl_cred = mock.Mock() + with mock.patch.multiple( + "google.auth.transport.grpc.SslCredentials", + __init__=mock.Mock(return_value=None), + ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), + ): + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + mock_cred = mock.Mock() + + with pytest.warns(DeprecationWarning): + transport = transport_class( + host="squid.clam.whelk", + credentials=mock_cred, + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=None, + ) + + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=mock_cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + + +def test_table_path(): + project = "squid" + dataset = "clam" + table = "whelk" + expected = "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + actual = BigQueryWriteClient.table_path(project, dataset, table) + assert expected == actual + + +def test_parse_table_path(): + expected = { + "project": "octopus", + "dataset": "oyster", + "table": "nudibranch", + } + path = BigQueryWriteClient.table_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_table_path(path) + assert expected == actual + +def test_write_stream_path(): + project = "cuttlefish" + dataset = "mussel" + table = "winkle" + stream = "nautilus" + expected = "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}".format(project=project, dataset=dataset, table=table, stream=stream, ) + actual = BigQueryWriteClient.write_stream_path(project, dataset, table, stream) + assert expected == actual + + +def test_parse_write_stream_path(): + expected = { + "project": "scallop", + "dataset": "abalone", + "table": "squid", + "stream": "clam", + } + path = BigQueryWriteClient.write_stream_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_write_stream_path(path) + assert expected == actual + +def test_common_billing_account_path(): + billing_account = "whelk" + expected = "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + actual = BigQueryWriteClient.common_billing_account_path(billing_account) + assert expected == actual + + +def test_parse_common_billing_account_path(): + expected = { + "billing_account": "octopus", + } + path = BigQueryWriteClient.common_billing_account_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_billing_account_path(path) + assert expected == actual + +def test_common_folder_path(): + folder = "oyster" + expected = "folders/{folder}".format(folder=folder, ) + actual = BigQueryWriteClient.common_folder_path(folder) + assert expected == actual + + +def test_parse_common_folder_path(): + expected = { + "folder": "nudibranch", + } + path = BigQueryWriteClient.common_folder_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_folder_path(path) + assert expected == actual + +def test_common_organization_path(): + organization = "cuttlefish" + expected = "organizations/{organization}".format(organization=organization, ) + actual = BigQueryWriteClient.common_organization_path(organization) + assert expected == actual + + +def test_parse_common_organization_path(): + expected = { + "organization": "mussel", + } + path = BigQueryWriteClient.common_organization_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_organization_path(path) + assert expected == actual + +def test_common_project_path(): + project = "winkle" + expected = "projects/{project}".format(project=project, ) + actual = BigQueryWriteClient.common_project_path(project) + assert expected == actual + + +def test_parse_common_project_path(): + expected = { + "project": "nautilus", + } + path = BigQueryWriteClient.common_project_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_project_path(path) + assert expected == actual + +def test_common_location_path(): + project = "scallop" + location = "abalone" + expected = "projects/{project}/locations/{location}".format(project=project, location=location, ) + actual = BigQueryWriteClient.common_location_path(project, location) + assert expected == actual + + +def test_parse_common_location_path(): + expected = { + "project": "squid", + "location": "clam", + } + path = BigQueryWriteClient.common_location_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_location_path(path) + assert expected == actual + + +def test_client_with_default_client_info(): + client_info = gapic_v1.client_info.ClientInfo() + + with mock.patch.object(transports.BigQueryWriteTransport, '_prep_wrapped_messages') as prep: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + with mock.patch.object(transports.BigQueryWriteTransport, '_prep_wrapped_messages') as prep: + transport_class = BigQueryWriteClient.get_transport_class() + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + +def test_transport_close_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc" + ) + with mock.patch.object(type(getattr(client.transport, "_grpc_channel")), "close") as close: + with client: + close.assert_not_called() + close.assert_called_once() + + +@pytest.mark.asyncio +async def test_transport_close_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio" + ) + with mock.patch.object(type(getattr(client.transport, "_grpc_channel")), "close") as close: + async with client: + close.assert_not_called() + close.assert_called_once() + + +def test_client_ctx(): + transports = [ + 'grpc', + ] + for transport in transports: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport + ) + # Test client calls underlying transport. + with mock.patch.object(type(client.transport), "close") as close: + close.assert_not_called() + with client: + pass + close.assert_called() + +@pytest.mark.parametrize("client_class,transport_class", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport), +]) +def test_api_key_credentials(client_class, transport_class): + with mock.patch.object( + google.auth._default, "get_api_key_credentials", create=True + ) as get_api_key_credentials: + mock_cred = mock.Mock() + get_api_key_credentials.return_value = mock_cred + options = client_options.ClientOptions() + options.api_key = "api_key" + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=mock_cred, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) diff --git a/owl-bot-staging/v1alpha/.coveragerc b/owl-bot-staging/v1alpha/.coveragerc new file mode 100644 index 00000000..23d2208d --- /dev/null +++ b/owl-bot-staging/v1alpha/.coveragerc @@ -0,0 +1,13 @@ +[run] +branch = True + +[report] +show_missing = True +omit = + google/cloud/bigquery_storage/__init__.py + google/cloud/bigquery_storage/gapic_version.py +exclude_lines = + # Re-enable the standard pragma + pragma: NO COVER + # Ignore debug-only repr + def __repr__ diff --git a/owl-bot-staging/v1alpha/.flake8 b/owl-bot-staging/v1alpha/.flake8 new file mode 100644 index 00000000..29227d4c --- /dev/null +++ b/owl-bot-staging/v1alpha/.flake8 @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! +[flake8] +ignore = E203, E266, E501, W503 +exclude = + # Exclude generated code. + **/proto/** + **/gapic/** + **/services/** + **/types/** + *_pb2.py + + # Standard linting exemptions. + **/.nox/** + __pycache__, + .git, + *.pyc, + conf.py diff --git a/owl-bot-staging/v1alpha/MANIFEST.in b/owl-bot-staging/v1alpha/MANIFEST.in new file mode 100644 index 00000000..6b8f0276 --- /dev/null +++ b/owl-bot-staging/v1alpha/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include google/cloud/bigquery_storage *.py +recursive-include google/cloud/bigquery_storage_v1alpha *.py diff --git a/owl-bot-staging/v1alpha/README.rst b/owl-bot-staging/v1alpha/README.rst new file mode 100644 index 00000000..13a45c14 --- /dev/null +++ b/owl-bot-staging/v1alpha/README.rst @@ -0,0 +1,49 @@ +Python Client for Google Cloud Bigquery Storage API +================================================= + +Quick Start +----------- + +In order to use this library, you first need to go through the following steps: + +1. `Select or create a Cloud Platform project.`_ +2. `Enable billing for your project.`_ +3. Enable the Google Cloud Bigquery Storage API. +4. `Setup Authentication.`_ + +.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project +.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project +.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html + +Installation +~~~~~~~~~~~~ + +Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to +create isolated Python environments. The basic problem it addresses is one of +dependencies and versions, and indirectly permissions. + +With `virtualenv`_, it's possible to install this library without needing system +install permissions, and without clashing with the installed system +dependencies. + +.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ + + +Mac/Linux +^^^^^^^^^ + +.. code-block:: console + + python3 -m venv + source /bin/activate + /bin/pip install /path/to/library + + +Windows +^^^^^^^ + +.. code-block:: console + + python3 -m venv + \Scripts\activate + \Scripts\pip.exe install \path\to\library diff --git a/owl-bot-staging/v1alpha/docs/_static/custom.css b/owl-bot-staging/v1alpha/docs/_static/custom.css new file mode 100644 index 00000000..06423be0 --- /dev/null +++ b/owl-bot-staging/v1alpha/docs/_static/custom.css @@ -0,0 +1,3 @@ +dl.field-list > dt { + min-width: 100px +} diff --git a/owl-bot-staging/v1alpha/docs/bigquery_storage_v1alpha/metastore_partition_service.rst b/owl-bot-staging/v1alpha/docs/bigquery_storage_v1alpha/metastore_partition_service.rst new file mode 100644 index 00000000..94af0211 --- /dev/null +++ b/owl-bot-staging/v1alpha/docs/bigquery_storage_v1alpha/metastore_partition_service.rst @@ -0,0 +1,6 @@ +MetastorePartitionService +------------------------------------------- + +.. automodule:: google.cloud.bigquery_storage_v1alpha.services.metastore_partition_service + :members: + :inherited-members: diff --git a/owl-bot-staging/v1alpha/docs/bigquery_storage_v1alpha/services_.rst b/owl-bot-staging/v1alpha/docs/bigquery_storage_v1alpha/services_.rst new file mode 100644 index 00000000..df517182 --- /dev/null +++ b/owl-bot-staging/v1alpha/docs/bigquery_storage_v1alpha/services_.rst @@ -0,0 +1,6 @@ +Services for Google Cloud Bigquery Storage v1alpha API +====================================================== +.. toctree:: + :maxdepth: 2 + + metastore_partition_service diff --git a/owl-bot-staging/v1alpha/docs/bigquery_storage_v1alpha/types_.rst b/owl-bot-staging/v1alpha/docs/bigquery_storage_v1alpha/types_.rst new file mode 100644 index 00000000..4a91fc13 --- /dev/null +++ b/owl-bot-staging/v1alpha/docs/bigquery_storage_v1alpha/types_.rst @@ -0,0 +1,6 @@ +Types for Google Cloud Bigquery Storage v1alpha API +=================================================== + +.. automodule:: google.cloud.bigquery_storage_v1alpha.types + :members: + :show-inheritance: diff --git a/owl-bot-staging/v1alpha/docs/conf.py b/owl-bot-staging/v1alpha/docs/conf.py new file mode 100644 index 00000000..33c4ea53 --- /dev/null +++ b/owl-bot-staging/v1alpha/docs/conf.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# google-cloud-bigquery-storage documentation build configuration file +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath("..")) + +__version__ = "0.1.0" + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +needs_sphinx = "4.0.1" + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.viewcode", +] + +# autodoc/autosummary flags +autoclass_content = "both" +autodoc_default_flags = ["members"] +autosummary_generate = True + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# Allow markdown includes (so releases.md can include CHANGLEOG.md) +# http://www.sphinx-doc.org/en/master/markdown.html +source_parsers = {".md": "recommonmark.parser.CommonMarkParser"} + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +source_suffix = [".rst", ".md"] + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The root toctree document. +root_doc = "index" + +# General information about the project. +project = u"google-cloud-bigquery-storage" +copyright = u"2023, Google, LLC" +author = u"Google APIs" # TODO: autogenerate this bit + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The full version, including alpha/beta/rc tags. +release = __version__ +# The short X.Y version. +version = ".".join(release.split(".")[0:2]) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ["_build"] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "alabaster" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + "description": "Google Cloud Client Libraries for Python", + "github_user": "googleapis", + "github_repo": "google-cloud-python", + "github_banner": True, + "font_family": "'Roboto', Georgia, sans", + "head_font_family": "'Roboto', Georgia, serif", + "code_font_family": "'Roboto Mono', 'Consolas', monospace", +} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = "google-cloud-bigquery-storage-doc" + +# -- Options for warnings ------------------------------------------------------ + + +suppress_warnings = [ + # Temporarily suppress this to avoid "more than one target found for + # cross-reference" warning, which are intractable for us to avoid while in + # a mono-repo. + # See https://github.com/sphinx-doc/sphinx/blob + # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843 + "ref.python" +] + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', + # Latex figure (float) alignment + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + root_doc, + "google-cloud-bigquery-storage.tex", + u"google-cloud-bigquery-storage Documentation", + author, + "manual", + ) +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ( + root_doc, + "google-cloud-bigquery-storage", + u"Google Cloud Bigquery Storage Documentation", + [author], + 1, + ) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + root_doc, + "google-cloud-bigquery-storage", + u"google-cloud-bigquery-storage Documentation", + author, + "google-cloud-bigquery-storage", + "GAPIC library for Google Cloud Bigquery Storage API", + "APIs", + ) +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + "python": ("http://python.readthedocs.org/en/latest/", None), + "gax": ("https://gax-python.readthedocs.org/en/latest/", None), + "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), + "google-gax": ("https://gax-python.readthedocs.io/en/latest/", None), + "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None), + "grpc": ("https://grpc.io/grpc/python/", None), + "requests": ("http://requests.kennethreitz.org/en/stable/", None), + "proto": ("https://proto-plus-python.readthedocs.io/en/stable", None), + "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), +} + + +# Napoleon settings +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = True +napoleon_use_admonition_for_examples = False +napoleon_use_admonition_for_notes = False +napoleon_use_admonition_for_references = False +napoleon_use_ivar = False +napoleon_use_param = True +napoleon_use_rtype = True diff --git a/owl-bot-staging/v1alpha/docs/index.rst b/owl-bot-staging/v1alpha/docs/index.rst new file mode 100644 index 00000000..a6c2fa8c --- /dev/null +++ b/owl-bot-staging/v1alpha/docs/index.rst @@ -0,0 +1,7 @@ +API Reference +------------- +.. toctree:: + :maxdepth: 2 + + bigquery_storage_v1alpha/services + bigquery_storage_v1alpha/types diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage/__init__.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage/__init__.py new file mode 100644 index 00000000..0673a702 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage/__init__.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from google.cloud.bigquery_storage import gapic_version as package_version + +__version__ = package_version.__version__ + + +from google.cloud.bigquery_storage_v1alpha.services.metastore_partition_service.client import MetastorePartitionServiceClient +from google.cloud.bigquery_storage_v1alpha.services.metastore_partition_service.async_client import MetastorePartitionServiceAsyncClient + +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import BatchCreateMetastorePartitionsRequest +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import BatchCreateMetastorePartitionsResponse +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import BatchDeleteMetastorePartitionsRequest +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import BatchSizeTooLargeError +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import BatchUpdateMetastorePartitionsRequest +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import BatchUpdateMetastorePartitionsResponse +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import CreateMetastorePartitionRequest +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import ListMetastorePartitionsRequest +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import ListMetastorePartitionsResponse +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import StreamMetastorePartitionsRequest +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import StreamMetastorePartitionsResponse +from google.cloud.bigquery_storage_v1alpha.types.metastore_partition import UpdateMetastorePartitionRequest +from google.cloud.bigquery_storage_v1alpha.types.partition import FieldSchema +from google.cloud.bigquery_storage_v1alpha.types.partition import MetastorePartition +from google.cloud.bigquery_storage_v1alpha.types.partition import MetastorePartitionList +from google.cloud.bigquery_storage_v1alpha.types.partition import MetastorePartitionValues +from google.cloud.bigquery_storage_v1alpha.types.partition import ReadStream +from google.cloud.bigquery_storage_v1alpha.types.partition import SerDeInfo +from google.cloud.bigquery_storage_v1alpha.types.partition import StorageDescriptor +from google.cloud.bigquery_storage_v1alpha.types.partition import StreamList + +__all__ = ('MetastorePartitionServiceClient', + 'MetastorePartitionServiceAsyncClient', + 'BatchCreateMetastorePartitionsRequest', + 'BatchCreateMetastorePartitionsResponse', + 'BatchDeleteMetastorePartitionsRequest', + 'BatchSizeTooLargeError', + 'BatchUpdateMetastorePartitionsRequest', + 'BatchUpdateMetastorePartitionsResponse', + 'CreateMetastorePartitionRequest', + 'ListMetastorePartitionsRequest', + 'ListMetastorePartitionsResponse', + 'StreamMetastorePartitionsRequest', + 'StreamMetastorePartitionsResponse', + 'UpdateMetastorePartitionRequest', + 'FieldSchema', + 'MetastorePartition', + 'MetastorePartitionList', + 'MetastorePartitionValues', + 'ReadStream', + 'SerDeInfo', + 'StorageDescriptor', + 'StreamList', +) diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage/gapic_version.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage/gapic_version.py new file mode 100644 index 00000000..558c8aab --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage/gapic_version.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__version__ = "0.0.0" # {x-release-please-version} diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage/py.typed b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage/py.typed new file mode 100644 index 00000000..e71b4749 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/__init__.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/__init__.py new file mode 100644 index 00000000..7268aa66 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/__init__.py @@ -0,0 +1,68 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from google.cloud.bigquery_storage_v1alpha import gapic_version as package_version + +__version__ = package_version.__version__ + + +from .services.metastore_partition_service import MetastorePartitionServiceClient +from .services.metastore_partition_service import MetastorePartitionServiceAsyncClient + +from .types.metastore_partition import BatchCreateMetastorePartitionsRequest +from .types.metastore_partition import BatchCreateMetastorePartitionsResponse +from .types.metastore_partition import BatchDeleteMetastorePartitionsRequest +from .types.metastore_partition import BatchSizeTooLargeError +from .types.metastore_partition import BatchUpdateMetastorePartitionsRequest +from .types.metastore_partition import BatchUpdateMetastorePartitionsResponse +from .types.metastore_partition import CreateMetastorePartitionRequest +from .types.metastore_partition import ListMetastorePartitionsRequest +from .types.metastore_partition import ListMetastorePartitionsResponse +from .types.metastore_partition import StreamMetastorePartitionsRequest +from .types.metastore_partition import StreamMetastorePartitionsResponse +from .types.metastore_partition import UpdateMetastorePartitionRequest +from .types.partition import FieldSchema +from .types.partition import MetastorePartition +from .types.partition import MetastorePartitionList +from .types.partition import MetastorePartitionValues +from .types.partition import ReadStream +from .types.partition import SerDeInfo +from .types.partition import StorageDescriptor +from .types.partition import StreamList + +__all__ = ( + 'MetastorePartitionServiceAsyncClient', +'BatchCreateMetastorePartitionsRequest', +'BatchCreateMetastorePartitionsResponse', +'BatchDeleteMetastorePartitionsRequest', +'BatchSizeTooLargeError', +'BatchUpdateMetastorePartitionsRequest', +'BatchUpdateMetastorePartitionsResponse', +'CreateMetastorePartitionRequest', +'FieldSchema', +'ListMetastorePartitionsRequest', +'ListMetastorePartitionsResponse', +'MetastorePartition', +'MetastorePartitionList', +'MetastorePartitionServiceClient', +'MetastorePartitionValues', +'ReadStream', +'SerDeInfo', +'StorageDescriptor', +'StreamList', +'StreamMetastorePartitionsRequest', +'StreamMetastorePartitionsResponse', +'UpdateMetastorePartitionRequest', +) diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/gapic_metadata.json b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/gapic_metadata.json new file mode 100644 index 00000000..39f3429f --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/gapic_metadata.json @@ -0,0 +1,73 @@ + { + "comment": "This file maps proto services/RPCs to the corresponding library clients/methods", + "language": "python", + "libraryPackage": "google.cloud.bigquery_storage_v1alpha", + "protoPackage": "google.cloud.bigquery.storage.v1alpha", + "schema": "1.0", + "services": { + "MetastorePartitionService": { + "clients": { + "grpc": { + "libraryClient": "MetastorePartitionServiceClient", + "rpcs": { + "BatchCreateMetastorePartitions": { + "methods": [ + "batch_create_metastore_partitions" + ] + }, + "BatchDeleteMetastorePartitions": { + "methods": [ + "batch_delete_metastore_partitions" + ] + }, + "BatchUpdateMetastorePartitions": { + "methods": [ + "batch_update_metastore_partitions" + ] + }, + "ListMetastorePartitions": { + "methods": [ + "list_metastore_partitions" + ] + }, + "StreamMetastorePartitions": { + "methods": [ + "stream_metastore_partitions" + ] + } + } + }, + "grpc-async": { + "libraryClient": "MetastorePartitionServiceAsyncClient", + "rpcs": { + "BatchCreateMetastorePartitions": { + "methods": [ + "batch_create_metastore_partitions" + ] + }, + "BatchDeleteMetastorePartitions": { + "methods": [ + "batch_delete_metastore_partitions" + ] + }, + "BatchUpdateMetastorePartitions": { + "methods": [ + "batch_update_metastore_partitions" + ] + }, + "ListMetastorePartitions": { + "methods": [ + "list_metastore_partitions" + ] + }, + "StreamMetastorePartitions": { + "methods": [ + "stream_metastore_partitions" + ] + } + } + } + } + } + } +} diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/gapic_version.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/gapic_version.py new file mode 100644 index 00000000..558c8aab --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/gapic_version.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__version__ = "0.0.0" # {x-release-please-version} diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/py.typed b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/py.typed new file mode 100644 index 00000000..e71b4749 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/__init__.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/__init__.py new file mode 100644 index 00000000..8f6cf068 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/__init__.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/__init__.py new file mode 100644 index 00000000..6b45dda6 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .client import MetastorePartitionServiceClient +from .async_client import MetastorePartitionServiceAsyncClient + +__all__ = ( + 'MetastorePartitionServiceClient', + 'MetastorePartitionServiceAsyncClient', +) diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/async_client.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/async_client.py new file mode 100644 index 00000000..21c2ca94 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/async_client.py @@ -0,0 +1,695 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import re +from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, AsyncIterable, Awaitable, AsyncIterator, Sequence, Tuple, Type, Union + +from google.cloud.bigquery_storage_v1alpha import gapic_version as package_version + +from google.api_core.client_options import ClientOptions +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry_async as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + + +try: + OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.AsyncRetry, object, None] # type: ignore + +from google.cloud.bigquery_storage_v1alpha.types import metastore_partition +from google.cloud.bigquery_storage_v1alpha.types import partition +from .transports.base import MetastorePartitionServiceTransport, DEFAULT_CLIENT_INFO +from .transports.grpc_asyncio import MetastorePartitionServiceGrpcAsyncIOTransport +from .client import MetastorePartitionServiceClient + + +class MetastorePartitionServiceAsyncClient: + """BigQuery Metastore Partition Service API. + This service is used for managing metastore partitions in + BigQuery metastore. The service supports only batch operations + for write. + """ + + _client: MetastorePartitionServiceClient + + # Copy defaults from the synchronous client for use here. + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + DEFAULT_ENDPOINT = MetastorePartitionServiceClient.DEFAULT_ENDPOINT + DEFAULT_MTLS_ENDPOINT = MetastorePartitionServiceClient.DEFAULT_MTLS_ENDPOINT + _DEFAULT_ENDPOINT_TEMPLATE = MetastorePartitionServiceClient._DEFAULT_ENDPOINT_TEMPLATE + _DEFAULT_UNIVERSE = MetastorePartitionServiceClient._DEFAULT_UNIVERSE + + read_stream_path = staticmethod(MetastorePartitionServiceClient.read_stream_path) + parse_read_stream_path = staticmethod(MetastorePartitionServiceClient.parse_read_stream_path) + table_path = staticmethod(MetastorePartitionServiceClient.table_path) + parse_table_path = staticmethod(MetastorePartitionServiceClient.parse_table_path) + common_billing_account_path = staticmethod(MetastorePartitionServiceClient.common_billing_account_path) + parse_common_billing_account_path = staticmethod(MetastorePartitionServiceClient.parse_common_billing_account_path) + common_folder_path = staticmethod(MetastorePartitionServiceClient.common_folder_path) + parse_common_folder_path = staticmethod(MetastorePartitionServiceClient.parse_common_folder_path) + common_organization_path = staticmethod(MetastorePartitionServiceClient.common_organization_path) + parse_common_organization_path = staticmethod(MetastorePartitionServiceClient.parse_common_organization_path) + common_project_path = staticmethod(MetastorePartitionServiceClient.common_project_path) + parse_common_project_path = staticmethod(MetastorePartitionServiceClient.parse_common_project_path) + common_location_path = staticmethod(MetastorePartitionServiceClient.common_location_path) + parse_common_location_path = staticmethod(MetastorePartitionServiceClient.parse_common_location_path) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + MetastorePartitionServiceAsyncClient: The constructed client. + """ + return MetastorePartitionServiceClient.from_service_account_info.__func__(MetastorePartitionServiceAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + MetastorePartitionServiceAsyncClient: The constructed client. + """ + return MetastorePartitionServiceClient.from_service_account_file.__func__(MetastorePartitionServiceAsyncClient, filename, *args, **kwargs) # type: ignore + + from_service_account_json = from_service_account_file + + @classmethod + def get_mtls_endpoint_and_cert_source(cls, client_options: Optional[ClientOptions] = None): + """Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variable is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + return MetastorePartitionServiceClient.get_mtls_endpoint_and_cert_source(client_options) # type: ignore + + @property + def transport(self) -> MetastorePartitionServiceTransport: + """Returns the transport used by the client instance. + + Returns: + MetastorePartitionServiceTransport: The transport used by the client instance. + """ + return self._client.transport + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._client._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used + by the client instance. + """ + return self._client._universe_domain + + get_transport_class = MetastorePartitionServiceClient.get_transport_class + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Optional[Union[str, MetastorePartitionServiceTransport, Callable[..., MetastorePartitionServiceTransport]]] = "grpc_asyncio", + client_options: Optional[ClientOptions] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the metastore partition service async client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Optional[Union[str,MetastorePartitionServiceTransport,Callable[..., MetastorePartitionServiceTransport]]]): + The transport to use, or a Callable that constructs and returns a new transport to use. + If a Callable is given, it will be called with the same set of initialization + arguments as used in the MetastorePartitionServiceTransport constructor. + If set to None, a transport is chosen automatically. + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide a client certificate for mTLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client = MetastorePartitionServiceClient( + credentials=credentials, + transport=transport, + client_options=client_options, + client_info=client_info, + + ) + + async def batch_create_metastore_partitions(self, + request: Optional[Union[metastore_partition.BatchCreateMetastorePartitionsRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> metastore_partition.BatchCreateMetastorePartitionsResponse: + r"""Adds metastore partitions to a table. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1alpha + + async def sample_batch_create_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient() + + # Initialize request argument(s) + requests = bigquery_storage_v1alpha.CreateMetastorePartitionRequest() + requests.parent = "parent_value" + requests.metastore_partition.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchCreateMetastorePartitionsRequest( + parent="parent_value", + requests=requests, + ) + + # Make the request + response = await client.batch_create_metastore_partitions(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1alpha.types.BatchCreateMetastorePartitionsRequest, dict]]): + The request object. Request message for + BatchCreateMetastorePartitions. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1alpha.types.BatchCreateMetastorePartitionsResponse: + Response message for + BatchCreateMetastorePartitions. + + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, metastore_partition.BatchCreateMetastorePartitionsRequest): + request = metastore_partition.BatchCreateMetastorePartitionsRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.batch_create_metastore_partitions] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def batch_delete_metastore_partitions(self, + request: Optional[Union[metastore_partition.BatchDeleteMetastorePartitionsRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> None: + r"""Deletes metastore partitions from a table. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1alpha + + async def sample_batch_delete_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient() + + # Initialize request argument(s) + partition_values = bigquery_storage_v1alpha.MetastorePartitionValues() + partition_values.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchDeleteMetastorePartitionsRequest( + parent="parent_value", + partition_values=partition_values, + ) + + # Make the request + await client.batch_delete_metastore_partitions(request=request) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1alpha.types.BatchDeleteMetastorePartitionsRequest, dict]]): + The request object. Request message for + BatchDeleteMetastorePartitions. The + MetastorePartition is uniquely + identified by values, which is an + ordered list. Hence, there is no + separate name or partition id field. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, metastore_partition.BatchDeleteMetastorePartitionsRequest): + request = metastore_partition.BatchDeleteMetastorePartitionsRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.batch_delete_metastore_partitions] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + async def batch_update_metastore_partitions(self, + request: Optional[Union[metastore_partition.BatchUpdateMetastorePartitionsRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> metastore_partition.BatchUpdateMetastorePartitionsResponse: + r"""Updates metastore partitions in a table. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1alpha + + async def sample_batch_update_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient() + + # Initialize request argument(s) + requests = bigquery_storage_v1alpha.UpdateMetastorePartitionRequest() + requests.metastore_partition.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchUpdateMetastorePartitionsRequest( + parent="parent_value", + requests=requests, + ) + + # Make the request + response = await client.batch_update_metastore_partitions(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1alpha.types.BatchUpdateMetastorePartitionsRequest, dict]]): + The request object. Request message for + BatchUpdateMetastorePartitions. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1alpha.types.BatchUpdateMetastorePartitionsResponse: + Response message for + BatchUpdateMetastorePartitions. + + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, metastore_partition.BatchUpdateMetastorePartitionsRequest): + request = metastore_partition.BatchUpdateMetastorePartitionsRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.batch_update_metastore_partitions] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def list_metastore_partitions(self, + request: Optional[Union[metastore_partition.ListMetastorePartitionsRequest, dict]] = None, + *, + parent: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> metastore_partition.ListMetastorePartitionsResponse: + r"""Gets metastore partitions from a table. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1alpha + + async def sample_list_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1alpha.ListMetastorePartitionsRequest( + parent="parent_value", + ) + + # Make the request + response = await client.list_metastore_partitions(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1alpha.types.ListMetastorePartitionsRequest, dict]]): + The request object. Request message for + ListMetastorePartitions. + parent (:class:`str`): + Required. Reference to the table to + which these metastore partitions belong, + in the format of + projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1alpha.types.ListMetastorePartitionsResponse: + Response message for + ListMetastorePartitions. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, metastore_partition.ListMetastorePartitionsRequest): + request = metastore_partition.ListMetastorePartitionsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.list_metastore_partitions] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def stream_metastore_partitions(self, + requests: Optional[AsyncIterator[metastore_partition.StreamMetastorePartitionsRequest]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Awaitable[AsyncIterable[metastore_partition.StreamMetastorePartitionsResponse]]: + r"""This is a bi-di streaming rpc method that allows the + client to send a stream of partitions and commit all of + them atomically at the end. If the commit is successful, + the server will return a response and close the stream. + If the commit fails (due to duplicate partitions or + other reason), the server will close the stream with an + error. This method is only available via the gRPC API + (not REST). + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1alpha + + async def sample_stream_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1alpha.StreamMetastorePartitionsRequest( + parent="parent_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1alpha.StreamMetastorePartitionsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = await client.stream_metastore_partitions(requests=request_generator()) + + # Handle the response + async for response in stream: + print(response) + + Args: + requests (AsyncIterator[`google.cloud.bigquery_storage_v1alpha.types.StreamMetastorePartitionsRequest`]): + The request object AsyncIterator. The top-level message sent by the client to the + [Partitions.StreamMetastorePartitions][] method. Follows + the default gRPC streaming maximum size of 4 MB. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + AsyncIterable[google.cloud.bigquery_storage_v1alpha.types.StreamMetastorePartitionsResponse]: + This is the response message sent by the server + to the client for the + [Partitions.StreamMetastorePartitions][] method when + the commit is successful. Server will close the + stream after sending this message. + + """ + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.stream_metastore_partitions] + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = rpc( + requests, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def __aenter__(self) -> "MetastorePartitionServiceAsyncClient": + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.transport.close() + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +__all__ = ( + "MetastorePartitionServiceAsyncClient", +) diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/client.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/client.py new file mode 100644 index 00000000..75056a0a --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/client.py @@ -0,0 +1,1058 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import os +import re +from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, Iterable, Iterator, Sequence, Tuple, Type, Union, cast +import warnings + +from google.cloud.bigquery_storage_v1alpha import gapic_version as package_version + +from google.api_core import client_options as client_options_lib +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport import mtls # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.oauth2 import service_account # type: ignore + +try: + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.Retry, object, None] # type: ignore + +from google.cloud.bigquery_storage_v1alpha.types import metastore_partition +from google.cloud.bigquery_storage_v1alpha.types import partition +from .transports.base import MetastorePartitionServiceTransport, DEFAULT_CLIENT_INFO +from .transports.grpc import MetastorePartitionServiceGrpcTransport +from .transports.grpc_asyncio import MetastorePartitionServiceGrpcAsyncIOTransport + + +class MetastorePartitionServiceClientMeta(type): + """Metaclass for the MetastorePartitionService client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + _transport_registry = OrderedDict() # type: Dict[str, Type[MetastorePartitionServiceTransport]] + _transport_registry["grpc"] = MetastorePartitionServiceGrpcTransport + _transport_registry["grpc_asyncio"] = MetastorePartitionServiceGrpcAsyncIOTransport + + def get_transport_class(cls, + label: Optional[str] = None, + ) -> Type[MetastorePartitionServiceTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class MetastorePartitionServiceClient(metaclass=MetastorePartitionServiceClientMeta): + """BigQuery Metastore Partition Service API. + This service is used for managing metastore partitions in + BigQuery metastore. The service supports only batch operations + for write. + """ + + @staticmethod + def _get_default_mtls_endpoint(api_endpoint): + """Converts api endpoint to mTLS endpoint. + + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to + "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. + Args: + api_endpoint (Optional[str]): the api endpoint to convert. + Returns: + str: converted mTLS api endpoint. + """ + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" + DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_ENDPOINT + ) + + _DEFAULT_ENDPOINT_TEMPLATE = "bigquerystorage.{UNIVERSE_DOMAIN}" + _DEFAULT_UNIVERSE = "googleapis.com" + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + MetastorePartitionServiceClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + MetastorePartitionServiceClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file( + filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> MetastorePartitionServiceTransport: + """Returns the transport used by the client instance. + + Returns: + MetastorePartitionServiceTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def read_stream_path(project: str,location: str,session: str,stream: str,) -> str: + """Returns a fully-qualified read_stream string.""" + return "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) + + @staticmethod + def parse_read_stream_path(path: str) -> Dict[str,str]: + """Parses a read_stream path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)/streams/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def table_path(project: str,dataset: str,table: str,) -> str: + """Returns a fully-qualified table string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + + @staticmethod + def parse_table_path(path: str) -> Dict[str,str]: + """Parses a table path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_billing_account_path(billing_account: str, ) -> str: + """Returns a fully-qualified billing_account string.""" + return "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + + @staticmethod + def parse_common_billing_account_path(path: str) -> Dict[str,str]: + """Parse a billing_account path into its component segments.""" + m = re.match(r"^billingAccounts/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_folder_path(folder: str, ) -> str: + """Returns a fully-qualified folder string.""" + return "folders/{folder}".format(folder=folder, ) + + @staticmethod + def parse_common_folder_path(path: str) -> Dict[str,str]: + """Parse a folder path into its component segments.""" + m = re.match(r"^folders/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_organization_path(organization: str, ) -> str: + """Returns a fully-qualified organization string.""" + return "organizations/{organization}".format(organization=organization, ) + + @staticmethod + def parse_common_organization_path(path: str) -> Dict[str,str]: + """Parse a organization path into its component segments.""" + m = re.match(r"^organizations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_project_path(project: str, ) -> str: + """Returns a fully-qualified project string.""" + return "projects/{project}".format(project=project, ) + + @staticmethod + def parse_common_project_path(path: str) -> Dict[str,str]: + """Parse a project path into its component segments.""" + m = re.match(r"^projects/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_location_path(project: str, location: str, ) -> str: + """Returns a fully-qualified location string.""" + return "projects/{project}/locations/{location}".format(project=project, location=location, ) + + @staticmethod + def parse_common_location_path(path: str) -> Dict[str,str]: + """Parse a location path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @classmethod + def get_mtls_endpoint_and_cert_source(cls, client_options: Optional[client_options_lib.ClientOptions] = None): + """Deprecated. Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variable is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + + warnings.warn("get_mtls_endpoint_and_cert_source is deprecated. Use the api_endpoint property instead.", + DeprecationWarning) + if client_options is None: + client_options = client_options_lib.ClientOptions() + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_client_cert not in ("true", "false"): + raise ValueError("Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`") + + # Figure out the client cert source to use. + client_cert_source = None + if use_client_cert == "true": + if client_options.client_cert_source: + client_cert_source = client_options.client_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + api_endpoint = cls.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = cls.DEFAULT_ENDPOINT + + return api_endpoint, client_cert_source + + @staticmethod + def _read_environment_variables(): + """Returns the environment variables used by the client. + + Returns: + Tuple[bool, str, str]: returns the GOOGLE_API_USE_CLIENT_CERTIFICATE, + GOOGLE_API_USE_MTLS_ENDPOINT, and GOOGLE_CLOUD_UNIVERSE_DOMAIN environment variables. + + Raises: + ValueError: If GOOGLE_API_USE_CLIENT_CERTIFICATE is not + any of ["true", "false"]. + google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT + is not any of ["auto", "never", "always"]. + """ + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false").lower() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_client_cert not in ("true", "false"): + raise ValueError("Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`") + return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + + @staticmethod + def _get_client_cert_source(provided_cert_source, use_cert_flag): + """Return the client cert source to be used by the client. + + Args: + provided_cert_source (bytes): The client certificate source provided. + use_cert_flag (bool): A flag indicating whether to use the client certificate. + + Returns: + bytes or None: The client cert source to be used by the client. + """ + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + return client_cert_source + + @staticmethod + def _get_api_endpoint(api_override, client_cert_source, universe_domain, use_mtls_endpoint): + """Return the API endpoint used by the client. + + Args: + api_override (str): The API endpoint override. If specified, this is always + the return value of this function and the other arguments are not used. + client_cert_source (bytes): The client certificate source used by the client. + universe_domain (str): The universe domain used by the client. + use_mtls_endpoint (str): How to use the mTLS endpoint, which depends also on the other parameters. + Possible values are "always", "auto", or "never". + + Returns: + str: The API endpoint to be used by the client. + """ + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + _default_universe = MetastorePartitionServiceClient._DEFAULT_UNIVERSE + if universe_domain != _default_universe: + raise MutualTLSChannelError(f"mTLS is not supported in any universe other than {_default_universe}.") + api_endpoint = MetastorePartitionServiceClient.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = MetastorePartitionServiceClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=universe_domain) + return api_endpoint + + @staticmethod + def _get_universe_domain(client_universe_domain: Optional[str], universe_domain_env: Optional[str]) -> str: + """Return the universe domain used by the client. + + Args: + client_universe_domain (Optional[str]): The universe domain configured via the client options. + universe_domain_env (Optional[str]): The universe domain configured via the "GOOGLE_CLOUD_UNIVERSE_DOMAIN" environment variable. + + Returns: + str: The universe domain to be used by the client. + + Raises: + ValueError: If the universe domain is an empty string. + """ + universe_domain = MetastorePartitionServiceClient._DEFAULT_UNIVERSE + if client_universe_domain is not None: + universe_domain = client_universe_domain + elif universe_domain_env is not None: + universe_domain = universe_domain_env + if len(universe_domain.strip()) == 0: + raise ValueError("Universe Domain cannot be an empty string.") + return universe_domain + + @staticmethod + def _compare_universes(client_universe: str, + credentials: ga_credentials.Credentials) -> bool: + """Returns True iff the universe domains used by the client and credentials match. + + Args: + client_universe (str): The universe domain configured via the client options. + credentials (ga_credentials.Credentials): The credentials being used in the client. + + Returns: + bool: True iff client_universe matches the universe in credentials. + + Raises: + ValueError: when client_universe does not match the universe in credentials. + """ + + default_universe = MetastorePartitionServiceClient._DEFAULT_UNIVERSE + credentials_universe = getattr(credentials, "universe_domain", default_universe) + + if client_universe != credentials_universe: + raise ValueError("The configured universe domain " + f"({client_universe}) does not match the universe domain " + f"found in the credentials ({credentials_universe}). " + "If you haven't configured the universe domain explicitly, " + f"`{default_universe}` is the default.") + return True + + def _validate_universe_domain(self): + """Validates client's and credentials' universe domains are consistent. + + Returns: + bool: True iff the configured universe domain is valid. + + Raises: + ValueError: If the configured universe domain is not valid. + """ + self._is_universe_domain_valid = (self._is_universe_domain_valid or + MetastorePartitionServiceClient._compare_universes(self.universe_domain, self.transport._credentials)) + return self._is_universe_domain_valid + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used by the client instance. + """ + return self._universe_domain + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Optional[Union[str, MetastorePartitionServiceTransport, Callable[..., MetastorePartitionServiceTransport]]] = None, + client_options: Optional[Union[client_options_lib.ClientOptions, dict]] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the metastore partition service client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Optional[Union[str,MetastorePartitionServiceTransport,Callable[..., MetastorePartitionServiceTransport]]]): + The transport to use, or a Callable that constructs and returns a new transport. + If a Callable is given, it will be called with the same set of initialization + arguments as used in the MetastorePartitionServiceTransport constructor. + If set to None, a transport is chosen automatically. + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide a client certificate for mTLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that the ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client_options = client_options + if isinstance(self._client_options, dict): + self._client_options = client_options_lib.from_dict(self._client_options) + if self._client_options is None: + self._client_options = client_options_lib.ClientOptions() + self._client_options = cast(client_options_lib.ClientOptions, self._client_options) + + universe_domain_opt = getattr(self._client_options, 'universe_domain', None) + + self._use_client_cert, self._use_mtls_endpoint, self._universe_domain_env = MetastorePartitionServiceClient._read_environment_variables() + self._client_cert_source = MetastorePartitionServiceClient._get_client_cert_source(self._client_options.client_cert_source, self._use_client_cert) + self._universe_domain = MetastorePartitionServiceClient._get_universe_domain(universe_domain_opt, self._universe_domain_env) + self._api_endpoint = None # updated below, depending on `transport` + + # Initialize the universe domain validation. + self._is_universe_domain_valid = False + + api_key_value = getattr(self._client_options, "api_key", None) + if api_key_value and credentials: + raise ValueError("client_options.api_key and credentials are mutually exclusive") + + # Save or instantiate the transport. + # Ordinarily, we provide the transport, but allowing a custom transport + # instance provides an extensibility point for unusual situations. + transport_provided = isinstance(transport, MetastorePartitionServiceTransport) + if transport_provided: + # transport is a MetastorePartitionServiceTransport instance. + if credentials or self._client_options.credentials_file or api_key_value: + raise ValueError("When providing a transport instance, " + "provide its credentials directly.") + if self._client_options.scopes: + raise ValueError( + "When providing a transport instance, provide its scopes " + "directly." + ) + self._transport = cast(MetastorePartitionServiceTransport, transport) + self._api_endpoint = self._transport.host + + self._api_endpoint = (self._api_endpoint or + MetastorePartitionServiceClient._get_api_endpoint( + self._client_options.api_endpoint, + self._client_cert_source, + self._universe_domain, + self._use_mtls_endpoint)) + + if not transport_provided: + import google.auth._default # type: ignore + + if api_key_value and hasattr(google.auth._default, "get_api_key_credentials"): + credentials = google.auth._default.get_api_key_credentials(api_key_value) + + transport_init: Union[Type[MetastorePartitionServiceTransport], Callable[..., MetastorePartitionServiceTransport]] = ( + MetastorePartitionServiceClient.get_transport_class(transport) + if isinstance(transport, str) or transport is None + else cast(Callable[..., MetastorePartitionServiceTransport], transport) + ) + # initialize with the provided callable or the passed in class + self._transport = transport_init( + credentials=credentials, + credentials_file=self._client_options.credentials_file, + host=self._api_endpoint, + scopes=self._client_options.scopes, + client_cert_source_for_mtls=self._client_cert_source, + quota_project_id=self._client_options.quota_project_id, + client_info=client_info, + always_use_jwt_access=True, + api_audience=self._client_options.api_audience, + ) + + def batch_create_metastore_partitions(self, + request: Optional[Union[metastore_partition.BatchCreateMetastorePartitionsRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> metastore_partition.BatchCreateMetastorePartitionsResponse: + r"""Adds metastore partitions to a table. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1alpha + + def sample_batch_create_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceClient() + + # Initialize request argument(s) + requests = bigquery_storage_v1alpha.CreateMetastorePartitionRequest() + requests.parent = "parent_value" + requests.metastore_partition.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchCreateMetastorePartitionsRequest( + parent="parent_value", + requests=requests, + ) + + # Make the request + response = client.batch_create_metastore_partitions(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1alpha.types.BatchCreateMetastorePartitionsRequest, dict]): + The request object. Request message for + BatchCreateMetastorePartitions. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1alpha.types.BatchCreateMetastorePartitionsResponse: + Response message for + BatchCreateMetastorePartitions. + + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, metastore_partition.BatchCreateMetastorePartitionsRequest): + request = metastore_partition.BatchCreateMetastorePartitionsRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.batch_create_metastore_partitions] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def batch_delete_metastore_partitions(self, + request: Optional[Union[metastore_partition.BatchDeleteMetastorePartitionsRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> None: + r"""Deletes metastore partitions from a table. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1alpha + + def sample_batch_delete_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceClient() + + # Initialize request argument(s) + partition_values = bigquery_storage_v1alpha.MetastorePartitionValues() + partition_values.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchDeleteMetastorePartitionsRequest( + parent="parent_value", + partition_values=partition_values, + ) + + # Make the request + client.batch_delete_metastore_partitions(request=request) + + Args: + request (Union[google.cloud.bigquery_storage_v1alpha.types.BatchDeleteMetastorePartitionsRequest, dict]): + The request object. Request message for + BatchDeleteMetastorePartitions. The + MetastorePartition is uniquely + identified by values, which is an + ordered list. Hence, there is no + separate name or partition id field. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, metastore_partition.BatchDeleteMetastorePartitionsRequest): + request = metastore_partition.BatchDeleteMetastorePartitionsRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.batch_delete_metastore_partitions] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + def batch_update_metastore_partitions(self, + request: Optional[Union[metastore_partition.BatchUpdateMetastorePartitionsRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> metastore_partition.BatchUpdateMetastorePartitionsResponse: + r"""Updates metastore partitions in a table. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1alpha + + def sample_batch_update_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceClient() + + # Initialize request argument(s) + requests = bigquery_storage_v1alpha.UpdateMetastorePartitionRequest() + requests.metastore_partition.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchUpdateMetastorePartitionsRequest( + parent="parent_value", + requests=requests, + ) + + # Make the request + response = client.batch_update_metastore_partitions(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1alpha.types.BatchUpdateMetastorePartitionsRequest, dict]): + The request object. Request message for + BatchUpdateMetastorePartitions. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1alpha.types.BatchUpdateMetastorePartitionsResponse: + Response message for + BatchUpdateMetastorePartitions. + + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, metastore_partition.BatchUpdateMetastorePartitionsRequest): + request = metastore_partition.BatchUpdateMetastorePartitionsRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.batch_update_metastore_partitions] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def list_metastore_partitions(self, + request: Optional[Union[metastore_partition.ListMetastorePartitionsRequest, dict]] = None, + *, + parent: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> metastore_partition.ListMetastorePartitionsResponse: + r"""Gets metastore partitions from a table. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1alpha + + def sample_list_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceClient() + + # Initialize request argument(s) + request = bigquery_storage_v1alpha.ListMetastorePartitionsRequest( + parent="parent_value", + ) + + # Make the request + response = client.list_metastore_partitions(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1alpha.types.ListMetastorePartitionsRequest, dict]): + The request object. Request message for + ListMetastorePartitions. + parent (str): + Required. Reference to the table to + which these metastore partitions belong, + in the format of + projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1alpha.types.ListMetastorePartitionsResponse: + Response message for + ListMetastorePartitions. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, metastore_partition.ListMetastorePartitionsRequest): + request = metastore_partition.ListMetastorePartitionsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.list_metastore_partitions] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def stream_metastore_partitions(self, + requests: Optional[Iterator[metastore_partition.StreamMetastorePartitionsRequest]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Iterable[metastore_partition.StreamMetastorePartitionsResponse]: + r"""This is a bi-di streaming rpc method that allows the + client to send a stream of partitions and commit all of + them atomically at the end. If the commit is successful, + the server will return a response and close the stream. + If the commit fails (due to duplicate partitions or + other reason), the server will close the stream with an + error. This method is only available via the gRPC API + (not REST). + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1alpha + + def sample_stream_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceClient() + + # Initialize request argument(s) + request = bigquery_storage_v1alpha.StreamMetastorePartitionsRequest( + parent="parent_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1alpha.StreamMetastorePartitionsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = client.stream_metastore_partitions(requests=request_generator()) + + # Handle the response + for response in stream: + print(response) + + Args: + requests (Iterator[google.cloud.bigquery_storage_v1alpha.types.StreamMetastorePartitionsRequest]): + The request object iterator. The top-level message sent by the client to the + [Partitions.StreamMetastorePartitions][] method. Follows + the default gRPC streaming maximum size of 4 MB. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + Iterable[google.cloud.bigquery_storage_v1alpha.types.StreamMetastorePartitionsResponse]: + This is the response message sent by the server + to the client for the + [Partitions.StreamMetastorePartitions][] method when + the commit is successful. Server will close the + stream after sending this message. + + """ + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.stream_metastore_partitions] + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + requests, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def __enter__(self) -> "MetastorePartitionServiceClient": + return self + + def __exit__(self, type, value, traceback): + """Releases underlying transport's resources. + + .. warning:: + ONLY use as a context manager if the transport is NOT shared + with other clients! Exiting the with block will CLOSE the transport + and may cause errors in other clients! + """ + self.transport.close() + + + + + + + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +__all__ = ( + "MetastorePartitionServiceClient", +) diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/README.rst b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/README.rst new file mode 100644 index 00000000..ff0f177a --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/README.rst @@ -0,0 +1,9 @@ + +transport inheritance structure +_______________________________ + +`MetastorePartitionServiceTransport` is the ABC for all transports. +- public child `MetastorePartitionServiceGrpcTransport` for sync gRPC transport (defined in `grpc.py`). +- public child `MetastorePartitionServiceGrpcAsyncIOTransport` for async gRPC transport (defined in `grpc_asyncio.py`). +- private child `_BaseMetastorePartitionServiceRestTransport` for base REST transport with inner classes `_BaseMETHOD` (defined in `rest_base.py`). +- public child `MetastorePartitionServiceRestTransport` for sync REST transport with inner classes `METHOD` derived from the parent's corresponding `_BaseMETHOD` classes (defined in `rest.py`). diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/__init__.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/__init__.py new file mode 100644 index 00000000..5a8e97bc --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from typing import Dict, Type + +from .base import MetastorePartitionServiceTransport +from .grpc import MetastorePartitionServiceGrpcTransport +from .grpc_asyncio import MetastorePartitionServiceGrpcAsyncIOTransport + + +# Compile a registry of transports. +_transport_registry = OrderedDict() # type: Dict[str, Type[MetastorePartitionServiceTransport]] +_transport_registry['grpc'] = MetastorePartitionServiceGrpcTransport +_transport_registry['grpc_asyncio'] = MetastorePartitionServiceGrpcAsyncIOTransport + +__all__ = ( + 'MetastorePartitionServiceTransport', + 'MetastorePartitionServiceGrpcTransport', + 'MetastorePartitionServiceGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/base.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/base.py new file mode 100644 index 00000000..d006cd89 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/base.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union + +from google.cloud.bigquery_storage_v1alpha import gapic_version as package_version + +import google.auth # type: ignore +import google.api_core +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1alpha.types import metastore_partition +from google.protobuf import empty_pb2 # type: ignore + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +class MetastorePartitionServiceTransport(abc.ABC): + """Abstract transport class for MetastorePartitionService.""" + + AUTH_SCOPES = ( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', + ) + + DEFAULT_HOST: str = 'bigquerystorage.googleapis.com' + def __init__( + self, *, + host: str = DEFAULT_HOST, + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + **kwargs, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A list of scopes. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + """ + + scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES} + + # Save the scopes. + self._scopes = scopes + if not hasattr(self, "_ignore_credentials"): + self._ignore_credentials: bool = False + + # If no credentials are provided, then determine the appropriate + # defaults. + if credentials and credentials_file: + raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive") + + if credentials_file is not None: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, + **scopes_kwargs, + quota_project_id=quota_project_id + ) + elif credentials is None and not self._ignore_credentials: + credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id) + # Don't apply audience if the credentials file passed from user. + if hasattr(credentials, "with_gdch_audience"): + credentials = credentials.with_gdch_audience(api_audience if api_audience else host) + + # If the credentials are service account credentials, then always try to use self signed JWT. + if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"): + credentials = credentials.with_always_use_jwt_access(True) + + # Save the credentials. + self._credentials = credentials + + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ':' not in host: + host += ':443' + self._host = host + + @property + def host(self): + return self._host + + def _prep_wrapped_messages(self, client_info): + # Precompute the wrapped methods. + self._wrapped_methods = { + self.batch_create_metastore_partitions: gapic_v1.method.wrap_method( + self.batch_create_metastore_partitions, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=240.0, + ), + default_timeout=240.0, + client_info=client_info, + ), + self.batch_delete_metastore_partitions: gapic_v1.method.wrap_method( + self.batch_delete_metastore_partitions, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=240.0, + ), + default_timeout=240.0, + client_info=client_info, + ), + self.batch_update_metastore_partitions: gapic_v1.method.wrap_method( + self.batch_update_metastore_partitions, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=240.0, + ), + default_timeout=240.0, + client_info=client_info, + ), + self.list_metastore_partitions: gapic_v1.method.wrap_method( + self.list_metastore_partitions, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=240.0, + ), + default_timeout=240.0, + client_info=client_info, + ), + self.stream_metastore_partitions: gapic_v1.method.wrap_method( + self.stream_metastore_partitions, + default_timeout=240.0, + client_info=client_info, + ), + } + + def close(self): + """Closes resources associated with the transport. + + .. warning:: + Only call this method if the transport is NOT shared + with other clients - this may cause errors in other clients! + """ + raise NotImplementedError() + + @property + def batch_create_metastore_partitions(self) -> Callable[ + [metastore_partition.BatchCreateMetastorePartitionsRequest], + Union[ + metastore_partition.BatchCreateMetastorePartitionsResponse, + Awaitable[metastore_partition.BatchCreateMetastorePartitionsResponse] + ]]: + raise NotImplementedError() + + @property + def batch_delete_metastore_partitions(self) -> Callable[ + [metastore_partition.BatchDeleteMetastorePartitionsRequest], + Union[ + empty_pb2.Empty, + Awaitable[empty_pb2.Empty] + ]]: + raise NotImplementedError() + + @property + def batch_update_metastore_partitions(self) -> Callable[ + [metastore_partition.BatchUpdateMetastorePartitionsRequest], + Union[ + metastore_partition.BatchUpdateMetastorePartitionsResponse, + Awaitable[metastore_partition.BatchUpdateMetastorePartitionsResponse] + ]]: + raise NotImplementedError() + + @property + def list_metastore_partitions(self) -> Callable[ + [metastore_partition.ListMetastorePartitionsRequest], + Union[ + metastore_partition.ListMetastorePartitionsResponse, + Awaitable[metastore_partition.ListMetastorePartitionsResponse] + ]]: + raise NotImplementedError() + + @property + def stream_metastore_partitions(self) -> Callable[ + [metastore_partition.StreamMetastorePartitionsRequest], + Union[ + metastore_partition.StreamMetastorePartitionsResponse, + Awaitable[metastore_partition.StreamMetastorePartitionsResponse] + ]]: + raise NotImplementedError() + + @property + def kind(self) -> str: + raise NotImplementedError() + + +__all__ = ( + 'MetastorePartitionServiceTransport', +) diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/grpc.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/grpc.py new file mode 100644 index 00000000..089e3e53 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/grpc.py @@ -0,0 +1,388 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import grpc_helpers +from google.api_core import gapic_v1 +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore + +from google.cloud.bigquery_storage_v1alpha.types import metastore_partition +from google.protobuf import empty_pb2 # type: ignore +from .base import MetastorePartitionServiceTransport, DEFAULT_CLIENT_INFO + + +class MetastorePartitionServiceGrpcTransport(MetastorePartitionServiceTransport): + """gRPC backend transport for MetastorePartitionService. + + BigQuery Metastore Partition Service API. + This service is used for managing metastore partitions in + BigQuery metastore. The service supports only batch operations + for write. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + _stubs: Dict[str, Callable] + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]] = None, + api_mtls_endpoint: Optional[str] = None, + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + ssl_channel_credentials: Optional[grpc.ChannelCredentials] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if a ``channel`` instance is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if a ``channel`` instance is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if a ``channel`` instance is provided. + channel (Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]]): + A ``Channel`` instance through which to make calls, or a Callable + that constructs and returns one. If set to None, ``self.create_channel`` + is used to create the channel. If a Callable is given, it will be called + with the same arguments as used in ``self.create_channel``. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or application default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for the grpc channel. It is ignored if a ``channel`` instance is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if a ``channel`` instance or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if isinstance(channel, grpc.Channel): + # Ignore credentials if a channel was passed. + credentials = None + self._ignore_credentials = True + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, + ) + + if not self._grpc_channel: + # initialize with the provided callable or the default channel + channel_init = channel or type(self).create_channel + self._grpc_channel = channel_init( + self._host, + # use the credentials which are saved + credentials=self._credentials, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> grpc.Channel: + """Create and return a gRPC channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + grpc.Channel: A gRPC channel object. + + Raises: + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + + return grpc_helpers.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + @property + def grpc_channel(self) -> grpc.Channel: + """Return the channel designed to connect to this service. + """ + return self._grpc_channel + + @property + def batch_create_metastore_partitions(self) -> Callable[ + [metastore_partition.BatchCreateMetastorePartitionsRequest], + metastore_partition.BatchCreateMetastorePartitionsResponse]: + r"""Return a callable for the batch create metastore + partitions method over gRPC. + + Adds metastore partitions to a table. + + Returns: + Callable[[~.BatchCreateMetastorePartitionsRequest], + ~.BatchCreateMetastorePartitionsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_create_metastore_partitions' not in self._stubs: + self._stubs['batch_create_metastore_partitions'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1alpha.MetastorePartitionService/BatchCreateMetastorePartitions', + request_serializer=metastore_partition.BatchCreateMetastorePartitionsRequest.serialize, + response_deserializer=metastore_partition.BatchCreateMetastorePartitionsResponse.deserialize, + ) + return self._stubs['batch_create_metastore_partitions'] + + @property + def batch_delete_metastore_partitions(self) -> Callable[ + [metastore_partition.BatchDeleteMetastorePartitionsRequest], + empty_pb2.Empty]: + r"""Return a callable for the batch delete metastore + partitions method over gRPC. + + Deletes metastore partitions from a table. + + Returns: + Callable[[~.BatchDeleteMetastorePartitionsRequest], + ~.Empty]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_delete_metastore_partitions' not in self._stubs: + self._stubs['batch_delete_metastore_partitions'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1alpha.MetastorePartitionService/BatchDeleteMetastorePartitions', + request_serializer=metastore_partition.BatchDeleteMetastorePartitionsRequest.serialize, + response_deserializer=empty_pb2.Empty.FromString, + ) + return self._stubs['batch_delete_metastore_partitions'] + + @property + def batch_update_metastore_partitions(self) -> Callable[ + [metastore_partition.BatchUpdateMetastorePartitionsRequest], + metastore_partition.BatchUpdateMetastorePartitionsResponse]: + r"""Return a callable for the batch update metastore + partitions method over gRPC. + + Updates metastore partitions in a table. + + Returns: + Callable[[~.BatchUpdateMetastorePartitionsRequest], + ~.BatchUpdateMetastorePartitionsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_update_metastore_partitions' not in self._stubs: + self._stubs['batch_update_metastore_partitions'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1alpha.MetastorePartitionService/BatchUpdateMetastorePartitions', + request_serializer=metastore_partition.BatchUpdateMetastorePartitionsRequest.serialize, + response_deserializer=metastore_partition.BatchUpdateMetastorePartitionsResponse.deserialize, + ) + return self._stubs['batch_update_metastore_partitions'] + + @property + def list_metastore_partitions(self) -> Callable[ + [metastore_partition.ListMetastorePartitionsRequest], + metastore_partition.ListMetastorePartitionsResponse]: + r"""Return a callable for the list metastore partitions method over gRPC. + + Gets metastore partitions from a table. + + Returns: + Callable[[~.ListMetastorePartitionsRequest], + ~.ListMetastorePartitionsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'list_metastore_partitions' not in self._stubs: + self._stubs['list_metastore_partitions'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1alpha.MetastorePartitionService/ListMetastorePartitions', + request_serializer=metastore_partition.ListMetastorePartitionsRequest.serialize, + response_deserializer=metastore_partition.ListMetastorePartitionsResponse.deserialize, + ) + return self._stubs['list_metastore_partitions'] + + @property + def stream_metastore_partitions(self) -> Callable[ + [metastore_partition.StreamMetastorePartitionsRequest], + metastore_partition.StreamMetastorePartitionsResponse]: + r"""Return a callable for the stream metastore partitions method over gRPC. + + This is a bi-di streaming rpc method that allows the + client to send a stream of partitions and commit all of + them atomically at the end. If the commit is successful, + the server will return a response and close the stream. + If the commit fails (due to duplicate partitions or + other reason), the server will close the stream with an + error. This method is only available via the gRPC API + (not REST). + + Returns: + Callable[[~.StreamMetastorePartitionsRequest], + ~.StreamMetastorePartitionsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'stream_metastore_partitions' not in self._stubs: + self._stubs['stream_metastore_partitions'] = self.grpc_channel.stream_stream( + '/google.cloud.bigquery.storage.v1alpha.MetastorePartitionService/StreamMetastorePartitions', + request_serializer=metastore_partition.StreamMetastorePartitionsRequest.serialize, + response_deserializer=metastore_partition.StreamMetastorePartitionsResponse.deserialize, + ) + return self._stubs['stream_metastore_partitions'] + + def close(self): + self.grpc_channel.close() + + @property + def kind(self) -> str: + return "grpc" + + +__all__ = ( + 'MetastorePartitionServiceGrpcTransport', +) diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/grpc_asyncio.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/grpc_asyncio.py new file mode 100644 index 00000000..547c8d4c --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/services/metastore_partition_service/transports/grpc_asyncio.py @@ -0,0 +1,465 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import inspect +import warnings +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers_async +from google.api_core import exceptions as core_exceptions +from google.api_core import retry_async as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore +from grpc.experimental import aio # type: ignore + +from google.cloud.bigquery_storage_v1alpha.types import metastore_partition +from google.protobuf import empty_pb2 # type: ignore +from .base import MetastorePartitionServiceTransport, DEFAULT_CLIENT_INFO +from .grpc import MetastorePartitionServiceGrpcTransport + + +class MetastorePartitionServiceGrpcAsyncIOTransport(MetastorePartitionServiceTransport): + """gRPC AsyncIO backend transport for MetastorePartitionService. + + BigQuery Metastore Partition Service API. + This service is used for managing metastore partitions in + BigQuery metastore. The service supports only batch operations + for write. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + + _grpc_channel: aio.Channel + _stubs: Dict[str, Callable] = {} + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> aio.Channel: + """Create and return a gRPC AsyncIO channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + aio.Channel: A gRPC AsyncIO channel object. + """ + + return grpc_helpers_async.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: Optional[Union[aio.Channel, Callable[..., aio.Channel]]] = None, + api_mtls_endpoint: Optional[str] = None, + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + ssl_channel_credentials: Optional[grpc.ChannelCredentials] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if a ``channel`` instance is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if a ``channel`` instance is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + channel (Optional[Union[aio.Channel, Callable[..., aio.Channel]]]): + A ``Channel`` instance through which to make calls, or a Callable + that constructs and returns one. If set to None, ``self.create_channel`` + is used to create the channel. If a Callable is given, it will be called + with the same arguments as used in ``self.create_channel``. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or application default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for the grpc channel. It is ignored if a ``channel`` instance is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if a ``channel`` instance or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if isinstance(channel, aio.Channel): + # Ignore credentials if a channel was passed. + credentials = None + self._ignore_credentials = True + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, + ) + + if not self._grpc_channel: + # initialize with the provided callable or the default channel + channel_init = channel or type(self).create_channel + self._grpc_channel = channel_init( + self._host, + # use the credentials which are saved + credentials=self._credentials, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._wrap_with_kind = "kind" in inspect.signature(gapic_v1.method_async.wrap_method).parameters + self._prep_wrapped_messages(client_info) + + @property + def grpc_channel(self) -> aio.Channel: + """Create the channel designed to connect to this service. + + This property caches on the instance; repeated calls return + the same channel. + """ + # Return the channel from cache. + return self._grpc_channel + + @property + def batch_create_metastore_partitions(self) -> Callable[ + [metastore_partition.BatchCreateMetastorePartitionsRequest], + Awaitable[metastore_partition.BatchCreateMetastorePartitionsResponse]]: + r"""Return a callable for the batch create metastore + partitions method over gRPC. + + Adds metastore partitions to a table. + + Returns: + Callable[[~.BatchCreateMetastorePartitionsRequest], + Awaitable[~.BatchCreateMetastorePartitionsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_create_metastore_partitions' not in self._stubs: + self._stubs['batch_create_metastore_partitions'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1alpha.MetastorePartitionService/BatchCreateMetastorePartitions', + request_serializer=metastore_partition.BatchCreateMetastorePartitionsRequest.serialize, + response_deserializer=metastore_partition.BatchCreateMetastorePartitionsResponse.deserialize, + ) + return self._stubs['batch_create_metastore_partitions'] + + @property + def batch_delete_metastore_partitions(self) -> Callable[ + [metastore_partition.BatchDeleteMetastorePartitionsRequest], + Awaitable[empty_pb2.Empty]]: + r"""Return a callable for the batch delete metastore + partitions method over gRPC. + + Deletes metastore partitions from a table. + + Returns: + Callable[[~.BatchDeleteMetastorePartitionsRequest], + Awaitable[~.Empty]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_delete_metastore_partitions' not in self._stubs: + self._stubs['batch_delete_metastore_partitions'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1alpha.MetastorePartitionService/BatchDeleteMetastorePartitions', + request_serializer=metastore_partition.BatchDeleteMetastorePartitionsRequest.serialize, + response_deserializer=empty_pb2.Empty.FromString, + ) + return self._stubs['batch_delete_metastore_partitions'] + + @property + def batch_update_metastore_partitions(self) -> Callable[ + [metastore_partition.BatchUpdateMetastorePartitionsRequest], + Awaitable[metastore_partition.BatchUpdateMetastorePartitionsResponse]]: + r"""Return a callable for the batch update metastore + partitions method over gRPC. + + Updates metastore partitions in a table. + + Returns: + Callable[[~.BatchUpdateMetastorePartitionsRequest], + Awaitable[~.BatchUpdateMetastorePartitionsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_update_metastore_partitions' not in self._stubs: + self._stubs['batch_update_metastore_partitions'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1alpha.MetastorePartitionService/BatchUpdateMetastorePartitions', + request_serializer=metastore_partition.BatchUpdateMetastorePartitionsRequest.serialize, + response_deserializer=metastore_partition.BatchUpdateMetastorePartitionsResponse.deserialize, + ) + return self._stubs['batch_update_metastore_partitions'] + + @property + def list_metastore_partitions(self) -> Callable[ + [metastore_partition.ListMetastorePartitionsRequest], + Awaitable[metastore_partition.ListMetastorePartitionsResponse]]: + r"""Return a callable for the list metastore partitions method over gRPC. + + Gets metastore partitions from a table. + + Returns: + Callable[[~.ListMetastorePartitionsRequest], + Awaitable[~.ListMetastorePartitionsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'list_metastore_partitions' not in self._stubs: + self._stubs['list_metastore_partitions'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1alpha.MetastorePartitionService/ListMetastorePartitions', + request_serializer=metastore_partition.ListMetastorePartitionsRequest.serialize, + response_deserializer=metastore_partition.ListMetastorePartitionsResponse.deserialize, + ) + return self._stubs['list_metastore_partitions'] + + @property + def stream_metastore_partitions(self) -> Callable[ + [metastore_partition.StreamMetastorePartitionsRequest], + Awaitable[metastore_partition.StreamMetastorePartitionsResponse]]: + r"""Return a callable for the stream metastore partitions method over gRPC. + + This is a bi-di streaming rpc method that allows the + client to send a stream of partitions and commit all of + them atomically at the end. If the commit is successful, + the server will return a response and close the stream. + If the commit fails (due to duplicate partitions or + other reason), the server will close the stream with an + error. This method is only available via the gRPC API + (not REST). + + Returns: + Callable[[~.StreamMetastorePartitionsRequest], + Awaitable[~.StreamMetastorePartitionsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'stream_metastore_partitions' not in self._stubs: + self._stubs['stream_metastore_partitions'] = self.grpc_channel.stream_stream( + '/google.cloud.bigquery.storage.v1alpha.MetastorePartitionService/StreamMetastorePartitions', + request_serializer=metastore_partition.StreamMetastorePartitionsRequest.serialize, + response_deserializer=metastore_partition.StreamMetastorePartitionsResponse.deserialize, + ) + return self._stubs['stream_metastore_partitions'] + + def _prep_wrapped_messages(self, client_info): + """ Precompute the wrapped methods, overriding the base class method to use async wrappers.""" + self._wrapped_methods = { + self.batch_create_metastore_partitions: self._wrap_method( + self.batch_create_metastore_partitions, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=240.0, + ), + default_timeout=240.0, + client_info=client_info, + ), + self.batch_delete_metastore_partitions: self._wrap_method( + self.batch_delete_metastore_partitions, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=240.0, + ), + default_timeout=240.0, + client_info=client_info, + ), + self.batch_update_metastore_partitions: self._wrap_method( + self.batch_update_metastore_partitions, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=240.0, + ), + default_timeout=240.0, + client_info=client_info, + ), + self.list_metastore_partitions: self._wrap_method( + self.list_metastore_partitions, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=240.0, + ), + default_timeout=240.0, + client_info=client_info, + ), + self.stream_metastore_partitions: self._wrap_method( + self.stream_metastore_partitions, + default_timeout=240.0, + client_info=client_info, + ), + } + + def _wrap_method(self, func, *args, **kwargs): + if self._wrap_with_kind: # pragma: NO COVER + kwargs["kind"] = self.kind + return gapic_v1.method_async.wrap_method(func, *args, **kwargs) + + def close(self): + return self.grpc_channel.close() + + @property + def kind(self) -> str: + return "grpc_asyncio" + + +__all__ = ( + 'MetastorePartitionServiceGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/types/__init__.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/types/__init__.py new file mode 100644 index 00000000..b49c84c9 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/types/__init__.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .metastore_partition import ( + BatchCreateMetastorePartitionsRequest, + BatchCreateMetastorePartitionsResponse, + BatchDeleteMetastorePartitionsRequest, + BatchSizeTooLargeError, + BatchUpdateMetastorePartitionsRequest, + BatchUpdateMetastorePartitionsResponse, + CreateMetastorePartitionRequest, + ListMetastorePartitionsRequest, + ListMetastorePartitionsResponse, + StreamMetastorePartitionsRequest, + StreamMetastorePartitionsResponse, + UpdateMetastorePartitionRequest, +) +from .partition import ( + FieldSchema, + MetastorePartition, + MetastorePartitionList, + MetastorePartitionValues, + ReadStream, + SerDeInfo, + StorageDescriptor, + StreamList, +) + +__all__ = ( + 'BatchCreateMetastorePartitionsRequest', + 'BatchCreateMetastorePartitionsResponse', + 'BatchDeleteMetastorePartitionsRequest', + 'BatchSizeTooLargeError', + 'BatchUpdateMetastorePartitionsRequest', + 'BatchUpdateMetastorePartitionsResponse', + 'CreateMetastorePartitionRequest', + 'ListMetastorePartitionsRequest', + 'ListMetastorePartitionsResponse', + 'StreamMetastorePartitionsRequest', + 'StreamMetastorePartitionsResponse', + 'UpdateMetastorePartitionRequest', + 'FieldSchema', + 'MetastorePartition', + 'MetastorePartitionList', + 'MetastorePartitionValues', + 'ReadStream', + 'SerDeInfo', + 'StorageDescriptor', + 'StreamList', +) diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/types/metastore_partition.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/types/metastore_partition.py new file mode 100644 index 00000000..0eaaa074 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/types/metastore_partition.py @@ -0,0 +1,378 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + +from google.cloud.bigquery_storage_v1alpha.types import partition +from google.protobuf import field_mask_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1alpha', + manifest={ + 'CreateMetastorePartitionRequest', + 'BatchCreateMetastorePartitionsRequest', + 'BatchCreateMetastorePartitionsResponse', + 'BatchDeleteMetastorePartitionsRequest', + 'UpdateMetastorePartitionRequest', + 'BatchUpdateMetastorePartitionsRequest', + 'BatchUpdateMetastorePartitionsResponse', + 'ListMetastorePartitionsRequest', + 'ListMetastorePartitionsResponse', + 'StreamMetastorePartitionsRequest', + 'StreamMetastorePartitionsResponse', + 'BatchSizeTooLargeError', + }, +) + + +class CreateMetastorePartitionRequest(proto.Message): + r"""Request message for CreateMetastorePartition. The + MetastorePartition is uniquely identified by values, which is an + ordered list. Hence, there is no separate name or partition id + field. + + Attributes: + parent (str): + Required. Reference to the table to where the + metastore partition to be added, in the format + of + projects/{project}/databases/{databases}/tables/{table}. + metastore_partition (google.cloud.bigquery_storage_v1alpha.types.MetastorePartition): + Required. The metastore partition to be + added. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + metastore_partition: partition.MetastorePartition = proto.Field( + proto.MESSAGE, + number=2, + message=partition.MetastorePartition, + ) + + +class BatchCreateMetastorePartitionsRequest(proto.Message): + r"""Request message for BatchCreateMetastorePartitions. + + Attributes: + parent (str): + Required. Reference to the table to where the + metastore partitions to be added, in the format + of + projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}. + requests (MutableSequence[google.cloud.bigquery_storage_v1alpha.types.CreateMetastorePartitionRequest]): + Required. Requests to add metastore + partitions to the table. + skip_existing_partitions (bool): + Optional. Mimics the ifNotExists flag in IMetaStoreClient + add_partitions(..). If the flag is set to false, the server + will return ALREADY_EXISTS if any partition already exists. + If the flag is set to true, the server will skip existing + partitions and insert only the non-existing partitions. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + requests: MutableSequence['CreateMetastorePartitionRequest'] = proto.RepeatedField( + proto.MESSAGE, + number=2, + message='CreateMetastorePartitionRequest', + ) + skip_existing_partitions: bool = proto.Field( + proto.BOOL, + number=3, + ) + + +class BatchCreateMetastorePartitionsResponse(proto.Message): + r"""Response message for BatchCreateMetastorePartitions. + + Attributes: + partitions (MutableSequence[google.cloud.bigquery_storage_v1alpha.types.MetastorePartition]): + The list of metastore partitions that have + been created. + """ + + partitions: MutableSequence[partition.MetastorePartition] = proto.RepeatedField( + proto.MESSAGE, + number=1, + message=partition.MetastorePartition, + ) + + +class BatchDeleteMetastorePartitionsRequest(proto.Message): + r"""Request message for BatchDeleteMetastorePartitions. The + MetastorePartition is uniquely identified by values, which is an + ordered list. Hence, there is no separate name or partition id + field. + + Attributes: + parent (str): + Required. Reference to the table to which + these metastore partitions belong, in the format + of + projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}. + partition_values (MutableSequence[google.cloud.bigquery_storage_v1alpha.types.MetastorePartitionValues]): + Required. The list of metastore partitions + (identified by its values) to be deleted. A + maximum of 100 partitions can be deleted in a + batch. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + partition_values: MutableSequence[partition.MetastorePartitionValues] = proto.RepeatedField( + proto.MESSAGE, + number=2, + message=partition.MetastorePartitionValues, + ) + + +class UpdateMetastorePartitionRequest(proto.Message): + r"""Request message for UpdateMetastorePartition. + + Attributes: + metastore_partition (google.cloud.bigquery_storage_v1alpha.types.MetastorePartition): + Required. The metastore partition to be + updated. + update_mask (google.protobuf.field_mask_pb2.FieldMask): + Optional. The list of fields to update. + """ + + metastore_partition: partition.MetastorePartition = proto.Field( + proto.MESSAGE, + number=1, + message=partition.MetastorePartition, + ) + update_mask: field_mask_pb2.FieldMask = proto.Field( + proto.MESSAGE, + number=2, + message=field_mask_pb2.FieldMask, + ) + + +class BatchUpdateMetastorePartitionsRequest(proto.Message): + r"""Request message for BatchUpdateMetastorePartitions. + + Attributes: + parent (str): + Required. Reference to the table to which + these metastore partitions belong, in the format + of + projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}. + requests (MutableSequence[google.cloud.bigquery_storage_v1alpha.types.UpdateMetastorePartitionRequest]): + Required. Requests to update metastore + partitions in the table. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + requests: MutableSequence['UpdateMetastorePartitionRequest'] = proto.RepeatedField( + proto.MESSAGE, + number=2, + message='UpdateMetastorePartitionRequest', + ) + + +class BatchUpdateMetastorePartitionsResponse(proto.Message): + r"""Response message for BatchUpdateMetastorePartitions. + + Attributes: + partitions (MutableSequence[google.cloud.bigquery_storage_v1alpha.types.MetastorePartition]): + The list of metastore partitions that have + been updated. + """ + + partitions: MutableSequence[partition.MetastorePartition] = proto.RepeatedField( + proto.MESSAGE, + number=1, + message=partition.MetastorePartition, + ) + + +class ListMetastorePartitionsRequest(proto.Message): + r"""Request message for ListMetastorePartitions. + + Attributes: + parent (str): + Required. Reference to the table to which + these metastore partitions belong, in the format + of + projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}. + filter (str): + Optional. SQL text filtering statement, similar to a WHERE + clause in a query. Only supports single-row expressions. + Aggregate functions are not supported. + + Examples: "int_field > 5" "date_field = CAST('2014-9-27' as + DATE)" "nullable_field is not NULL" "st_equals(geo_field, + st_geofromtext("POINT(2, 2)"))" "numeric_field BETWEEN 1.0 + AND 5.0" Restricted to a maximum length for 1 MB. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + filter: str = proto.Field( + proto.STRING, + number=2, + ) + + +class ListMetastorePartitionsResponse(proto.Message): + r"""Response message for ListMetastorePartitions. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + partitions (google.cloud.bigquery_storage_v1alpha.types.MetastorePartitionList): + The list of partitions. + + This field is a member of `oneof`_ ``response``. + streams (google.cloud.bigquery_storage_v1alpha.types.StreamList): + The list of streams. + + This field is a member of `oneof`_ ``response``. + """ + + partitions: partition.MetastorePartitionList = proto.Field( + proto.MESSAGE, + number=1, + oneof='response', + message=partition.MetastorePartitionList, + ) + streams: partition.StreamList = proto.Field( + proto.MESSAGE, + number=2, + oneof='response', + message=partition.StreamList, + ) + + +class StreamMetastorePartitionsRequest(proto.Message): + r"""The top-level message sent by the client to the + [Partitions.StreamMetastorePartitions][] method. Follows the default + gRPC streaming maximum size of 4 MB. + + Attributes: + parent (str): + Required. Reference to the table to where the + partition to be added, in the format of + projects/{project}/locations/{location}/datasets/{dataset}/tables/{table}. + metastore_partitions (MutableSequence[google.cloud.bigquery_storage_v1alpha.types.MetastorePartition]): + Optional. A list of metastore partitions to + be added to the table. + skip_existing_partitions (bool): + Optional. Mimics the ifNotExists flag in IMetaStoreClient + add_partitions(..). If the flag is set to false, the server + will return ALREADY_EXISTS on commit if any partition + already exists. If the flag is set to true: + + 1) the server will skip existing partitions insert only the + non-existing partitions as part of the commit. + 2) The client must set the ``skip_existing_partitions`` + field to true for all requests in the stream. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + metastore_partitions: MutableSequence[partition.MetastorePartition] = proto.RepeatedField( + proto.MESSAGE, + number=2, + message=partition.MetastorePartition, + ) + skip_existing_partitions: bool = proto.Field( + proto.BOOL, + number=3, + ) + + +class StreamMetastorePartitionsResponse(proto.Message): + r"""This is the response message sent by the server to the client for + the [Partitions.StreamMetastorePartitions][] method when the commit + is successful. Server will close the stream after sending this + message. + + Attributes: + total_partitions_streamed_count (int): + Total count of partitions streamed by the + client during the lifetime of the stream. This + is only set in the final response message before + closing the stream. + total_partitions_inserted_count (int): + Total count of partitions inserted by the + server during the lifetime of the stream. This + is only set in the final response message before + closing the stream. + """ + + total_partitions_streamed_count: int = proto.Field( + proto.INT64, + number=2, + ) + total_partitions_inserted_count: int = proto.Field( + proto.INT64, + number=3, + ) + + +class BatchSizeTooLargeError(proto.Message): + r"""Structured custom error message for batch size too large + error. The error can be attached as error details in the + returned rpc Status for more structured error handling in the + client. + + Attributes: + max_batch_size (int): + The maximum number of items that are + supported in a single batch. This is returned as + a hint to the client to adjust the batch size. + error_message (str): + Optional. The error message that is returned + to the client. + """ + + max_batch_size: int = proto.Field( + proto.INT64, + number=1, + ) + error_message: str = proto.Field( + proto.STRING, + number=2, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/types/partition.py b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/types/partition.py new file mode 100644 index 00000000..a4568286 --- /dev/null +++ b/owl-bot-staging/v1alpha/google/cloud/bigquery_storage_v1alpha/types/partition.py @@ -0,0 +1,256 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + +from google.protobuf import timestamp_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1alpha', + manifest={ + 'FieldSchema', + 'StorageDescriptor', + 'SerDeInfo', + 'MetastorePartition', + 'MetastorePartitionList', + 'ReadStream', + 'StreamList', + 'MetastorePartitionValues', + }, +) + + +class FieldSchema(proto.Message): + r"""Schema description of a metastore partition column. + + Attributes: + name (str): + Required. The name of the column. + The maximum length of the name is 1024 + characters + type_ (str): + Required. The type of the metastore partition + column. Maximum allowed length is 1024 + characters. + """ + + name: str = proto.Field( + proto.STRING, + number=1, + ) + type_: str = proto.Field( + proto.STRING, + number=2, + ) + + +class StorageDescriptor(proto.Message): + r"""Contains information about the physical storage of the data + in the metastore partition. + + Attributes: + location_uri (str): + Optional. The physical location of the metastore partition + (e.g. + ``gs://spark-dataproc-data/pangea-data/case_sensitive/`` or + ``gs://spark-dataproc-data/pangea-data/*``). + input_format (str): + Optional. Specifies the fully qualified class + name of the InputFormat (e.g. + "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat"). + The maximum length is 128 characters. + output_format (str): + Optional. Specifies the fully qualified class + name of the OutputFormat (e.g. + "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat"). + The maximum length is 128 characters. + serde_info (google.cloud.bigquery_storage_v1alpha.types.SerDeInfo): + Optional. Serializer and deserializer + information. + """ + + location_uri: str = proto.Field( + proto.STRING, + number=1, + ) + input_format: str = proto.Field( + proto.STRING, + number=2, + ) + output_format: str = proto.Field( + proto.STRING, + number=3, + ) + serde_info: 'SerDeInfo' = proto.Field( + proto.MESSAGE, + number=4, + message='SerDeInfo', + ) + + +class SerDeInfo(proto.Message): + r"""Serializer and deserializer information. + + Attributes: + name (str): + Optional. Name of the SerDe. + The maximum length is 256 characters. + serialization_library (str): + Required. Specifies a fully-qualified class + name of the serialization library that is + responsible for the translation of data between + table representation and the underlying + low-level input and output format structures. + The maximum length is 256 characters. + parameters (MutableMapping[str, str]): + Optional. Key-value pairs that define the + initialization parameters for the serialization + library. Maximum size 10 Kib. + """ + + name: str = proto.Field( + proto.STRING, + number=1, + ) + serialization_library: str = proto.Field( + proto.STRING, + number=2, + ) + parameters: MutableMapping[str, str] = proto.MapField( + proto.STRING, + proto.STRING, + number=3, + ) + + +class MetastorePartition(proto.Message): + r"""Information about a Hive partition. + + Attributes: + values (MutableSequence[str]): + Required. Represents the values of the + partition keys, where each value corresponds to + a specific partition key in the order in which + the keys are defined. Each value is limited to + 1024 characters. + create_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. The creation time of the + partition. + storage_descriptor (google.cloud.bigquery_storage_v1alpha.types.StorageDescriptor): + Optional. Contains information about the + physical storage of the data in the partition. + parameters (MutableMapping[str, str]): + Optional. Additional parameters or metadata + associated with the partition. Maximum size 10 + KiB. + fields (MutableSequence[google.cloud.bigquery_storage_v1alpha.types.FieldSchema]): + Optional. List of columns. + """ + + values: MutableSequence[str] = proto.RepeatedField( + proto.STRING, + number=1, + ) + create_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=2, + message=timestamp_pb2.Timestamp, + ) + storage_descriptor: 'StorageDescriptor' = proto.Field( + proto.MESSAGE, + number=3, + message='StorageDescriptor', + ) + parameters: MutableMapping[str, str] = proto.MapField( + proto.STRING, + proto.STRING, + number=4, + ) + fields: MutableSequence['FieldSchema'] = proto.RepeatedField( + proto.MESSAGE, + number=5, + message='FieldSchema', + ) + + +class MetastorePartitionList(proto.Message): + r"""List of metastore partitions. + + Attributes: + partitions (MutableSequence[google.cloud.bigquery_storage_v1alpha.types.MetastorePartition]): + Required. List of partitions. + """ + + partitions: MutableSequence['MetastorePartition'] = proto.RepeatedField( + proto.MESSAGE, + number=1, + message='MetastorePartition', + ) + + +class ReadStream(proto.Message): + r"""Information about a single stream that is used to read + partitions. + + Attributes: + name (str): + Output only. Identifier. Name of the stream, in the form + ``projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}``. + """ + + name: str = proto.Field( + proto.STRING, + number=1, + ) + + +class StreamList(proto.Message): + r"""List of streams. + + Attributes: + streams (MutableSequence[google.cloud.bigquery_storage_v1alpha.types.ReadStream]): + Output only. List of streams. + """ + + streams: MutableSequence['ReadStream'] = proto.RepeatedField( + proto.MESSAGE, + number=1, + message='ReadStream', + ) + + +class MetastorePartitionValues(proto.Message): + r"""Represents the values of a metastore partition. + + Attributes: + values (MutableSequence[str]): + Required. The values of the partition keys, + where each value corresponds to a specific + partition key in the order in which the keys are + defined. + """ + + values: MutableSequence[str] = proto.RepeatedField( + proto.STRING, + number=1, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1alpha/mypy.ini b/owl-bot-staging/v1alpha/mypy.ini new file mode 100644 index 00000000..574c5aed --- /dev/null +++ b/owl-bot-staging/v1alpha/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +python_version = 3.7 +namespace_packages = True diff --git a/owl-bot-staging/v1alpha/noxfile.py b/owl-bot-staging/v1alpha/noxfile.py new file mode 100644 index 00000000..b1bb157c --- /dev/null +++ b/owl-bot-staging/v1alpha/noxfile.py @@ -0,0 +1,278 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import pathlib +import re +import shutil +import subprocess +import sys + + +import nox # type: ignore + +ALL_PYTHON = [ + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", + "3.12" +] + +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt" +PACKAGE_NAME = 'google-cloud-bigquery-storage' + +BLACK_VERSION = "black==22.3.0" +BLACK_PATHS = ["docs", "google", "tests", "samples", "noxfile.py", "setup.py"] +DEFAULT_PYTHON_VERSION = "3.12" + +nox.sessions = [ + "unit", + "cover", + "mypy", + "check_lower_bounds" + # exclude update_lower_bounds from default + "docs", + "blacken", + "lint", + "prerelease_deps", +] + +@nox.session(python=ALL_PYTHON) +@nox.parametrize( + "protobuf_implementation", + [ "python", "upb", "cpp" ], +) +def unit(session, protobuf_implementation): + """Run the unit test suite.""" + + if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"): + session.skip("cpp implementation is not supported in python 3.11+") + + session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"') + session.install('-e', '.', "-c", f"testing/constraints-{session.python}.txt") + + # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. + # The 'cpp' implementation requires Protobuf<4. + if protobuf_implementation == "cpp": + session.install("protobuf<4") + + session.run( + 'py.test', + '--quiet', + '--cov=google/cloud/bigquery_storage_v1alpha/', + '--cov=tests/', + '--cov-config=.coveragerc', + '--cov-report=term', + '--cov-report=html', + os.path.join('tests', 'unit', ''.join(session.posargs)), + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + +@nox.session(python=ALL_PYTHON[-1]) +@nox.parametrize( + "protobuf_implementation", + [ "python", "upb", "cpp" ], +) +def prerelease_deps(session, protobuf_implementation): + """Run the unit test suite against pre-release versions of dependencies.""" + + if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"): + session.skip("cpp implementation is not supported in python 3.11+") + + # Install test environment dependencies + session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"') + + # Install the package without dependencies + session.install('-e', '.', '--no-deps') + + # We test the minimum dependency versions using the minimum Python + # version so the lowest python runtime that we test has a corresponding constraints + # file, located at `testing/constraints--.txt`, which contains all of the + # dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{ALL_PYTHON[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + constraints_deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + session.install(*constraints_deps) + + prerel_deps = [ + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpcio", + "grpcio-status", + "protobuf", + "proto-plus", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = [ + "requests", + ] + session.install(*other_deps) + + # Print out prerelease package versions + + session.run("python", "-c", "import google.api_core; print(google.api_core.__version__)") + session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + session.run("python", "-c", "import grpc; print(grpc.__version__)") + session.run( + "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" + ) + session.run( + "python", "-c", "import proto; print(proto.__version__)" + ) + + session.run( + 'py.test', + '--quiet', + '--cov=google/cloud/bigquery_storage_v1alpha/', + '--cov=tests/', + '--cov-config=.coveragerc', + '--cov-report=term', + '--cov-report=html', + os.path.join('tests', 'unit', ''.join(session.posargs)), + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def cover(session): + """Run the final coverage report. + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=100") + + session.run("coverage", "erase") + + +@nox.session(python=ALL_PYTHON) +def mypy(session): + """Run the type checker.""" + session.install( + 'mypy', + 'types-requests', + 'types-protobuf' + ) + session.install('.') + session.run( + 'mypy', + '-p', + 'google', + ) + + +@nox.session +def update_lower_bounds(session): + """Update lower bounds in constraints.txt to match setup.py""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'update', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + + +@nox.session +def check_lower_bounds(session): + """Check lower bounds in setup.py are reflected in constraints file""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'check', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docs(session): + """Build the docs for this library.""" + + session.install("-e", ".") + session.install("sphinx==7.0.1", "alabaster", "recommonmark") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint(session): + """Run linters. + + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + session.install("flake8", BLACK_VERSION) + session.run( + "black", + "--check", + *BLACK_PATHS, + ) + session.run("flake8", "google", "tests", "samples") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def blacken(session): + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + session.run( + "black", + *BLACK_PATHS, + ) diff --git a/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_create_metastore_partitions_async.py b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_create_metastore_partitions_async.py new file mode 100644 index 00000000..ee7b53f7 --- /dev/null +++ b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_create_metastore_partitions_async.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for BatchCreateMetastorePartitions +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchCreateMetastorePartitions_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1alpha + + +async def sample_batch_create_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient() + + # Initialize request argument(s) + requests = bigquery_storage_v1alpha.CreateMetastorePartitionRequest() + requests.parent = "parent_value" + requests.metastore_partition.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchCreateMetastorePartitionsRequest( + parent="parent_value", + requests=requests, + ) + + # Make the request + response = await client.batch_create_metastore_partitions(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchCreateMetastorePartitions_async] diff --git a/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_create_metastore_partitions_sync.py b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_create_metastore_partitions_sync.py new file mode 100644 index 00000000..4c2c33cf --- /dev/null +++ b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_create_metastore_partitions_sync.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for BatchCreateMetastorePartitions +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchCreateMetastorePartitions_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1alpha + + +def sample_batch_create_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceClient() + + # Initialize request argument(s) + requests = bigquery_storage_v1alpha.CreateMetastorePartitionRequest() + requests.parent = "parent_value" + requests.metastore_partition.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchCreateMetastorePartitionsRequest( + parent="parent_value", + requests=requests, + ) + + # Make the request + response = client.batch_create_metastore_partitions(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchCreateMetastorePartitions_sync] diff --git a/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_delete_metastore_partitions_async.py b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_delete_metastore_partitions_async.py new file mode 100644 index 00000000..4f2974c0 --- /dev/null +++ b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_delete_metastore_partitions_async.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for BatchDeleteMetastorePartitions +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchDeleteMetastorePartitions_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1alpha + + +async def sample_batch_delete_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient() + + # Initialize request argument(s) + partition_values = bigquery_storage_v1alpha.MetastorePartitionValues() + partition_values.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchDeleteMetastorePartitionsRequest( + parent="parent_value", + partition_values=partition_values, + ) + + # Make the request + await client.batch_delete_metastore_partitions(request=request) + + +# [END bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchDeleteMetastorePartitions_async] diff --git a/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_delete_metastore_partitions_sync.py b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_delete_metastore_partitions_sync.py new file mode 100644 index 00000000..38ea8f2c --- /dev/null +++ b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_delete_metastore_partitions_sync.py @@ -0,0 +1,54 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for BatchDeleteMetastorePartitions +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchDeleteMetastorePartitions_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1alpha + + +def sample_batch_delete_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceClient() + + # Initialize request argument(s) + partition_values = bigquery_storage_v1alpha.MetastorePartitionValues() + partition_values.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchDeleteMetastorePartitionsRequest( + parent="parent_value", + partition_values=partition_values, + ) + + # Make the request + client.batch_delete_metastore_partitions(request=request) + + +# [END bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchDeleteMetastorePartitions_sync] diff --git a/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_update_metastore_partitions_async.py b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_update_metastore_partitions_async.py new file mode 100644 index 00000000..1cc5b862 --- /dev/null +++ b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_update_metastore_partitions_async.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for BatchUpdateMetastorePartitions +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchUpdateMetastorePartitions_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1alpha + + +async def sample_batch_update_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient() + + # Initialize request argument(s) + requests = bigquery_storage_v1alpha.UpdateMetastorePartitionRequest() + requests.metastore_partition.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchUpdateMetastorePartitionsRequest( + parent="parent_value", + requests=requests, + ) + + # Make the request + response = await client.batch_update_metastore_partitions(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchUpdateMetastorePartitions_async] diff --git a/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_update_metastore_partitions_sync.py b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_update_metastore_partitions_sync.py new file mode 100644 index 00000000..4e7743e9 --- /dev/null +++ b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_batch_update_metastore_partitions_sync.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for BatchUpdateMetastorePartitions +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchUpdateMetastorePartitions_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1alpha + + +def sample_batch_update_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceClient() + + # Initialize request argument(s) + requests = bigquery_storage_v1alpha.UpdateMetastorePartitionRequest() + requests.metastore_partition.values = ['values_value1', 'values_value2'] + + request = bigquery_storage_v1alpha.BatchUpdateMetastorePartitionsRequest( + parent="parent_value", + requests=requests, + ) + + # Make the request + response = client.batch_update_metastore_partitions(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchUpdateMetastorePartitions_sync] diff --git a/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_list_metastore_partitions_async.py b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_list_metastore_partitions_async.py new file mode 100644 index 00000000..8196a2a1 --- /dev/null +++ b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_list_metastore_partitions_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for ListMetastorePartitions +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1alpha_generated_MetastorePartitionService_ListMetastorePartitions_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1alpha + + +async def sample_list_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1alpha.ListMetastorePartitionsRequest( + parent="parent_value", + ) + + # Make the request + response = await client.list_metastore_partitions(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1alpha_generated_MetastorePartitionService_ListMetastorePartitions_async] diff --git a/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_list_metastore_partitions_sync.py b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_list_metastore_partitions_sync.py new file mode 100644 index 00000000..a3f66e76 --- /dev/null +++ b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_list_metastore_partitions_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for ListMetastorePartitions +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1alpha_generated_MetastorePartitionService_ListMetastorePartitions_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1alpha + + +def sample_list_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceClient() + + # Initialize request argument(s) + request = bigquery_storage_v1alpha.ListMetastorePartitionsRequest( + parent="parent_value", + ) + + # Make the request + response = client.list_metastore_partitions(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1alpha_generated_MetastorePartitionService_ListMetastorePartitions_sync] diff --git a/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_stream_metastore_partitions_async.py b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_stream_metastore_partitions_async.py new file mode 100644 index 00000000..947bac86 --- /dev/null +++ b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_stream_metastore_partitions_async.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for StreamMetastorePartitions +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1alpha_generated_MetastorePartitionService_StreamMetastorePartitions_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1alpha + + +async def sample_stream_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1alpha.StreamMetastorePartitionsRequest( + parent="parent_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1alpha.StreamMetastorePartitionsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = await client.stream_metastore_partitions(requests=request_generator()) + + # Handle the response + async for response in stream: + print(response) + +# [END bigquerystorage_v1alpha_generated_MetastorePartitionService_StreamMetastorePartitions_async] diff --git a/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_stream_metastore_partitions_sync.py b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_stream_metastore_partitions_sync.py new file mode 100644 index 00000000..701bcb57 --- /dev/null +++ b/owl-bot-staging/v1alpha/samples/generated_samples/bigquerystorage_v1alpha_generated_metastore_partition_service_stream_metastore_partitions_sync.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for StreamMetastorePartitions +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1alpha_generated_MetastorePartitionService_StreamMetastorePartitions_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1alpha + + +def sample_stream_metastore_partitions(): + # Create a client + client = bigquery_storage_v1alpha.MetastorePartitionServiceClient() + + # Initialize request argument(s) + request = bigquery_storage_v1alpha.StreamMetastorePartitionsRequest( + parent="parent_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1alpha.StreamMetastorePartitionsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = client.stream_metastore_partitions(requests=request_generator()) + + # Handle the response + for response in stream: + print(response) + +# [END bigquerystorage_v1alpha_generated_MetastorePartitionService_StreamMetastorePartitions_sync] diff --git a/owl-bot-staging/v1alpha/samples/generated_samples/snippet_metadata_google.cloud.bigquery.storage.v1alpha.json b/owl-bot-staging/v1alpha/samples/generated_samples/snippet_metadata_google.cloud.bigquery.storage.v1alpha.json new file mode 100644 index 00000000..3c29286e --- /dev/null +++ b/owl-bot-staging/v1alpha/samples/generated_samples/snippet_metadata_google.cloud.bigquery.storage.v1alpha.json @@ -0,0 +1,782 @@ +{ + "clientLibrary": { + "apis": [ + { + "id": "google.cloud.bigquery.storage.v1alpha", + "version": "v1alpha" + } + ], + "language": "PYTHON", + "name": "google-cloud-bigquery-storage", + "version": "0.1.0" + }, + "snippets": [ + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient", + "shortName": "MetastorePartitionServiceAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient.batch_create_metastore_partitions", + "method": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService.BatchCreateMetastorePartitions", + "service": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService", + "shortName": "MetastorePartitionService" + }, + "shortName": "BatchCreateMetastorePartitions" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1alpha.types.BatchCreateMetastorePartitionsRequest" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1alpha.types.BatchCreateMetastorePartitionsResponse", + "shortName": "batch_create_metastore_partitions" + }, + "description": "Sample for BatchCreateMetastorePartitions", + "file": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_create_metastore_partitions_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchCreateMetastorePartitions_async", + "segments": [ + { + "end": 56, + "start": 27, + "type": "FULL" + }, + { + "end": 56, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 50, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 53, + "start": 51, + "type": "REQUEST_EXECUTION" + }, + { + "end": 57, + "start": 54, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_create_metastore_partitions_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceClient", + "shortName": "MetastorePartitionServiceClient" + }, + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceClient.batch_create_metastore_partitions", + "method": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService.BatchCreateMetastorePartitions", + "service": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService", + "shortName": "MetastorePartitionService" + }, + "shortName": "BatchCreateMetastorePartitions" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1alpha.types.BatchCreateMetastorePartitionsRequest" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1alpha.types.BatchCreateMetastorePartitionsResponse", + "shortName": "batch_create_metastore_partitions" + }, + "description": "Sample for BatchCreateMetastorePartitions", + "file": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_create_metastore_partitions_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchCreateMetastorePartitions_sync", + "segments": [ + { + "end": 56, + "start": 27, + "type": "FULL" + }, + { + "end": 56, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 50, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 53, + "start": 51, + "type": "REQUEST_EXECUTION" + }, + { + "end": 57, + "start": 54, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_create_metastore_partitions_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient", + "shortName": "MetastorePartitionServiceAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient.batch_delete_metastore_partitions", + "method": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService.BatchDeleteMetastorePartitions", + "service": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService", + "shortName": "MetastorePartitionService" + }, + "shortName": "BatchDeleteMetastorePartitions" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1alpha.types.BatchDeleteMetastorePartitionsRequest" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "shortName": "batch_delete_metastore_partitions" + }, + "description": "Sample for BatchDeleteMetastorePartitions", + "file": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_delete_metastore_partitions_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchDeleteMetastorePartitions_async", + "segments": [ + { + "end": 53, + "start": 27, + "type": "FULL" + }, + { + "end": 53, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 49, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "start": 50, + "type": "REQUEST_EXECUTION" + }, + { + "end": 54, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_delete_metastore_partitions_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceClient", + "shortName": "MetastorePartitionServiceClient" + }, + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceClient.batch_delete_metastore_partitions", + "method": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService.BatchDeleteMetastorePartitions", + "service": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService", + "shortName": "MetastorePartitionService" + }, + "shortName": "BatchDeleteMetastorePartitions" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1alpha.types.BatchDeleteMetastorePartitionsRequest" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "shortName": "batch_delete_metastore_partitions" + }, + "description": "Sample for BatchDeleteMetastorePartitions", + "file": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_delete_metastore_partitions_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchDeleteMetastorePartitions_sync", + "segments": [ + { + "end": 53, + "start": 27, + "type": "FULL" + }, + { + "end": 53, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 49, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "start": 50, + "type": "REQUEST_EXECUTION" + }, + { + "end": 54, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_delete_metastore_partitions_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient", + "shortName": "MetastorePartitionServiceAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient.batch_update_metastore_partitions", + "method": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService.BatchUpdateMetastorePartitions", + "service": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService", + "shortName": "MetastorePartitionService" + }, + "shortName": "BatchUpdateMetastorePartitions" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1alpha.types.BatchUpdateMetastorePartitionsRequest" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1alpha.types.BatchUpdateMetastorePartitionsResponse", + "shortName": "batch_update_metastore_partitions" + }, + "description": "Sample for BatchUpdateMetastorePartitions", + "file": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_update_metastore_partitions_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchUpdateMetastorePartitions_async", + "segments": [ + { + "end": 55, + "start": 27, + "type": "FULL" + }, + { + "end": 55, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 49, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 52, + "start": 50, + "type": "REQUEST_EXECUTION" + }, + { + "end": 56, + "start": 53, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_update_metastore_partitions_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceClient", + "shortName": "MetastorePartitionServiceClient" + }, + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceClient.batch_update_metastore_partitions", + "method": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService.BatchUpdateMetastorePartitions", + "service": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService", + "shortName": "MetastorePartitionService" + }, + "shortName": "BatchUpdateMetastorePartitions" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1alpha.types.BatchUpdateMetastorePartitionsRequest" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1alpha.types.BatchUpdateMetastorePartitionsResponse", + "shortName": "batch_update_metastore_partitions" + }, + "description": "Sample for BatchUpdateMetastorePartitions", + "file": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_update_metastore_partitions_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1alpha_generated_MetastorePartitionService_BatchUpdateMetastorePartitions_sync", + "segments": [ + { + "end": 55, + "start": 27, + "type": "FULL" + }, + { + "end": 55, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 49, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 52, + "start": 50, + "type": "REQUEST_EXECUTION" + }, + { + "end": 56, + "start": 53, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1alpha_generated_metastore_partition_service_batch_update_metastore_partitions_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient", + "shortName": "MetastorePartitionServiceAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient.list_metastore_partitions", + "method": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService.ListMetastorePartitions", + "service": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService", + "shortName": "MetastorePartitionService" + }, + "shortName": "ListMetastorePartitions" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1alpha.types.ListMetastorePartitionsRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1alpha.types.ListMetastorePartitionsResponse", + "shortName": "list_metastore_partitions" + }, + "description": "Sample for ListMetastorePartitions", + "file": "bigquerystorage_v1alpha_generated_metastore_partition_service_list_metastore_partitions_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1alpha_generated_MetastorePartitionService_ListMetastorePartitions_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1alpha_generated_metastore_partition_service_list_metastore_partitions_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceClient", + "shortName": "MetastorePartitionServiceClient" + }, + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceClient.list_metastore_partitions", + "method": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService.ListMetastorePartitions", + "service": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService", + "shortName": "MetastorePartitionService" + }, + "shortName": "ListMetastorePartitions" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1alpha.types.ListMetastorePartitionsRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1alpha.types.ListMetastorePartitionsResponse", + "shortName": "list_metastore_partitions" + }, + "description": "Sample for ListMetastorePartitions", + "file": "bigquerystorage_v1alpha_generated_metastore_partition_service_list_metastore_partitions_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1alpha_generated_MetastorePartitionService_ListMetastorePartitions_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1alpha_generated_metastore_partition_service_list_metastore_partitions_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient", + "shortName": "MetastorePartitionServiceAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceAsyncClient.stream_metastore_partitions", + "method": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService.StreamMetastorePartitions", + "service": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService", + "shortName": "MetastorePartitionService" + }, + "shortName": "StreamMetastorePartitions" + }, + "parameters": [ + { + "name": "requests", + "type": "Iterator[google.cloud.bigquery_storage_v1alpha.types.StreamMetastorePartitionsRequest]" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "Iterable[google.cloud.bigquery_storage_v1alpha.types.StreamMetastorePartitionsResponse]", + "shortName": "stream_metastore_partitions" + }, + "description": "Sample for StreamMetastorePartitions", + "file": "bigquerystorage_v1alpha_generated_metastore_partition_service_stream_metastore_partitions_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1alpha_generated_MetastorePartitionService_StreamMetastorePartitions_async", + "segments": [ + { + "end": 62, + "start": 27, + "type": "FULL" + }, + { + "end": 62, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 55, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 58, + "start": 56, + "type": "REQUEST_EXECUTION" + }, + { + "end": 63, + "start": 59, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1alpha_generated_metastore_partition_service_stream_metastore_partitions_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceClient", + "shortName": "MetastorePartitionServiceClient" + }, + "fullName": "google.cloud.bigquery_storage_v1alpha.MetastorePartitionServiceClient.stream_metastore_partitions", + "method": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService.StreamMetastorePartitions", + "service": { + "fullName": "google.cloud.bigquery.storage.v1alpha.MetastorePartitionService", + "shortName": "MetastorePartitionService" + }, + "shortName": "StreamMetastorePartitions" + }, + "parameters": [ + { + "name": "requests", + "type": "Iterator[google.cloud.bigquery_storage_v1alpha.types.StreamMetastorePartitionsRequest]" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "Iterable[google.cloud.bigquery_storage_v1alpha.types.StreamMetastorePartitionsResponse]", + "shortName": "stream_metastore_partitions" + }, + "description": "Sample for StreamMetastorePartitions", + "file": "bigquerystorage_v1alpha_generated_metastore_partition_service_stream_metastore_partitions_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1alpha_generated_MetastorePartitionService_StreamMetastorePartitions_sync", + "segments": [ + { + "end": 62, + "start": 27, + "type": "FULL" + }, + { + "end": 62, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 55, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 58, + "start": 56, + "type": "REQUEST_EXECUTION" + }, + { + "end": 63, + "start": 59, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1alpha_generated_metastore_partition_service_stream_metastore_partitions_sync.py" + } + ] +} diff --git a/owl-bot-staging/v1alpha/scripts/fixup_bigquery_storage_v1alpha_keywords.py b/owl-bot-staging/v1alpha/scripts/fixup_bigquery_storage_v1alpha_keywords.py new file mode 100644 index 00000000..5a01e5af --- /dev/null +++ b/owl-bot-staging/v1alpha/scripts/fixup_bigquery_storage_v1alpha_keywords.py @@ -0,0 +1,180 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import os +import libcst as cst +import pathlib +import sys +from typing import (Any, Callable, Dict, List, Sequence, Tuple) + + +def partition( + predicate: Callable[[Any], bool], + iterator: Sequence[Any] +) -> Tuple[List[Any], List[Any]]: + """A stable, out-of-place partition.""" + results = ([], []) + + for i in iterator: + results[int(predicate(i))].append(i) + + # Returns trueList, falseList + return results[1], results[0] + + +class bigquery_storageCallTransformer(cst.CSTTransformer): + CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') + METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { + 'batch_create_metastore_partitions': ('parent', 'requests', 'skip_existing_partitions', ), + 'batch_delete_metastore_partitions': ('parent', 'partition_values', ), + 'batch_update_metastore_partitions': ('parent', 'requests', ), + 'list_metastore_partitions': ('parent', 'filter', ), + 'stream_metastore_partitions': ('parent', 'metastore_partitions', 'skip_existing_partitions', ), + } + + def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: + try: + key = original.func.attr.value + kword_params = self.METHOD_TO_PARAMS[key] + except (AttributeError, KeyError): + # Either not a method from the API or too convoluted to be sure. + return updated + + # If the existing code is valid, keyword args come after positional args. + # Therefore, all positional args must map to the first parameters. + args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) + if any(k.keyword.value == "request" for k in kwargs): + # We've already fixed this file, don't fix it again. + return updated + + kwargs, ctrl_kwargs = partition( + lambda a: a.keyword.value not in self.CTRL_PARAMS, + kwargs + ) + + args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] + ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) + for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) + + request_arg = cst.Arg( + value=cst.Dict([ + cst.DictElement( + cst.SimpleString("'{}'".format(name)), +cst.Element(value=arg.value) + ) + # Note: the args + kwargs looks silly, but keep in mind that + # the control parameters had to be stripped out, and that + # those could have been passed positionally or by keyword. + for name, arg in zip(kword_params, args + kwargs)]), + keyword=cst.Name("request") + ) + + return updated.with_changes( + args=[request_arg] + ctrl_kwargs + ) + + +def fix_files( + in_dir: pathlib.Path, + out_dir: pathlib.Path, + *, + transformer=bigquery_storageCallTransformer(), +): + """Duplicate the input dir to the output dir, fixing file method calls. + + Preconditions: + * in_dir is a real directory + * out_dir is a real, empty directory + """ + pyfile_gen = ( + pathlib.Path(os.path.join(root, f)) + for root, _, files in os.walk(in_dir) + for f in files if os.path.splitext(f)[1] == ".py" + ) + + for fpath in pyfile_gen: + with open(fpath, 'r') as f: + src = f.read() + + # Parse the code and insert method call fixes. + tree = cst.parse_module(src) + updated = tree.visit(transformer) + + # Create the path and directory structure for the new file. + updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) + updated_path.parent.mkdir(parents=True, exist_ok=True) + + # Generate the updated source file at the corresponding path. + with open(updated_path, 'w') as f: + f.write(updated.code) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="""Fix up source that uses the bigquery_storage client library. + +The existing sources are NOT overwritten but are copied to output_dir with changes made. + +Note: This tool operates at a best-effort level at converting positional + parameters in client method calls to keyword based parameters. + Cases where it WILL FAIL include + A) * or ** expansion in a method call. + B) Calls via function or method alias (includes free function calls) + C) Indirect or dispatched calls (e.g. the method is looked up dynamically) + + These all constitute false negatives. The tool will also detect false + positives when an API method shares a name with another method. +""") + parser.add_argument( + '-d', + '--input-directory', + required=True, + dest='input_dir', + help='the input directory to walk for python files to fix up', + ) + parser.add_argument( + '-o', + '--output-directory', + required=True, + dest='output_dir', + help='the directory to output files fixed via un-flattening', + ) + args = parser.parse_args() + input_dir = pathlib.Path(args.input_dir) + output_dir = pathlib.Path(args.output_dir) + if not input_dir.is_dir(): + print( + f"input directory '{input_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if not output_dir.is_dir(): + print( + f"output directory '{output_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if os.listdir(output_dir): + print( + f"output directory '{output_dir}' is not empty", + file=sys.stderr, + ) + sys.exit(-1) + + fix_files(input_dir, output_dir) diff --git a/owl-bot-staging/v1alpha/setup.py b/owl-bot-staging/v1alpha/setup.py new file mode 100644 index 00000000..b688ea84 --- /dev/null +++ b/owl-bot-staging/v1alpha/setup.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import io +import os +import re + +import setuptools # type: ignore + +package_root = os.path.abspath(os.path.dirname(__file__)) + +name = 'google-cloud-bigquery-storage' + + +description = "Google Cloud Bigquery Storage API client library" + +version = None + +with open(os.path.join(package_root, 'google/cloud/bigquery_storage/gapic_version.py')) as fp: + version_candidates = re.findall(r"(?<=\")\d+.\d+.\d+(?=\")", fp.read()) + assert (len(version_candidates) == 1) + version = version_candidates[0] + +if version[0] == "0": + release_status = "Development Status :: 4 - Beta" +else: + release_status = "Development Status :: 5 - Production/Stable" + +dependencies = [ + "google-api-core[grpc] >= 1.34.1, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", + # Exclude incompatible versions of `google-auth` + # See https://github.com/googleapis/google-cloud-python/issues/12364 + "google-auth >= 2.14.1, <3.0.0dev,!=2.24.0,!=2.25.0", + "proto-plus >= 1.22.3, <2.0.0dev", + "protobuf>=3.20.2,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", +] +extras = { +} +url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-cloud-bigquery-storage" + +package_root = os.path.abspath(os.path.dirname(__file__)) + +readme_filename = os.path.join(package_root, "README.rst") +with io.open(readme_filename, encoding="utf-8") as readme_file: + readme = readme_file.read() + +packages = [ + package + for package in setuptools.find_namespace_packages() + if package.startswith("google") +] + +setuptools.setup( + name=name, + version=version, + description=description, + long_description=readme, + author="Google LLC", + author_email="googleapis-packages@google.com", + license="Apache 2.0", + url=url, + classifiers=[ + release_status, + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Topic :: Internet", + ], + platforms="Posix; MacOS X; Windows", + packages=packages, + python_requires=">=3.7", + install_requires=dependencies, + extras_require=extras, + include_package_data=True, + zip_safe=False, +) diff --git a/owl-bot-staging/v1alpha/testing/constraints-3.10.txt b/owl-bot-staging/v1alpha/testing/constraints-3.10.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1alpha/testing/constraints-3.10.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1alpha/testing/constraints-3.11.txt b/owl-bot-staging/v1alpha/testing/constraints-3.11.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1alpha/testing/constraints-3.11.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1alpha/testing/constraints-3.12.txt b/owl-bot-staging/v1alpha/testing/constraints-3.12.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1alpha/testing/constraints-3.12.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1alpha/testing/constraints-3.7.txt b/owl-bot-staging/v1alpha/testing/constraints-3.7.txt new file mode 100644 index 00000000..fc812592 --- /dev/null +++ b/owl-bot-staging/v1alpha/testing/constraints-3.7.txt @@ -0,0 +1,10 @@ +# This constraints file is used to check that lower bounds +# are correct in setup.py +# List all library dependencies and extras in this file. +# Pin the version to the lower bound. +# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0dev", +# Then this file should have google-cloud-foo==1.14.0 +google-api-core==1.34.1 +google-auth==2.14.1 +proto-plus==1.22.3 +protobuf==3.20.2 diff --git a/owl-bot-staging/v1alpha/testing/constraints-3.8.txt b/owl-bot-staging/v1alpha/testing/constraints-3.8.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1alpha/testing/constraints-3.8.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1alpha/testing/constraints-3.9.txt b/owl-bot-staging/v1alpha/testing/constraints-3.9.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1alpha/testing/constraints-3.9.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1alpha/tests/__init__.py b/owl-bot-staging/v1alpha/tests/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1alpha/tests/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1alpha/tests/unit/__init__.py b/owl-bot-staging/v1alpha/tests/unit/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1alpha/tests/unit/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1alpha/tests/unit/gapic/__init__.py b/owl-bot-staging/v1alpha/tests/unit/gapic/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1alpha/tests/unit/gapic/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1alpha/tests/unit/gapic/bigquery_storage_v1alpha/__init__.py b/owl-bot-staging/v1alpha/tests/unit/gapic/bigquery_storage_v1alpha/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1alpha/tests/unit/gapic/bigquery_storage_v1alpha/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1alpha/tests/unit/gapic/bigquery_storage_v1alpha/test_metastore_partition_service.py b/owl-bot-staging/v1alpha/tests/unit/gapic/bigquery_storage_v1alpha/test_metastore_partition_service.py new file mode 100644 index 00000000..cce71d8e --- /dev/null +++ b/owl-bot-staging/v1alpha/tests/unit/gapic/bigquery_storage_v1alpha/test_metastore_partition_service.py @@ -0,0 +1,2751 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +# try/except added for compatibility with python < 3.8 +try: + from unittest import mock + from unittest.mock import AsyncMock # pragma: NO COVER +except ImportError: # pragma: NO COVER + import mock + +import grpc +from grpc.experimental import aio +import math +import pytest +from google.api_core import api_core_version +from proto.marshal.rules.dates import DurationRule, TimestampRule +from proto.marshal.rules import wrappers + +try: + from google.auth.aio import credentials as ga_credentials_async + HAS_GOOGLE_AUTH_AIO = True +except ImportError: # pragma: NO COVER + HAS_GOOGLE_AUTH_AIO = False + +from google.api_core import client_options +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers +from google.api_core import grpc_helpers_async +from google.api_core import path_template +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials +from google.auth.exceptions import MutualTLSChannelError +from google.cloud.bigquery_storage_v1alpha.services.metastore_partition_service import MetastorePartitionServiceAsyncClient +from google.cloud.bigquery_storage_v1alpha.services.metastore_partition_service import MetastorePartitionServiceClient +from google.cloud.bigquery_storage_v1alpha.services.metastore_partition_service import transports +from google.cloud.bigquery_storage_v1alpha.types import metastore_partition +from google.cloud.bigquery_storage_v1alpha.types import partition +from google.oauth2 import service_account +from google.protobuf import field_mask_pb2 # type: ignore +from google.protobuf import timestamp_pb2 # type: ignore +import google.auth + + +async def mock_async_gen(data, chunk_size=1): + for i in range(0, len(data)): # pragma: NO COVER + chunk = data[i : i + chunk_size] + yield chunk.encode("utf-8") + +def client_cert_source_callback(): + return b"cert bytes", b"key bytes" + +# TODO: use async auth anon credentials by default once the minimum version of google-auth is upgraded. +# See related issue: https://github.com/googleapis/gapic-generator-python/issues/2107. +def async_anonymous_credentials(): + if HAS_GOOGLE_AUTH_AIO: + return ga_credentials_async.AnonymousCredentials() + return ga_credentials.AnonymousCredentials() + +# If default endpoint is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint(client): + return "foo.googleapis.com" if ("localhost" in client.DEFAULT_ENDPOINT) else client.DEFAULT_ENDPOINT + +# If default endpoint template is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint template so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint_template(client): + return "test.{UNIVERSE_DOMAIN}" if ("localhost" in client._DEFAULT_ENDPOINT_TEMPLATE) else client._DEFAULT_ENDPOINT_TEMPLATE + + +def test__get_default_mtls_endpoint(): + api_endpoint = "example.googleapis.com" + api_mtls_endpoint = "example.mtls.googleapis.com" + sandbox_endpoint = "example.sandbox.googleapis.com" + sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" + non_googleapi = "api.example.com" + + assert MetastorePartitionServiceClient._get_default_mtls_endpoint(None) is None + assert MetastorePartitionServiceClient._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint + assert MetastorePartitionServiceClient._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint + assert MetastorePartitionServiceClient._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint + assert MetastorePartitionServiceClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint + assert MetastorePartitionServiceClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi + +def test__read_environment_variables(): + assert MetastorePartitionServiceClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert MetastorePartitionServiceClient._read_environment_variables() == (True, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + assert MetastorePartitionServiceClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + MetastorePartitionServiceClient._read_environment_variables() + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + assert MetastorePartitionServiceClient._read_environment_variables() == (False, "never", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + assert MetastorePartitionServiceClient._read_environment_variables() == (False, "always", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}): + assert MetastorePartitionServiceClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + MetastorePartitionServiceClient._read_environment_variables() + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + with mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}): + assert MetastorePartitionServiceClient._read_environment_variables() == (False, "auto", "foo.com") + +def test__get_client_cert_source(): + mock_provided_cert_source = mock.Mock() + mock_default_cert_source = mock.Mock() + + assert MetastorePartitionServiceClient._get_client_cert_source(None, False) is None + assert MetastorePartitionServiceClient._get_client_cert_source(mock_provided_cert_source, False) is None + assert MetastorePartitionServiceClient._get_client_cert_source(mock_provided_cert_source, True) == mock_provided_cert_source + + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=mock_default_cert_source): + assert MetastorePartitionServiceClient._get_client_cert_source(None, True) is mock_default_cert_source + assert MetastorePartitionServiceClient._get_client_cert_source(mock_provided_cert_source, "true") is mock_provided_cert_source + +@mock.patch.object(MetastorePartitionServiceClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(MetastorePartitionServiceClient)) +@mock.patch.object(MetastorePartitionServiceAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(MetastorePartitionServiceAsyncClient)) +def test__get_api_endpoint(): + api_override = "foo.com" + mock_client_cert_source = mock.Mock() + default_universe = MetastorePartitionServiceClient._DEFAULT_UNIVERSE + default_endpoint = MetastorePartitionServiceClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=default_universe) + mock_universe = "bar.com" + mock_endpoint = MetastorePartitionServiceClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=mock_universe) + + assert MetastorePartitionServiceClient._get_api_endpoint(api_override, mock_client_cert_source, default_universe, "always") == api_override + assert MetastorePartitionServiceClient._get_api_endpoint(None, mock_client_cert_source, default_universe, "auto") == MetastorePartitionServiceClient.DEFAULT_MTLS_ENDPOINT + assert MetastorePartitionServiceClient._get_api_endpoint(None, None, default_universe, "auto") == default_endpoint + assert MetastorePartitionServiceClient._get_api_endpoint(None, None, default_universe, "always") == MetastorePartitionServiceClient.DEFAULT_MTLS_ENDPOINT + assert MetastorePartitionServiceClient._get_api_endpoint(None, mock_client_cert_source, default_universe, "always") == MetastorePartitionServiceClient.DEFAULT_MTLS_ENDPOINT + assert MetastorePartitionServiceClient._get_api_endpoint(None, None, mock_universe, "never") == mock_endpoint + assert MetastorePartitionServiceClient._get_api_endpoint(None, None, default_universe, "never") == default_endpoint + + with pytest.raises(MutualTLSChannelError) as excinfo: + MetastorePartitionServiceClient._get_api_endpoint(None, mock_client_cert_source, mock_universe, "auto") + assert str(excinfo.value) == "mTLS is not supported in any universe other than googleapis.com." + + +def test__get_universe_domain(): + client_universe_domain = "foo.com" + universe_domain_env = "bar.com" + + assert MetastorePartitionServiceClient._get_universe_domain(client_universe_domain, universe_domain_env) == client_universe_domain + assert MetastorePartitionServiceClient._get_universe_domain(None, universe_domain_env) == universe_domain_env + assert MetastorePartitionServiceClient._get_universe_domain(None, None) == MetastorePartitionServiceClient._DEFAULT_UNIVERSE + + with pytest.raises(ValueError) as excinfo: + MetastorePartitionServiceClient._get_universe_domain("", None) + assert str(excinfo.value) == "Universe Domain cannot be an empty string." + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (MetastorePartitionServiceClient, transports.MetastorePartitionServiceGrpcTransport, "grpc"), +]) +def test__validate_universe_domain(client_class, transport_class, transport_name): + client = client_class( + transport=transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + ) + assert client._validate_universe_domain() == True + + # Test the case when universe is already validated. + assert client._validate_universe_domain() == True + + if transport_name == "grpc": + # Test the case where credentials are provided by the + # `local_channel_credentials`. The default universes in both match. + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + client = client_class(transport=transport_class(channel=channel)) + assert client._validate_universe_domain() == True + + # Test the case where credentials do not exist: e.g. a transport is provided + # with no credentials. Validation should still succeed because there is no + # mismatch with non-existent credentials. + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + transport=transport_class(channel=channel) + transport._credentials = None + client = client_class(transport=transport) + assert client._validate_universe_domain() == True + + # TODO: This is needed to cater for older versions of google-auth + # Make this test unconditional once the minimum supported version of + # google-auth becomes 2.23.0 or higher. + google_auth_major, google_auth_minor = [int(part) for part in google.auth.__version__.split(".")[0:2]] + if google_auth_major > 2 or (google_auth_major == 2 and google_auth_minor >= 23): + credentials = ga_credentials.AnonymousCredentials() + credentials._universe_domain = "foo.com" + # Test the case when there is a universe mismatch from the credentials. + client = client_class( + transport=transport_class(credentials=credentials) + ) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert str(excinfo.value) == "The configured universe domain (googleapis.com) does not match the universe domain found in the credentials (foo.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + + # Test the case when there is a universe mismatch from the client. + # + # TODO: Make this test unconditional once the minimum supported version of + # google-api-core becomes 2.15.0 or higher. + api_core_major, api_core_minor = [int(part) for part in api_core_version.__version__.split(".")[0:2]] + if api_core_major > 2 or (api_core_major == 2 and api_core_minor >= 15): + client = client_class(client_options={"universe_domain": "bar.com"}, transport=transport_class(credentials=ga_credentials.AnonymousCredentials(),)) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert str(excinfo.value) == "The configured universe domain (bar.com) does not match the universe domain found in the credentials (googleapis.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + + # Test that ValueError is raised if universe_domain is provided via client options and credentials is None + with pytest.raises(ValueError): + client._compare_universes("foo.bar", None) + + +@pytest.mark.parametrize("client_class,transport_name", [ + (MetastorePartitionServiceClient, "grpc"), + (MetastorePartitionServiceAsyncClient, "grpc_asyncio"), +]) +def test_metastore_partition_service_client_from_service_account_info(client_class, transport_name): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_info') as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info, transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + + +@pytest.mark.parametrize("transport_class,transport_name", [ + (transports.MetastorePartitionServiceGrpcTransport, "grpc"), + (transports.MetastorePartitionServiceGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_metastore_partition_service_client_service_account_always_use_jwt(transport_class, transport_name): + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize("client_class,transport_name", [ + (MetastorePartitionServiceClient, "grpc"), + (MetastorePartitionServiceAsyncClient, "grpc_asyncio"), +]) +def test_metastore_partition_service_client_from_service_account_file(client_class, transport_name): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_file') as factory: + factory.return_value = creds + client = client_class.from_service_account_file("dummy/file/path.json", transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + client = client_class.from_service_account_json("dummy/file/path.json", transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + + +def test_metastore_partition_service_client_get_transport_class(): + transport = MetastorePartitionServiceClient.get_transport_class() + available_transports = [ + transports.MetastorePartitionServiceGrpcTransport, + ] + assert transport in available_transports + + transport = MetastorePartitionServiceClient.get_transport_class("grpc") + assert transport == transports.MetastorePartitionServiceGrpcTransport + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (MetastorePartitionServiceClient, transports.MetastorePartitionServiceGrpcTransport, "grpc"), + (MetastorePartitionServiceAsyncClient, transports.MetastorePartitionServiceGrpcAsyncIOTransport, "grpc_asyncio"), +]) +@mock.patch.object(MetastorePartitionServiceClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(MetastorePartitionServiceClient)) +@mock.patch.object(MetastorePartitionServiceAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(MetastorePartitionServiceAsyncClient)) +def test_metastore_partition_service_client_client_options(client_class, transport_class, transport_name): + # Check that if channel is provided we won't create a new one. + with mock.patch.object(MetastorePartitionServiceClient, 'get_transport_class') as gtc: + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + client = client_class(transport=transport) + gtc.assert_not_called() + + # Check that if channel is provided via str we will create a new one. + with mock.patch.object(MetastorePartitionServiceClient, 'get_transport_class') as gtc: + client = client_class(transport=transport_name) + gtc.assert_called() + + # Check the case api_endpoint is provided. + options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name, client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_MTLS_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client = client_class(transport=transport_name) + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + client = client_class(transport=transport_name) + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + # Check the case api_endpoint is provided + options = client_options.ClientOptions(api_audience="https://language.googleapis.com") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience="https://language.googleapis.com" + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,use_client_cert_env", [ + (MetastorePartitionServiceClient, transports.MetastorePartitionServiceGrpcTransport, "grpc", "true"), + (MetastorePartitionServiceAsyncClient, transports.MetastorePartitionServiceGrpcAsyncIOTransport, "grpc_asyncio", "true"), + (MetastorePartitionServiceClient, transports.MetastorePartitionServiceGrpcTransport, "grpc", "false"), + (MetastorePartitionServiceAsyncClient, transports.MetastorePartitionServiceGrpcAsyncIOTransport, "grpc_asyncio", "false"), +]) +@mock.patch.object(MetastorePartitionServiceClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(MetastorePartitionServiceClient)) +@mock.patch.object(MetastorePartitionServiceAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(MetastorePartitionServiceAsyncClient)) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_metastore_partition_service_client_mtls_env_auto(client_class, transport_class, transport_name, use_client_cert_env): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + options = client_options.ClientOptions(client_cert_source=client_cert_source_callback) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE) + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT + + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=client_cert_source_callback): + if use_client_cert_env == "false": + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE) + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback + + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=False): + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + +@pytest.mark.parametrize("client_class", [ + MetastorePartitionServiceClient, MetastorePartitionServiceAsyncClient +]) +@mock.patch.object(MetastorePartitionServiceClient, "DEFAULT_ENDPOINT", modify_default_endpoint(MetastorePartitionServiceClient)) +@mock.patch.object(MetastorePartitionServiceAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(MetastorePartitionServiceAsyncClient)) +def test_metastore_partition_service_client_get_mtls_endpoint_and_cert_source(client_class): + mock_client_cert_source = mock.Mock() + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "true". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + mock_api_endpoint = "foo" + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source == mock_client_cert_source + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "false". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert doesn't exist. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=False): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert exists. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=mock_client_cert_source): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source == mock_client_cert_source + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + +@pytest.mark.parametrize("client_class", [ + MetastorePartitionServiceClient, MetastorePartitionServiceAsyncClient +]) +@mock.patch.object(MetastorePartitionServiceClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(MetastorePartitionServiceClient)) +@mock.patch.object(MetastorePartitionServiceAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(MetastorePartitionServiceAsyncClient)) +def test_metastore_partition_service_client_client_api_endpoint(client_class): + mock_client_cert_source = client_cert_source_callback + api_override = "foo.com" + default_universe = MetastorePartitionServiceClient._DEFAULT_UNIVERSE + default_endpoint = MetastorePartitionServiceClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=default_universe) + mock_universe = "bar.com" + mock_endpoint = MetastorePartitionServiceClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=mock_universe) + + # If ClientOptions.api_endpoint is set and GOOGLE_API_USE_CLIENT_CERTIFICATE="true", + # use ClientOptions.api_endpoint as the api endpoint regardless. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch("google.auth.transport.requests.AuthorizedSession.configure_mtls_channel"): + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=api_override) + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == api_override + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="always", + # use the DEFAULT_MTLS_ENDPOINT as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + + # If ClientOptions.api_endpoint is not set, GOOGLE_API_USE_MTLS_ENDPOINT="auto" (default), + # GOOGLE_API_USE_CLIENT_CERTIFICATE="false" (default), default cert source doesn't exist, + # and ClientOptions.universe_domain="bar.com", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with universe domain as the api endpoint. + options = client_options.ClientOptions() + universe_exists = hasattr(options, "universe_domain") + if universe_exists: + options = client_options.ClientOptions(universe_domain=mock_universe) + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + else: + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == (mock_endpoint if universe_exists else default_endpoint) + assert client.universe_domain == (mock_universe if universe_exists else default_universe) + + # If ClientOptions does not have a universe domain attribute and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + options = client_options.ClientOptions() + if hasattr(options, "universe_domain"): + delattr(options, "universe_domain") + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (MetastorePartitionServiceClient, transports.MetastorePartitionServiceGrpcTransport, "grpc"), + (MetastorePartitionServiceAsyncClient, transports.MetastorePartitionServiceGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_metastore_partition_service_client_client_options_scopes(client_class, transport_class, transport_name): + # Check the case scopes are provided. + options = client_options.ClientOptions( + scopes=["1", "2"], + ) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=["1", "2"], + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,grpc_helpers", [ + (MetastorePartitionServiceClient, transports.MetastorePartitionServiceGrpcTransport, "grpc", grpc_helpers), + (MetastorePartitionServiceAsyncClient, transports.MetastorePartitionServiceGrpcAsyncIOTransport, "grpc_asyncio", grpc_helpers_async), +]) +def test_metastore_partition_service_client_client_options_credentials_file(client_class, transport_class, transport_name, grpc_helpers): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + +def test_metastore_partition_service_client_client_options_from_dict(): + with mock.patch('google.cloud.bigquery_storage_v1alpha.services.metastore_partition_service.transports.MetastorePartitionServiceGrpcTransport.__init__') as grpc_transport: + grpc_transport.return_value = None + client = MetastorePartitionServiceClient( + client_options={'api_endpoint': 'squid.clam.whelk'} + ) + grpc_transport.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + +@pytest.mark.parametrize("client_class,transport_class,transport_name,grpc_helpers", [ + (MetastorePartitionServiceClient, transports.MetastorePartitionServiceGrpcTransport, "grpc", grpc_helpers), + (MetastorePartitionServiceAsyncClient, transports.MetastorePartitionServiceGrpcAsyncIOTransport, "grpc_asyncio", grpc_helpers_async), +]) +def test_metastore_partition_service_client_create_channel_credentials_file(client_class, transport_class, transport_name, grpc_helpers): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=None, + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("request_type", [ + metastore_partition.BatchCreateMetastorePartitionsRequest, + dict, +]) +def test_batch_create_metastore_partitions(request_type, transport: str = 'grpc'): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_create_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = metastore_partition.BatchCreateMetastorePartitionsResponse( + ) + response = client.batch_create_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = metastore_partition.BatchCreateMetastorePartitionsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, metastore_partition.BatchCreateMetastorePartitionsResponse) + + +def test_batch_create_metastore_partitions_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = metastore_partition.BatchCreateMetastorePartitionsRequest( + parent='parent_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_create_metastore_partitions), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.batch_create_metastore_partitions(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == metastore_partition.BatchCreateMetastorePartitionsRequest( + parent='parent_value', + ) + +def test_batch_create_metastore_partitions_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.batch_create_metastore_partitions in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.batch_create_metastore_partitions] = mock_rpc + request = {} + client.batch_create_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.batch_create_metastore_partitions(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_batch_create_metastore_partitions_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.batch_create_metastore_partitions in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.batch_create_metastore_partitions] = mock_rpc + + request = {} + await client.batch_create_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.batch_create_metastore_partitions(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_batch_create_metastore_partitions_async(transport: str = 'grpc_asyncio', request_type=metastore_partition.BatchCreateMetastorePartitionsRequest): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_create_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(metastore_partition.BatchCreateMetastorePartitionsResponse( + )) + response = await client.batch_create_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = metastore_partition.BatchCreateMetastorePartitionsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, metastore_partition.BatchCreateMetastorePartitionsResponse) + + +@pytest.mark.asyncio +async def test_batch_create_metastore_partitions_async_from_dict(): + await test_batch_create_metastore_partitions_async(request_type=dict) + +def test_batch_create_metastore_partitions_field_headers(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = metastore_partition.BatchCreateMetastorePartitionsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_create_metastore_partitions), + '__call__') as call: + call.return_value = metastore_partition.BatchCreateMetastorePartitionsResponse() + client.batch_create_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_batch_create_metastore_partitions_field_headers_async(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = metastore_partition.BatchCreateMetastorePartitionsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_create_metastore_partitions), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(metastore_partition.BatchCreateMetastorePartitionsResponse()) + await client.batch_create_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +@pytest.mark.parametrize("request_type", [ + metastore_partition.BatchDeleteMetastorePartitionsRequest, + dict, +]) +def test_batch_delete_metastore_partitions(request_type, transport: str = 'grpc'): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_delete_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = None + response = client.batch_delete_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = metastore_partition.BatchDeleteMetastorePartitionsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert response is None + + +def test_batch_delete_metastore_partitions_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = metastore_partition.BatchDeleteMetastorePartitionsRequest( + parent='parent_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_delete_metastore_partitions), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.batch_delete_metastore_partitions(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == metastore_partition.BatchDeleteMetastorePartitionsRequest( + parent='parent_value', + ) + +def test_batch_delete_metastore_partitions_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.batch_delete_metastore_partitions in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.batch_delete_metastore_partitions] = mock_rpc + request = {} + client.batch_delete_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.batch_delete_metastore_partitions(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_batch_delete_metastore_partitions_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.batch_delete_metastore_partitions in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.batch_delete_metastore_partitions] = mock_rpc + + request = {} + await client.batch_delete_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.batch_delete_metastore_partitions(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_batch_delete_metastore_partitions_async(transport: str = 'grpc_asyncio', request_type=metastore_partition.BatchDeleteMetastorePartitionsRequest): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_delete_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(None) + response = await client.batch_delete_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = metastore_partition.BatchDeleteMetastorePartitionsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert response is None + + +@pytest.mark.asyncio +async def test_batch_delete_metastore_partitions_async_from_dict(): + await test_batch_delete_metastore_partitions_async(request_type=dict) + +def test_batch_delete_metastore_partitions_field_headers(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = metastore_partition.BatchDeleteMetastorePartitionsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_delete_metastore_partitions), + '__call__') as call: + call.return_value = None + client.batch_delete_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_batch_delete_metastore_partitions_field_headers_async(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = metastore_partition.BatchDeleteMetastorePartitionsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_delete_metastore_partitions), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(None) + await client.batch_delete_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +@pytest.mark.parametrize("request_type", [ + metastore_partition.BatchUpdateMetastorePartitionsRequest, + dict, +]) +def test_batch_update_metastore_partitions(request_type, transport: str = 'grpc'): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_update_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = metastore_partition.BatchUpdateMetastorePartitionsResponse( + ) + response = client.batch_update_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = metastore_partition.BatchUpdateMetastorePartitionsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, metastore_partition.BatchUpdateMetastorePartitionsResponse) + + +def test_batch_update_metastore_partitions_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = metastore_partition.BatchUpdateMetastorePartitionsRequest( + parent='parent_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_update_metastore_partitions), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.batch_update_metastore_partitions(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == metastore_partition.BatchUpdateMetastorePartitionsRequest( + parent='parent_value', + ) + +def test_batch_update_metastore_partitions_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.batch_update_metastore_partitions in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.batch_update_metastore_partitions] = mock_rpc + request = {} + client.batch_update_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.batch_update_metastore_partitions(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_batch_update_metastore_partitions_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.batch_update_metastore_partitions in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.batch_update_metastore_partitions] = mock_rpc + + request = {} + await client.batch_update_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.batch_update_metastore_partitions(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_batch_update_metastore_partitions_async(transport: str = 'grpc_asyncio', request_type=metastore_partition.BatchUpdateMetastorePartitionsRequest): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_update_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(metastore_partition.BatchUpdateMetastorePartitionsResponse( + )) + response = await client.batch_update_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = metastore_partition.BatchUpdateMetastorePartitionsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, metastore_partition.BatchUpdateMetastorePartitionsResponse) + + +@pytest.mark.asyncio +async def test_batch_update_metastore_partitions_async_from_dict(): + await test_batch_update_metastore_partitions_async(request_type=dict) + +def test_batch_update_metastore_partitions_field_headers(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = metastore_partition.BatchUpdateMetastorePartitionsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_update_metastore_partitions), + '__call__') as call: + call.return_value = metastore_partition.BatchUpdateMetastorePartitionsResponse() + client.batch_update_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_batch_update_metastore_partitions_field_headers_async(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = metastore_partition.BatchUpdateMetastorePartitionsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_update_metastore_partitions), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(metastore_partition.BatchUpdateMetastorePartitionsResponse()) + await client.batch_update_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +@pytest.mark.parametrize("request_type", [ + metastore_partition.ListMetastorePartitionsRequest, + dict, +]) +def test_list_metastore_partitions(request_type, transport: str = 'grpc'): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.list_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = metastore_partition.ListMetastorePartitionsResponse( + ) + response = client.list_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = metastore_partition.ListMetastorePartitionsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, metastore_partition.ListMetastorePartitionsResponse) + + +def test_list_metastore_partitions_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = metastore_partition.ListMetastorePartitionsRequest( + parent='parent_value', + filter='filter_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.list_metastore_partitions), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.list_metastore_partitions(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == metastore_partition.ListMetastorePartitionsRequest( + parent='parent_value', + filter='filter_value', + ) + +def test_list_metastore_partitions_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.list_metastore_partitions in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.list_metastore_partitions] = mock_rpc + request = {} + client.list_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.list_metastore_partitions(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_list_metastore_partitions_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.list_metastore_partitions in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.list_metastore_partitions] = mock_rpc + + request = {} + await client.list_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.list_metastore_partitions(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_list_metastore_partitions_async(transport: str = 'grpc_asyncio', request_type=metastore_partition.ListMetastorePartitionsRequest): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.list_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(metastore_partition.ListMetastorePartitionsResponse( + )) + response = await client.list_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = metastore_partition.ListMetastorePartitionsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, metastore_partition.ListMetastorePartitionsResponse) + + +@pytest.mark.asyncio +async def test_list_metastore_partitions_async_from_dict(): + await test_list_metastore_partitions_async(request_type=dict) + +def test_list_metastore_partitions_field_headers(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = metastore_partition.ListMetastorePartitionsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.list_metastore_partitions), + '__call__') as call: + call.return_value = metastore_partition.ListMetastorePartitionsResponse() + client.list_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_list_metastore_partitions_field_headers_async(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = metastore_partition.ListMetastorePartitionsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.list_metastore_partitions), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(metastore_partition.ListMetastorePartitionsResponse()) + await client.list_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +def test_list_metastore_partitions_flattened(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.list_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = metastore_partition.ListMetastorePartitionsResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.list_metastore_partitions( + parent='parent_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + + +def test_list_metastore_partitions_flattened_error(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.list_metastore_partitions( + metastore_partition.ListMetastorePartitionsRequest(), + parent='parent_value', + ) + +@pytest.mark.asyncio +async def test_list_metastore_partitions_flattened_async(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.list_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = metastore_partition.ListMetastorePartitionsResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(metastore_partition.ListMetastorePartitionsResponse()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.list_metastore_partitions( + parent='parent_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + +@pytest.mark.asyncio +async def test_list_metastore_partitions_flattened_error_async(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.list_metastore_partitions( + metastore_partition.ListMetastorePartitionsRequest(), + parent='parent_value', + ) + + +@pytest.mark.parametrize("request_type", [ + metastore_partition.StreamMetastorePartitionsRequest, + dict, +]) +def test_stream_metastore_partitions(request_type, transport: str = 'grpc'): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + requests = [request] + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.stream_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([metastore_partition.StreamMetastorePartitionsResponse()]) + response = client.stream_metastore_partitions(iter(requests)) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert next(args[0]) == request + + # Establish that the response is the type that we expect. + for message in response: + assert isinstance(message, metastore_partition.StreamMetastorePartitionsResponse) + + +def test_stream_metastore_partitions_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.stream_metastore_partitions in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.stream_metastore_partitions] = mock_rpc + request = [{}] + client.stream_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.stream_metastore_partitions(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_stream_metastore_partitions_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.stream_metastore_partitions in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.stream_metastore_partitions] = mock_rpc + + request = [{}] + await client.stream_metastore_partitions(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.stream_metastore_partitions(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_stream_metastore_partitions_async(transport: str = 'grpc_asyncio', request_type=metastore_partition.StreamMetastorePartitionsRequest): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + requests = [request] + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.stream_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = mock.Mock(aio.StreamStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[metastore_partition.StreamMetastorePartitionsResponse()]) + response = await client.stream_metastore_partitions(iter(requests)) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert next(args[0]) == request + + # Establish that the response is the type that we expect. + message = await response.read() + assert isinstance(message, metastore_partition.StreamMetastorePartitionsResponse) + + +@pytest.mark.asyncio +async def test_stream_metastore_partitions_async_from_dict(): + await test_stream_metastore_partitions_async(request_type=dict) + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.MetastorePartitionServiceGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.MetastorePartitionServiceGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = MetastorePartitionServiceClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide an api_key and a transport instance. + transport = transports.MetastorePartitionServiceGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = MetastorePartitionServiceClient( + client_options=options, + transport=transport, + ) + + # It is an error to provide an api_key and a credential. + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = MetastorePartitionServiceClient( + client_options=options, + credentials=ga_credentials.AnonymousCredentials() + ) + + # It is an error to provide scopes and a transport instance. + transport = transports.MetastorePartitionServiceGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = MetastorePartitionServiceClient( + client_options={"scopes": ["1", "2"]}, + transport=transport, + ) + + +def test_transport_instance(): + # A client may be instantiated with a custom transport instance. + transport = transports.MetastorePartitionServiceGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + client = MetastorePartitionServiceClient(transport=transport) + assert client.transport is transport + +def test_transport_get_channel(): + # A client may be instantiated with a custom transport instance. + transport = transports.MetastorePartitionServiceGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + + transport = transports.MetastorePartitionServiceGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + +@pytest.mark.parametrize("transport_class", [ + transports.MetastorePartitionServiceGrpcTransport, + transports.MetastorePartitionServiceGrpcAsyncIOTransport, +]) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + +def test_transport_kind_grpc(): + transport = MetastorePartitionServiceClient.get_transport_class("grpc")( + credentials=ga_credentials.AnonymousCredentials() + ) + assert transport.kind == "grpc" + + +def test_initialize_client_w_grpc(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc" + ) + assert client is not None + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_batch_create_metastore_partitions_empty_call_grpc(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.batch_create_metastore_partitions), + '__call__') as call: + call.return_value = metastore_partition.BatchCreateMetastorePartitionsResponse() + client.batch_create_metastore_partitions(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = metastore_partition.BatchCreateMetastorePartitionsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_batch_delete_metastore_partitions_empty_call_grpc(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.batch_delete_metastore_partitions), + '__call__') as call: + call.return_value = None + client.batch_delete_metastore_partitions(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = metastore_partition.BatchDeleteMetastorePartitionsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_batch_update_metastore_partitions_empty_call_grpc(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.batch_update_metastore_partitions), + '__call__') as call: + call.return_value = metastore_partition.BatchUpdateMetastorePartitionsResponse() + client.batch_update_metastore_partitions(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = metastore_partition.BatchUpdateMetastorePartitionsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_list_metastore_partitions_empty_call_grpc(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.list_metastore_partitions), + '__call__') as call: + call.return_value = metastore_partition.ListMetastorePartitionsResponse() + client.list_metastore_partitions(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = metastore_partition.ListMetastorePartitionsRequest() + + assert args[0] == request_msg + + +def test_transport_kind_grpc_asyncio(): + transport = MetastorePartitionServiceAsyncClient.get_transport_class("grpc_asyncio")( + credentials=async_anonymous_credentials() + ) + assert transport.kind == "grpc_asyncio" + + +def test_initialize_client_w_grpc_asyncio(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio" + ) + assert client is not None + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_batch_create_metastore_partitions_empty_call_grpc_asyncio(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.batch_create_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(metastore_partition.BatchCreateMetastorePartitionsResponse( + )) + await client.batch_create_metastore_partitions(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = metastore_partition.BatchCreateMetastorePartitionsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_batch_delete_metastore_partitions_empty_call_grpc_asyncio(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.batch_delete_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(None) + await client.batch_delete_metastore_partitions(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = metastore_partition.BatchDeleteMetastorePartitionsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_batch_update_metastore_partitions_empty_call_grpc_asyncio(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.batch_update_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(metastore_partition.BatchUpdateMetastorePartitionsResponse( + )) + await client.batch_update_metastore_partitions(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = metastore_partition.BatchUpdateMetastorePartitionsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_list_metastore_partitions_empty_call_grpc_asyncio(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.list_metastore_partitions), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(metastore_partition.ListMetastorePartitionsResponse( + )) + await client.list_metastore_partitions(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = metastore_partition.ListMetastorePartitionsRequest() + + assert args[0] == request_msg + + +def test_transport_grpc_default(): + # A client should use the gRPC transport by default. + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert isinstance( + client.transport, + transports.MetastorePartitionServiceGrpcTransport, + ) + +def test_metastore_partition_service_base_transport_error(): + # Passing both a credentials object and credentials_file should raise an error + with pytest.raises(core_exceptions.DuplicateCredentialArgs): + transport = transports.MetastorePartitionServiceTransport( + credentials=ga_credentials.AnonymousCredentials(), + credentials_file="credentials.json" + ) + + +def test_metastore_partition_service_base_transport(): + # Instantiate the base transport. + with mock.patch('google.cloud.bigquery_storage_v1alpha.services.metastore_partition_service.transports.MetastorePartitionServiceTransport.__init__') as Transport: + Transport.return_value = None + transport = transports.MetastorePartitionServiceTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Every method on the transport should just blindly + # raise NotImplementedError. + methods = ( + 'batch_create_metastore_partitions', + 'batch_delete_metastore_partitions', + 'batch_update_metastore_partitions', + 'list_metastore_partitions', + 'stream_metastore_partitions', + ) + for method in methods: + with pytest.raises(NotImplementedError): + getattr(transport, method)(request=object()) + + with pytest.raises(NotImplementedError): + transport.close() + + # Catch all for all remaining methods and properties + remainder = [ + 'kind', + ] + for r in remainder: + with pytest.raises(NotImplementedError): + getattr(transport, r)() + + +def test_metastore_partition_service_base_transport_with_credentials_file(): + # Instantiate the base transport with a credentials file + with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1alpha.services.metastore_partition_service.transports.MetastorePartitionServiceTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.MetastorePartitionServiceTransport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id="octopus", + ) + + +def test_metastore_partition_service_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(google.auth, 'default', autospec=True) as adc, mock.patch('google.cloud.bigquery_storage_v1alpha.services.metastore_partition_service.transports.MetastorePartitionServiceTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.MetastorePartitionServiceTransport() + adc.assert_called_once() + + +def test_metastore_partition_service_auth_adc(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + MetastorePartitionServiceClient() + adc.assert_called_once_with( + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id=None, + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.MetastorePartitionServiceGrpcTransport, + transports.MetastorePartitionServiceGrpcAsyncIOTransport, + ], +) +def test_metastore_partition_service_transport_auth_adc(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + adc.assert_called_once_with( + scopes=["1", "2"], + default_scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/cloud-platform',), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.MetastorePartitionServiceGrpcTransport, + transports.MetastorePartitionServiceGrpcAsyncIOTransport, + ], +) +def test_metastore_partition_service_transport_auth_gdch_credentials(transport_class): + host = 'https://language.com' + api_audience_tests = [None, 'https://language2.com'] + api_audience_expect = [host, 'https://language2.com'] + for t, e in zip(api_audience_tests, api_audience_expect): + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + gdch_mock = mock.MagicMock() + type(gdch_mock).with_gdch_audience = mock.PropertyMock(return_value=gdch_mock) + adc.return_value = (gdch_mock, None) + transport_class(host=host, api_audience=t) + gdch_mock.with_gdch_audience.assert_called_once_with( + e + ) + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.MetastorePartitionServiceGrpcTransport, grpc_helpers), + (transports.MetastorePartitionServiceGrpcAsyncIOTransport, grpc_helpers_async) + ], +) +def test_metastore_partition_service_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class( + quota_project_id="octopus", + scopes=["1", "2"] + ) + + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=["1", "2"], + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("transport_class", [transports.MetastorePartitionServiceGrpcTransport, transports.MetastorePartitionServiceGrpcAsyncIOTransport]) +def test_metastore_partition_service_grpc_transport_client_cert_source_for_mtls( + transport_class +): + cred = ga_credentials.AnonymousCredentials() + + # Check ssl_channel_credentials is used if provided. + with mock.patch.object(transport_class, "create_channel") as mock_create_channel: + mock_ssl_channel_creds = mock.Mock() + transport_class( + host="squid.clam.whelk", + credentials=cred, + ssl_channel_credentials=mock_ssl_channel_creds + ) + mock_create_channel.assert_called_once_with( + "squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_channel_creds, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls + # is used. + with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): + with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: + transport_class( + credentials=cred, + client_cert_source_for_mtls=client_cert_source_callback + ) + expected_cert, expected_key = client_cert_source_callback() + mock_ssl_cred.assert_called_once_with( + certificate_chain=expected_cert, + private_key=expected_key + ) + + +@pytest.mark.parametrize("transport_name", [ + "grpc", + "grpc_asyncio", +]) +def test_metastore_partition_service_host_no_port(transport_name): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com'), + transport=transport_name, + ) + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + +@pytest.mark.parametrize("transport_name", [ + "grpc", + "grpc_asyncio", +]) +def test_metastore_partition_service_host_with_port(transport_name): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com:8000'), + transport=transport_name, + ) + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:8000' + ) + +def test_metastore_partition_service_grpc_transport_channel(): + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.MetastorePartitionServiceGrpcTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +def test_metastore_partition_service_grpc_asyncio_transport_channel(): + channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.MetastorePartitionServiceGrpcAsyncIOTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.MetastorePartitionServiceGrpcTransport, transports.MetastorePartitionServiceGrpcAsyncIOTransport]) +def test_metastore_partition_service_transport_channel_mtls_with_client_cert_source( + transport_class +): + with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred: + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_ssl_cred = mock.Mock() + grpc_ssl_channel_cred.return_value = mock_ssl_cred + + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + + cred = ga_credentials.AnonymousCredentials() + with pytest.warns(DeprecationWarning): + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (cred, None) + transport = transport_class( + host="squid.clam.whelk", + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=client_cert_source_callback, + ) + adc.assert_called_once() + + grpc_ssl_channel_cred.assert_called_once_with( + certificate_chain=b"cert bytes", private_key=b"key bytes" + ) + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + assert transport._ssl_channel_credentials == mock_ssl_cred + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.MetastorePartitionServiceGrpcTransport, transports.MetastorePartitionServiceGrpcAsyncIOTransport]) +def test_metastore_partition_service_transport_channel_mtls_with_adc( + transport_class +): + mock_ssl_cred = mock.Mock() + with mock.patch.multiple( + "google.auth.transport.grpc.SslCredentials", + __init__=mock.Mock(return_value=None), + ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), + ): + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + mock_cred = mock.Mock() + + with pytest.warns(DeprecationWarning): + transport = transport_class( + host="squid.clam.whelk", + credentials=mock_cred, + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=None, + ) + + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=mock_cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + + +def test_read_stream_path(): + project = "squid" + location = "clam" + session = "whelk" + stream = "octopus" + expected = "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) + actual = MetastorePartitionServiceClient.read_stream_path(project, location, session, stream) + assert expected == actual + + +def test_parse_read_stream_path(): + expected = { + "project": "oyster", + "location": "nudibranch", + "session": "cuttlefish", + "stream": "mussel", + } + path = MetastorePartitionServiceClient.read_stream_path(**expected) + + # Check that the path construction is reversible. + actual = MetastorePartitionServiceClient.parse_read_stream_path(path) + assert expected == actual + +def test_table_path(): + project = "winkle" + dataset = "nautilus" + table = "scallop" + expected = "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + actual = MetastorePartitionServiceClient.table_path(project, dataset, table) + assert expected == actual + + +def test_parse_table_path(): + expected = { + "project": "abalone", + "dataset": "squid", + "table": "clam", + } + path = MetastorePartitionServiceClient.table_path(**expected) + + # Check that the path construction is reversible. + actual = MetastorePartitionServiceClient.parse_table_path(path) + assert expected == actual + +def test_common_billing_account_path(): + billing_account = "whelk" + expected = "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + actual = MetastorePartitionServiceClient.common_billing_account_path(billing_account) + assert expected == actual + + +def test_parse_common_billing_account_path(): + expected = { + "billing_account": "octopus", + } + path = MetastorePartitionServiceClient.common_billing_account_path(**expected) + + # Check that the path construction is reversible. + actual = MetastorePartitionServiceClient.parse_common_billing_account_path(path) + assert expected == actual + +def test_common_folder_path(): + folder = "oyster" + expected = "folders/{folder}".format(folder=folder, ) + actual = MetastorePartitionServiceClient.common_folder_path(folder) + assert expected == actual + + +def test_parse_common_folder_path(): + expected = { + "folder": "nudibranch", + } + path = MetastorePartitionServiceClient.common_folder_path(**expected) + + # Check that the path construction is reversible. + actual = MetastorePartitionServiceClient.parse_common_folder_path(path) + assert expected == actual + +def test_common_organization_path(): + organization = "cuttlefish" + expected = "organizations/{organization}".format(organization=organization, ) + actual = MetastorePartitionServiceClient.common_organization_path(organization) + assert expected == actual + + +def test_parse_common_organization_path(): + expected = { + "organization": "mussel", + } + path = MetastorePartitionServiceClient.common_organization_path(**expected) + + # Check that the path construction is reversible. + actual = MetastorePartitionServiceClient.parse_common_organization_path(path) + assert expected == actual + +def test_common_project_path(): + project = "winkle" + expected = "projects/{project}".format(project=project, ) + actual = MetastorePartitionServiceClient.common_project_path(project) + assert expected == actual + + +def test_parse_common_project_path(): + expected = { + "project": "nautilus", + } + path = MetastorePartitionServiceClient.common_project_path(**expected) + + # Check that the path construction is reversible. + actual = MetastorePartitionServiceClient.parse_common_project_path(path) + assert expected == actual + +def test_common_location_path(): + project = "scallop" + location = "abalone" + expected = "projects/{project}/locations/{location}".format(project=project, location=location, ) + actual = MetastorePartitionServiceClient.common_location_path(project, location) + assert expected == actual + + +def test_parse_common_location_path(): + expected = { + "project": "squid", + "location": "clam", + } + path = MetastorePartitionServiceClient.common_location_path(**expected) + + # Check that the path construction is reversible. + actual = MetastorePartitionServiceClient.parse_common_location_path(path) + assert expected == actual + + +def test_client_with_default_client_info(): + client_info = gapic_v1.client_info.ClientInfo() + + with mock.patch.object(transports.MetastorePartitionServiceTransport, '_prep_wrapped_messages') as prep: + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + with mock.patch.object(transports.MetastorePartitionServiceTransport, '_prep_wrapped_messages') as prep: + transport_class = MetastorePartitionServiceClient.get_transport_class() + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + +def test_transport_close_grpc(): + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc" + ) + with mock.patch.object(type(getattr(client.transport, "_grpc_channel")), "close") as close: + with client: + close.assert_not_called() + close.assert_called_once() + + +@pytest.mark.asyncio +async def test_transport_close_grpc_asyncio(): + client = MetastorePartitionServiceAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio" + ) + with mock.patch.object(type(getattr(client.transport, "_grpc_channel")), "close") as close: + async with client: + close.assert_not_called() + close.assert_called_once() + + +def test_client_ctx(): + transports = [ + 'grpc', + ] + for transport in transports: + client = MetastorePartitionServiceClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport + ) + # Test client calls underlying transport. + with mock.patch.object(type(client.transport), "close") as close: + close.assert_not_called() + with client: + pass + close.assert_called() + +@pytest.mark.parametrize("client_class,transport_class", [ + (MetastorePartitionServiceClient, transports.MetastorePartitionServiceGrpcTransport), + (MetastorePartitionServiceAsyncClient, transports.MetastorePartitionServiceGrpcAsyncIOTransport), +]) +def test_api_key_credentials(client_class, transport_class): + with mock.patch.object( + google.auth._default, "get_api_key_credentials", create=True + ) as get_api_key_credentials: + mock_cred = mock.Mock() + get_api_key_credentials.return_value = mock_cred + options = client_options.ClientOptions() + options.api_key = "api_key" + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=mock_cred, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) diff --git a/owl-bot-staging/v1beta2/.coveragerc b/owl-bot-staging/v1beta2/.coveragerc new file mode 100644 index 00000000..23d2208d --- /dev/null +++ b/owl-bot-staging/v1beta2/.coveragerc @@ -0,0 +1,13 @@ +[run] +branch = True + +[report] +show_missing = True +omit = + google/cloud/bigquery_storage/__init__.py + google/cloud/bigquery_storage/gapic_version.py +exclude_lines = + # Re-enable the standard pragma + pragma: NO COVER + # Ignore debug-only repr + def __repr__ diff --git a/owl-bot-staging/v1beta2/.flake8 b/owl-bot-staging/v1beta2/.flake8 new file mode 100644 index 00000000..29227d4c --- /dev/null +++ b/owl-bot-staging/v1beta2/.flake8 @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Generated by synthtool. DO NOT EDIT! +[flake8] +ignore = E203, E266, E501, W503 +exclude = + # Exclude generated code. + **/proto/** + **/gapic/** + **/services/** + **/types/** + *_pb2.py + + # Standard linting exemptions. + **/.nox/** + __pycache__, + .git, + *.pyc, + conf.py diff --git a/owl-bot-staging/v1beta2/MANIFEST.in b/owl-bot-staging/v1beta2/MANIFEST.in new file mode 100644 index 00000000..6b94fe3e --- /dev/null +++ b/owl-bot-staging/v1beta2/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include google/cloud/bigquery_storage *.py +recursive-include google/cloud/bigquery_storage_v1beta2 *.py diff --git a/owl-bot-staging/v1beta2/README.rst b/owl-bot-staging/v1beta2/README.rst new file mode 100644 index 00000000..13a45c14 --- /dev/null +++ b/owl-bot-staging/v1beta2/README.rst @@ -0,0 +1,49 @@ +Python Client for Google Cloud Bigquery Storage API +================================================= + +Quick Start +----------- + +In order to use this library, you first need to go through the following steps: + +1. `Select or create a Cloud Platform project.`_ +2. `Enable billing for your project.`_ +3. Enable the Google Cloud Bigquery Storage API. +4. `Setup Authentication.`_ + +.. _Select or create a Cloud Platform project.: https://console.cloud.google.com/project +.. _Enable billing for your project.: https://cloud.google.com/billing/docs/how-to/modify-project#enable_billing_for_a_project +.. _Setup Authentication.: https://googleapis.dev/python/google-api-core/latest/auth.html + +Installation +~~~~~~~~~~~~ + +Install this library in a `virtualenv`_ using pip. `virtualenv`_ is a tool to +create isolated Python environments. The basic problem it addresses is one of +dependencies and versions, and indirectly permissions. + +With `virtualenv`_, it's possible to install this library without needing system +install permissions, and without clashing with the installed system +dependencies. + +.. _`virtualenv`: https://virtualenv.pypa.io/en/latest/ + + +Mac/Linux +^^^^^^^^^ + +.. code-block:: console + + python3 -m venv + source /bin/activate + /bin/pip install /path/to/library + + +Windows +^^^^^^^ + +.. code-block:: console + + python3 -m venv + \Scripts\activate + \Scripts\pip.exe install \path\to\library diff --git a/owl-bot-staging/v1beta2/docs/_static/custom.css b/owl-bot-staging/v1beta2/docs/_static/custom.css new file mode 100644 index 00000000..06423be0 --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/_static/custom.css @@ -0,0 +1,3 @@ +dl.field-list > dt { + min-width: 100px +} diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_read.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_read.rst new file mode 100644 index 00000000..10b3714b --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_read.rst @@ -0,0 +1,6 @@ +BigQueryRead +------------------------------ + +.. automodule:: google.cloud.bigquery_storage_v1beta2.services.big_query_read + :members: + :inherited-members: diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_write.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_write.rst new file mode 100644 index 00000000..c685994d --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/big_query_write.rst @@ -0,0 +1,6 @@ +BigQueryWrite +------------------------------- + +.. automodule:: google.cloud.bigquery_storage_v1beta2.services.big_query_write + :members: + :inherited-members: diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/services_.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/services_.rst new file mode 100644 index 00000000..67dae0ab --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/services_.rst @@ -0,0 +1,7 @@ +Services for Google Cloud Bigquery Storage v1beta2 API +====================================================== +.. toctree:: + :maxdepth: 2 + + big_query_read + big_query_write diff --git a/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/types_.rst b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/types_.rst new file mode 100644 index 00000000..c70a8cf3 --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/bigquery_storage_v1beta2/types_.rst @@ -0,0 +1,6 @@ +Types for Google Cloud Bigquery Storage v1beta2 API +=================================================== + +.. automodule:: google.cloud.bigquery_storage_v1beta2.types + :members: + :show-inheritance: diff --git a/owl-bot-staging/v1beta2/docs/conf.py b/owl-bot-staging/v1beta2/docs/conf.py new file mode 100644 index 00000000..33c4ea53 --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/conf.py @@ -0,0 +1,376 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# google-cloud-bigquery-storage documentation build configuration file +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath("..")) + +__version__ = "0.1.0" + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +needs_sphinx = "4.0.1" + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.autosummary", + "sphinx.ext.intersphinx", + "sphinx.ext.coverage", + "sphinx.ext.napoleon", + "sphinx.ext.todo", + "sphinx.ext.viewcode", +] + +# autodoc/autosummary flags +autoclass_content = "both" +autodoc_default_flags = ["members"] +autosummary_generate = True + + +# Add any paths that contain templates here, relative to this directory. +templates_path = ["_templates"] + +# Allow markdown includes (so releases.md can include CHANGLEOG.md) +# http://www.sphinx-doc.org/en/master/markdown.html +source_parsers = {".md": "recommonmark.parser.CommonMarkParser"} + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +source_suffix = [".rst", ".md"] + +# The encoding of source files. +# source_encoding = 'utf-8-sig' + +# The root toctree document. +root_doc = "index" + +# General information about the project. +project = u"google-cloud-bigquery-storage" +copyright = u"2023, Google, LLC" +author = u"Google APIs" # TODO: autogenerate this bit + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The full version, including alpha/beta/rc tags. +release = __version__ +# The short X.Y version. +version = ".".join(release.split(".")[0:2]) + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +# today = '' +# Else, today_fmt is used as the format for a strftime call. +# today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ["_build"] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +# default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +# add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +# add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +# show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = "sphinx" + +# A list of ignored prefixes for module index sorting. +# modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +# keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = "alabaster" + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + "description": "Google Cloud Client Libraries for Python", + "github_user": "googleapis", + "github_repo": "google-cloud-python", + "github_banner": True, + "font_family": "'Roboto', Georgia, sans", + "head_font_family": "'Roboto', Georgia, serif", + "code_font_family": "'Roboto Mono', 'Consolas', monospace", +} + +# Add any paths that contain custom themes here, relative to this directory. +# html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +# html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +# html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +# html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +# html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ["_static"] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +# html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +# html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +# html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +# html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +# html_additional_pages = {} + +# If false, no module index is generated. +# html_domain_indices = True + +# If false, no index is generated. +# html_use_index = True + +# If true, the index is split into individual pages for each letter. +# html_split_index = False + +# If true, links to the reST sources are added to the pages. +# html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +# html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +# html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +# html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +# html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +# html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +# html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +# html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = "google-cloud-bigquery-storage-doc" + +# -- Options for warnings ------------------------------------------------------ + + +suppress_warnings = [ + # Temporarily suppress this to avoid "more than one target found for + # cross-reference" warning, which are intractable for us to avoid while in + # a mono-repo. + # See https://github.com/sphinx-doc/sphinx/blob + # /2a65ffeef5c107c19084fabdd706cdff3f52d93c/sphinx/domains/python.py#L843 + "ref.python" +] + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', + # Latex figure (float) alignment + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + ( + root_doc, + "google-cloud-bigquery-storage.tex", + u"google-cloud-bigquery-storage Documentation", + author, + "manual", + ) +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +# latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +# latex_use_parts = False + +# If true, show page references after internal links. +# latex_show_pagerefs = False + +# If true, show URL addresses after external links. +# latex_show_urls = False + +# Documents to append as an appendix to all manuals. +# latex_appendices = [] + +# If false, no module index is generated. +# latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + ( + root_doc, + "google-cloud-bigquery-storage", + u"Google Cloud Bigquery Storage Documentation", + [author], + 1, + ) +] + +# If true, show URL addresses after external links. +# man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + ( + root_doc, + "google-cloud-bigquery-storage", + u"google-cloud-bigquery-storage Documentation", + author, + "google-cloud-bigquery-storage", + "GAPIC library for Google Cloud Bigquery Storage API", + "APIs", + ) +] + +# Documents to append as an appendix to all manuals. +# texinfo_appendices = [] + +# If false, no module index is generated. +# texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +# texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +# texinfo_no_detailmenu = False + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + "python": ("http://python.readthedocs.org/en/latest/", None), + "gax": ("https://gax-python.readthedocs.org/en/latest/", None), + "google-auth": ("https://google-auth.readthedocs.io/en/stable", None), + "google-gax": ("https://gax-python.readthedocs.io/en/latest/", None), + "google.api_core": ("https://googleapis.dev/python/google-api-core/latest/", None), + "grpc": ("https://grpc.io/grpc/python/", None), + "requests": ("http://requests.kennethreitz.org/en/stable/", None), + "proto": ("https://proto-plus-python.readthedocs.io/en/stable", None), + "protobuf": ("https://googleapis.dev/python/protobuf/latest/", None), +} + + +# Napoleon settings +napoleon_google_docstring = True +napoleon_numpy_docstring = True +napoleon_include_private_with_doc = False +napoleon_include_special_with_doc = True +napoleon_use_admonition_for_examples = False +napoleon_use_admonition_for_notes = False +napoleon_use_admonition_for_references = False +napoleon_use_ivar = False +napoleon_use_param = True +napoleon_use_rtype = True diff --git a/owl-bot-staging/v1beta2/docs/index.rst b/owl-bot-staging/v1beta2/docs/index.rst new file mode 100644 index 00000000..ef2e07e2 --- /dev/null +++ b/owl-bot-staging/v1beta2/docs/index.rst @@ -0,0 +1,7 @@ +API Reference +------------- +.. toctree:: + :maxdepth: 2 + + bigquery_storage_v1beta2/services + bigquery_storage_v1beta2/types diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/__init__.py new file mode 100644 index 00000000..4e42782e --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/__init__.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from google.cloud.bigquery_storage import gapic_version as package_version + +__version__ = package_version.__version__ + + +from google.cloud.bigquery_storage_v1beta2.services.big_query_read.client import BigQueryReadClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_read.async_client import BigQueryReadAsyncClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_write.client import BigQueryWriteClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_write.async_client import BigQueryWriteAsyncClient + +from google.cloud.bigquery_storage_v1beta2.types.arrow import ArrowRecordBatch +from google.cloud.bigquery_storage_v1beta2.types.arrow import ArrowSchema +from google.cloud.bigquery_storage_v1beta2.types.arrow import ArrowSerializationOptions +from google.cloud.bigquery_storage_v1beta2.types.avro import AvroRows +from google.cloud.bigquery_storage_v1beta2.types.avro import AvroSchema +from google.cloud.bigquery_storage_v1beta2.types.protobuf import ProtoRows +from google.cloud.bigquery_storage_v1beta2.types.protobuf import ProtoSchema +from google.cloud.bigquery_storage_v1beta2.types.storage import AppendRowsRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import AppendRowsResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import BatchCommitWriteStreamsRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import BatchCommitWriteStreamsResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import CreateReadSessionRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import CreateWriteStreamRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import FinalizeWriteStreamRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import FinalizeWriteStreamResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import FlushRowsRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import FlushRowsResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import GetWriteStreamRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import ReadRowsRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import ReadRowsResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import SplitReadStreamRequest +from google.cloud.bigquery_storage_v1beta2.types.storage import SplitReadStreamResponse +from google.cloud.bigquery_storage_v1beta2.types.storage import StorageError +from google.cloud.bigquery_storage_v1beta2.types.storage import StreamStats +from google.cloud.bigquery_storage_v1beta2.types.storage import ThrottleState +from google.cloud.bigquery_storage_v1beta2.types.stream import ReadSession +from google.cloud.bigquery_storage_v1beta2.types.stream import ReadStream +from google.cloud.bigquery_storage_v1beta2.types.stream import WriteStream +from google.cloud.bigquery_storage_v1beta2.types.stream import DataFormat +from google.cloud.bigquery_storage_v1beta2.types.table import TableFieldSchema +from google.cloud.bigquery_storage_v1beta2.types.table import TableSchema + +__all__ = ('BigQueryReadClient', + 'BigQueryReadAsyncClient', + 'BigQueryWriteClient', + 'BigQueryWriteAsyncClient', + 'ArrowRecordBatch', + 'ArrowSchema', + 'ArrowSerializationOptions', + 'AvroRows', + 'AvroSchema', + 'ProtoRows', + 'ProtoSchema', + 'AppendRowsRequest', + 'AppendRowsResponse', + 'BatchCommitWriteStreamsRequest', + 'BatchCommitWriteStreamsResponse', + 'CreateReadSessionRequest', + 'CreateWriteStreamRequest', + 'FinalizeWriteStreamRequest', + 'FinalizeWriteStreamResponse', + 'FlushRowsRequest', + 'FlushRowsResponse', + 'GetWriteStreamRequest', + 'ReadRowsRequest', + 'ReadRowsResponse', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + 'StorageError', + 'StreamStats', + 'ThrottleState', + 'ReadSession', + 'ReadStream', + 'WriteStream', + 'DataFormat', + 'TableFieldSchema', + 'TableSchema', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/gapic_version.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/gapic_version.py new file mode 100644 index 00000000..558c8aab --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/gapic_version.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__version__ = "0.0.0" # {x-release-please-version} diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/py.typed b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/py.typed new file mode 100644 index 00000000..e71b4749 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/__init__.py new file mode 100644 index 00000000..270f5a22 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/__init__.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from google.cloud.bigquery_storage_v1beta2 import gapic_version as package_version + +__version__ = package_version.__version__ + + +from .services.big_query_read import BigQueryReadClient +from .services.big_query_read import BigQueryReadAsyncClient +from .services.big_query_write import BigQueryWriteClient +from .services.big_query_write import BigQueryWriteAsyncClient + +from .types.arrow import ArrowRecordBatch +from .types.arrow import ArrowSchema +from .types.arrow import ArrowSerializationOptions +from .types.avro import AvroRows +from .types.avro import AvroSchema +from .types.protobuf import ProtoRows +from .types.protobuf import ProtoSchema +from .types.storage import AppendRowsRequest +from .types.storage import AppendRowsResponse +from .types.storage import BatchCommitWriteStreamsRequest +from .types.storage import BatchCommitWriteStreamsResponse +from .types.storage import CreateReadSessionRequest +from .types.storage import CreateWriteStreamRequest +from .types.storage import FinalizeWriteStreamRequest +from .types.storage import FinalizeWriteStreamResponse +from .types.storage import FlushRowsRequest +from .types.storage import FlushRowsResponse +from .types.storage import GetWriteStreamRequest +from .types.storage import ReadRowsRequest +from .types.storage import ReadRowsResponse +from .types.storage import SplitReadStreamRequest +from .types.storage import SplitReadStreamResponse +from .types.storage import StorageError +from .types.storage import StreamStats +from .types.storage import ThrottleState +from .types.stream import ReadSession +from .types.stream import ReadStream +from .types.stream import WriteStream +from .types.stream import DataFormat +from .types.table import TableFieldSchema +from .types.table import TableSchema + +__all__ = ( + 'BigQueryReadAsyncClient', + 'BigQueryWriteAsyncClient', +'AppendRowsRequest', +'AppendRowsResponse', +'ArrowRecordBatch', +'ArrowSchema', +'ArrowSerializationOptions', +'AvroRows', +'AvroSchema', +'BatchCommitWriteStreamsRequest', +'BatchCommitWriteStreamsResponse', +'BigQueryReadClient', +'BigQueryWriteClient', +'CreateReadSessionRequest', +'CreateWriteStreamRequest', +'DataFormat', +'FinalizeWriteStreamRequest', +'FinalizeWriteStreamResponse', +'FlushRowsRequest', +'FlushRowsResponse', +'GetWriteStreamRequest', +'ProtoRows', +'ProtoSchema', +'ReadRowsRequest', +'ReadRowsResponse', +'ReadSession', +'ReadStream', +'SplitReadStreamRequest', +'SplitReadStreamResponse', +'StorageError', +'StreamStats', +'TableFieldSchema', +'TableSchema', +'ThrottleState', +'WriteStream', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_metadata.json b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_metadata.json new file mode 100644 index 00000000..5aba90d6 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_metadata.json @@ -0,0 +1,127 @@ + { + "comment": "This file maps proto services/RPCs to the corresponding library clients/methods", + "language": "python", + "libraryPackage": "google.cloud.bigquery_storage_v1beta2", + "protoPackage": "google.cloud.bigquery.storage.v1beta2", + "schema": "1.0", + "services": { + "BigQueryRead": { + "clients": { + "grpc": { + "libraryClient": "BigQueryReadClient", + "rpcs": { + "CreateReadSession": { + "methods": [ + "create_read_session" + ] + }, + "ReadRows": { + "methods": [ + "read_rows" + ] + }, + "SplitReadStream": { + "methods": [ + "split_read_stream" + ] + } + } + }, + "grpc-async": { + "libraryClient": "BigQueryReadAsyncClient", + "rpcs": { + "CreateReadSession": { + "methods": [ + "create_read_session" + ] + }, + "ReadRows": { + "methods": [ + "read_rows" + ] + }, + "SplitReadStream": { + "methods": [ + "split_read_stream" + ] + } + } + } + } + }, + "BigQueryWrite": { + "clients": { + "grpc": { + "libraryClient": "BigQueryWriteClient", + "rpcs": { + "AppendRows": { + "methods": [ + "append_rows" + ] + }, + "BatchCommitWriteStreams": { + "methods": [ + "batch_commit_write_streams" + ] + }, + "CreateWriteStream": { + "methods": [ + "create_write_stream" + ] + }, + "FinalizeWriteStream": { + "methods": [ + "finalize_write_stream" + ] + }, + "FlushRows": { + "methods": [ + "flush_rows" + ] + }, + "GetWriteStream": { + "methods": [ + "get_write_stream" + ] + } + } + }, + "grpc-async": { + "libraryClient": "BigQueryWriteAsyncClient", + "rpcs": { + "AppendRows": { + "methods": [ + "append_rows" + ] + }, + "BatchCommitWriteStreams": { + "methods": [ + "batch_commit_write_streams" + ] + }, + "CreateWriteStream": { + "methods": [ + "create_write_stream" + ] + }, + "FinalizeWriteStream": { + "methods": [ + "finalize_write_stream" + ] + }, + "FlushRows": { + "methods": [ + "flush_rows" + ] + }, + "GetWriteStream": { + "methods": [ + "get_write_stream" + ] + } + } + } + } + } + } +} diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_version.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_version.py new file mode 100644 index 00000000..558c8aab --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/gapic_version.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +__version__ = "0.0.0" # {x-release-please-version} diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/py.typed b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/py.typed new file mode 100644 index 00000000..e71b4749 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/py.typed @@ -0,0 +1,2 @@ +# Marker file for PEP 561. +# The google-cloud-bigquery-storage package uses inline types. diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/__init__.py new file mode 100644 index 00000000..8f6cf068 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/__init__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/__init__.py new file mode 100644 index 00000000..194280aa --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .client import BigQueryReadClient +from .async_client import BigQueryReadAsyncClient + +__all__ = ( + 'BigQueryReadClient', + 'BigQueryReadAsyncClient', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py new file mode 100644 index 00000000..ce998e76 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/async_client.py @@ -0,0 +1,616 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import re +from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, AsyncIterable, Awaitable, Sequence, Tuple, Type, Union + +from google.cloud.bigquery_storage_v1beta2 import gapic_version as package_version + +from google.api_core.client_options import ClientOptions +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry_async as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + + +try: + OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.AsyncRetry, object, None] # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import arrow +from google.cloud.bigquery_storage_v1beta2.types import avro +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.protobuf import timestamp_pb2 # type: ignore +from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport +from .client import BigQueryReadClient + + +class BigQueryReadAsyncClient: + """BigQuery Read API. + + The Read API can be used to read data from BigQuery. + + New code should use the v1 Read API going forward, if they don't + use Write API at the same time. + """ + + _client: BigQueryReadClient + + # Copy defaults from the synchronous client for use here. + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + DEFAULT_ENDPOINT = BigQueryReadClient.DEFAULT_ENDPOINT + DEFAULT_MTLS_ENDPOINT = BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + _DEFAULT_ENDPOINT_TEMPLATE = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE + _DEFAULT_UNIVERSE = BigQueryReadClient._DEFAULT_UNIVERSE + + read_session_path = staticmethod(BigQueryReadClient.read_session_path) + parse_read_session_path = staticmethod(BigQueryReadClient.parse_read_session_path) + read_stream_path = staticmethod(BigQueryReadClient.read_stream_path) + parse_read_stream_path = staticmethod(BigQueryReadClient.parse_read_stream_path) + table_path = staticmethod(BigQueryReadClient.table_path) + parse_table_path = staticmethod(BigQueryReadClient.parse_table_path) + common_billing_account_path = staticmethod(BigQueryReadClient.common_billing_account_path) + parse_common_billing_account_path = staticmethod(BigQueryReadClient.parse_common_billing_account_path) + common_folder_path = staticmethod(BigQueryReadClient.common_folder_path) + parse_common_folder_path = staticmethod(BigQueryReadClient.parse_common_folder_path) + common_organization_path = staticmethod(BigQueryReadClient.common_organization_path) + parse_common_organization_path = staticmethod(BigQueryReadClient.parse_common_organization_path) + common_project_path = staticmethod(BigQueryReadClient.common_project_path) + parse_common_project_path = staticmethod(BigQueryReadClient.parse_common_project_path) + common_location_path = staticmethod(BigQueryReadClient.common_location_path) + parse_common_location_path = staticmethod(BigQueryReadClient.parse_common_location_path) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_info.__func__(BigQueryReadAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadAsyncClient: The constructed client. + """ + return BigQueryReadClient.from_service_account_file.__func__(BigQueryReadAsyncClient, filename, *args, **kwargs) # type: ignore + + from_service_account_json = from_service_account_file + + @classmethod + def get_mtls_endpoint_and_cert_source(cls, client_options: Optional[ClientOptions] = None): + """Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variable is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + return BigQueryReadClient.get_mtls_endpoint_and_cert_source(client_options) # type: ignore + + @property + def transport(self) -> BigQueryReadTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryReadTransport: The transport used by the client instance. + """ + return self._client.transport + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._client._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used + by the client instance. + """ + return self._client._universe_domain + + get_transport_class = BigQueryReadClient.get_transport_class + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Optional[Union[str, BigQueryReadTransport, Callable[..., BigQueryReadTransport]]] = "grpc_asyncio", + client_options: Optional[ClientOptions] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query read async client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Optional[Union[str,BigQueryReadTransport,Callable[..., BigQueryReadTransport]]]): + The transport to use, or a Callable that constructs and returns a new transport to use. + If a Callable is given, it will be called with the same set of initialization + arguments as used in the BigQueryReadTransport constructor. + If set to None, a transport is chosen automatically. + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide a client certificate for mTLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client = BigQueryReadClient( + credentials=credentials, + transport=transport, + client_options=client_options, + client_info=client_info, + + ) + + async def create_read_session(self, + request: Optional[Union[storage.CreateReadSessionRequest, dict]] = None, + *, + parent: Optional[str] = None, + read_session: Optional[stream.ReadSession] = None, + max_stream_count: Optional[int] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.ReadSession: + r"""Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a + push-down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + async def sample_create_read_session(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.CreateReadSessionRequest( + parent="parent_value", + ) + + # Make the request + response = await client.create_read_session(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1beta2.types.CreateReadSessionRequest, dict]]): + The request object. Request message for ``CreateReadSession``. + parent (:class:`str`): + Required. The request project that owns the session, in + the form of ``projects/{project_id}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + read_session (:class:`google.cloud.bigquery_storage_v1beta2.types.ReadSession`): + Required. Session to be created. + This corresponds to the ``read_session`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + max_stream_count (:class:`int`): + Max initial number of streams. If + unset or zero, the server will provide a + value of streams so as to produce + reasonable throughput. Must be + non-negative. The number of streams may + be lower than the requested number, + depending on the amount parallelism that + is reasonable for the table. Error will + be returned if the max count is greater + than the current system max limit of + 1,000. + + Streams must be read starting from + offset 0. + + This corresponds to the ``max_stream_count`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.ReadSession: + Information about the ReadSession. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, read_session, max_stream_count]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.CreateReadSessionRequest): + request = storage.CreateReadSessionRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if read_session is not None: + request.read_session = read_session + if max_stream_count is not None: + request.max_stream_count = max_stream_count + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.create_read_session] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_session.table", request.read_session.table), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def read_rows(self, + request: Optional[Union[storage.ReadRowsRequest, dict]] = None, + *, + read_stream: Optional[str] = None, + offset: Optional[int] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Awaitable[AsyncIterable[storage.ReadRowsResponse]]: + r"""Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + async def sample_read_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.ReadRowsRequest( + read_stream="read_stream_value", + ) + + # Make the request + stream = await client.read_rows(request=request) + + # Handle the response + async for response in stream: + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1beta2.types.ReadRowsRequest, dict]]): + The request object. Request message for ``ReadRows``. + read_stream (:class:`str`): + Required. Stream to read rows from. + This corresponds to the ``read_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + offset (:class:`int`): + The offset requested must be less + than the last row read from Read. + Requesting a larger offset is undefined. + If not specified, start reading from + offset zero. + + This corresponds to the ``offset`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + AsyncIterable[google.cloud.bigquery_storage_v1beta2.types.ReadRowsResponse]: + Response from calling ReadRows may include row data, progress and + throttling information. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([read_stream, offset]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.ReadRowsRequest): + request = storage.ReadRowsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if read_stream is not None: + request.read_stream = read_stream + if offset is not None: + request.offset = offset + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.read_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_stream", request.read_stream), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def split_read_stream(self, + request: Optional[Union[storage.SplitReadStreamRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.SplitReadStreamResponse: + r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + async def sample_split_read_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.SplitReadStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.split_read_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamRequest, dict]]): + The request object. Request message for ``SplitReadStream``. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamResponse: + + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.SplitReadStreamRequest): + request = storage.SplitReadStreamRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.split_read_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def __aenter__(self) -> "BigQueryReadAsyncClient": + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.transport.close() + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +__all__ = ( + "BigQueryReadAsyncClient", +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py new file mode 100644 index 00000000..9d59f4f8 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/client.py @@ -0,0 +1,987 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import os +import re +from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, Iterable, Sequence, Tuple, Type, Union, cast +import warnings + +from google.cloud.bigquery_storage_v1beta2 import gapic_version as package_version + +from google.api_core import client_options as client_options_lib +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport import mtls # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.oauth2 import service_account # type: ignore + +try: + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.Retry, object, None] # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import arrow +from google.cloud.bigquery_storage_v1beta2.types import avro +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.protobuf import timestamp_pb2 # type: ignore +from .transports.base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .transports.grpc import BigQueryReadGrpcTransport +from .transports.grpc_asyncio import BigQueryReadGrpcAsyncIOTransport + + +class BigQueryReadClientMeta(type): + """Metaclass for the BigQueryRead client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + _transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] + _transport_registry["grpc"] = BigQueryReadGrpcTransport + _transport_registry["grpc_asyncio"] = BigQueryReadGrpcAsyncIOTransport + + def get_transport_class(cls, + label: Optional[str] = None, + ) -> Type[BigQueryReadTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class BigQueryReadClient(metaclass=BigQueryReadClientMeta): + """BigQuery Read API. + + The Read API can be used to read data from BigQuery. + + New code should use the v1 Read API going forward, if they don't + use Write API at the same time. + """ + + @staticmethod + def _get_default_mtls_endpoint(api_endpoint): + """Converts api endpoint to mTLS endpoint. + + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to + "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. + Args: + api_endpoint (Optional[str]): the api endpoint to convert. + Returns: + str: converted mTLS api endpoint. + """ + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" + DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_ENDPOINT + ) + + _DEFAULT_ENDPOINT_TEMPLATE = "bigquerystorage.{UNIVERSE_DOMAIN}" + _DEFAULT_UNIVERSE = "googleapis.com" + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryReadClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file( + filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryReadTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryReadTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def read_session_path(project: str,location: str,session: str,) -> str: + """Returns a fully-qualified read_session string.""" + return "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) + + @staticmethod + def parse_read_session_path(path: str) -> Dict[str,str]: + """Parses a read_session path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def read_stream_path(project: str,location: str,session: str,stream: str,) -> str: + """Returns a fully-qualified read_stream string.""" + return "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) + + @staticmethod + def parse_read_stream_path(path: str) -> Dict[str,str]: + """Parses a read_stream path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)/sessions/(?P.+?)/streams/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def table_path(project: str,dataset: str,table: str,) -> str: + """Returns a fully-qualified table string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + + @staticmethod + def parse_table_path(path: str) -> Dict[str,str]: + """Parses a table path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_billing_account_path(billing_account: str, ) -> str: + """Returns a fully-qualified billing_account string.""" + return "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + + @staticmethod + def parse_common_billing_account_path(path: str) -> Dict[str,str]: + """Parse a billing_account path into its component segments.""" + m = re.match(r"^billingAccounts/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_folder_path(folder: str, ) -> str: + """Returns a fully-qualified folder string.""" + return "folders/{folder}".format(folder=folder, ) + + @staticmethod + def parse_common_folder_path(path: str) -> Dict[str,str]: + """Parse a folder path into its component segments.""" + m = re.match(r"^folders/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_organization_path(organization: str, ) -> str: + """Returns a fully-qualified organization string.""" + return "organizations/{organization}".format(organization=organization, ) + + @staticmethod + def parse_common_organization_path(path: str) -> Dict[str,str]: + """Parse a organization path into its component segments.""" + m = re.match(r"^organizations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_project_path(project: str, ) -> str: + """Returns a fully-qualified project string.""" + return "projects/{project}".format(project=project, ) + + @staticmethod + def parse_common_project_path(path: str) -> Dict[str,str]: + """Parse a project path into its component segments.""" + m = re.match(r"^projects/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_location_path(project: str, location: str, ) -> str: + """Returns a fully-qualified location string.""" + return "projects/{project}/locations/{location}".format(project=project, location=location, ) + + @staticmethod + def parse_common_location_path(path: str) -> Dict[str,str]: + """Parse a location path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @classmethod + def get_mtls_endpoint_and_cert_source(cls, client_options: Optional[client_options_lib.ClientOptions] = None): + """Deprecated. Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variable is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + + warnings.warn("get_mtls_endpoint_and_cert_source is deprecated. Use the api_endpoint property instead.", + DeprecationWarning) + if client_options is None: + client_options = client_options_lib.ClientOptions() + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_client_cert not in ("true", "false"): + raise ValueError("Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`") + + # Figure out the client cert source to use. + client_cert_source = None + if use_client_cert == "true": + if client_options.client_cert_source: + client_cert_source = client_options.client_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + api_endpoint = cls.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = cls.DEFAULT_ENDPOINT + + return api_endpoint, client_cert_source + + @staticmethod + def _read_environment_variables(): + """Returns the environment variables used by the client. + + Returns: + Tuple[bool, str, str]: returns the GOOGLE_API_USE_CLIENT_CERTIFICATE, + GOOGLE_API_USE_MTLS_ENDPOINT, and GOOGLE_CLOUD_UNIVERSE_DOMAIN environment variables. + + Raises: + ValueError: If GOOGLE_API_USE_CLIENT_CERTIFICATE is not + any of ["true", "false"]. + google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT + is not any of ["auto", "never", "always"]. + """ + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false").lower() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_client_cert not in ("true", "false"): + raise ValueError("Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`") + return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + + @staticmethod + def _get_client_cert_source(provided_cert_source, use_cert_flag): + """Return the client cert source to be used by the client. + + Args: + provided_cert_source (bytes): The client certificate source provided. + use_cert_flag (bool): A flag indicating whether to use the client certificate. + + Returns: + bytes or None: The client cert source to be used by the client. + """ + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + return client_cert_source + + @staticmethod + def _get_api_endpoint(api_override, client_cert_source, universe_domain, use_mtls_endpoint): + """Return the API endpoint used by the client. + + Args: + api_override (str): The API endpoint override. If specified, this is always + the return value of this function and the other arguments are not used. + client_cert_source (bytes): The client certificate source used by the client. + universe_domain (str): The universe domain used by the client. + use_mtls_endpoint (str): How to use the mTLS endpoint, which depends also on the other parameters. + Possible values are "always", "auto", or "never". + + Returns: + str: The API endpoint to be used by the client. + """ + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + _default_universe = BigQueryReadClient._DEFAULT_UNIVERSE + if universe_domain != _default_universe: + raise MutualTLSChannelError(f"mTLS is not supported in any universe other than {_default_universe}.") + api_endpoint = BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=universe_domain) + return api_endpoint + + @staticmethod + def _get_universe_domain(client_universe_domain: Optional[str], universe_domain_env: Optional[str]) -> str: + """Return the universe domain used by the client. + + Args: + client_universe_domain (Optional[str]): The universe domain configured via the client options. + universe_domain_env (Optional[str]): The universe domain configured via the "GOOGLE_CLOUD_UNIVERSE_DOMAIN" environment variable. + + Returns: + str: The universe domain to be used by the client. + + Raises: + ValueError: If the universe domain is an empty string. + """ + universe_domain = BigQueryReadClient._DEFAULT_UNIVERSE + if client_universe_domain is not None: + universe_domain = client_universe_domain + elif universe_domain_env is not None: + universe_domain = universe_domain_env + if len(universe_domain.strip()) == 0: + raise ValueError("Universe Domain cannot be an empty string.") + return universe_domain + + @staticmethod + def _compare_universes(client_universe: str, + credentials: ga_credentials.Credentials) -> bool: + """Returns True iff the universe domains used by the client and credentials match. + + Args: + client_universe (str): The universe domain configured via the client options. + credentials (ga_credentials.Credentials): The credentials being used in the client. + + Returns: + bool: True iff client_universe matches the universe in credentials. + + Raises: + ValueError: when client_universe does not match the universe in credentials. + """ + + default_universe = BigQueryReadClient._DEFAULT_UNIVERSE + credentials_universe = getattr(credentials, "universe_domain", default_universe) + + if client_universe != credentials_universe: + raise ValueError("The configured universe domain " + f"({client_universe}) does not match the universe domain " + f"found in the credentials ({credentials_universe}). " + "If you haven't configured the universe domain explicitly, " + f"`{default_universe}` is the default.") + return True + + def _validate_universe_domain(self): + """Validates client's and credentials' universe domains are consistent. + + Returns: + bool: True iff the configured universe domain is valid. + + Raises: + ValueError: If the configured universe domain is not valid. + """ + self._is_universe_domain_valid = (self._is_universe_domain_valid or + BigQueryReadClient._compare_universes(self.universe_domain, self.transport._credentials)) + return self._is_universe_domain_valid + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used by the client instance. + """ + return self._universe_domain + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Optional[Union[str, BigQueryReadTransport, Callable[..., BigQueryReadTransport]]] = None, + client_options: Optional[Union[client_options_lib.ClientOptions, dict]] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query read client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Optional[Union[str,BigQueryReadTransport,Callable[..., BigQueryReadTransport]]]): + The transport to use, or a Callable that constructs and returns a new transport. + If a Callable is given, it will be called with the same set of initialization + arguments as used in the BigQueryReadTransport constructor. + If set to None, a transport is chosen automatically. + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide a client certificate for mTLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that the ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client_options = client_options + if isinstance(self._client_options, dict): + self._client_options = client_options_lib.from_dict(self._client_options) + if self._client_options is None: + self._client_options = client_options_lib.ClientOptions() + self._client_options = cast(client_options_lib.ClientOptions, self._client_options) + + universe_domain_opt = getattr(self._client_options, 'universe_domain', None) + + self._use_client_cert, self._use_mtls_endpoint, self._universe_domain_env = BigQueryReadClient._read_environment_variables() + self._client_cert_source = BigQueryReadClient._get_client_cert_source(self._client_options.client_cert_source, self._use_client_cert) + self._universe_domain = BigQueryReadClient._get_universe_domain(universe_domain_opt, self._universe_domain_env) + self._api_endpoint = None # updated below, depending on `transport` + + # Initialize the universe domain validation. + self._is_universe_domain_valid = False + + api_key_value = getattr(self._client_options, "api_key", None) + if api_key_value and credentials: + raise ValueError("client_options.api_key and credentials are mutually exclusive") + + # Save or instantiate the transport. + # Ordinarily, we provide the transport, but allowing a custom transport + # instance provides an extensibility point for unusual situations. + transport_provided = isinstance(transport, BigQueryReadTransport) + if transport_provided: + # transport is a BigQueryReadTransport instance. + if credentials or self._client_options.credentials_file or api_key_value: + raise ValueError("When providing a transport instance, " + "provide its credentials directly.") + if self._client_options.scopes: + raise ValueError( + "When providing a transport instance, provide its scopes " + "directly." + ) + self._transport = cast(BigQueryReadTransport, transport) + self._api_endpoint = self._transport.host + + self._api_endpoint = (self._api_endpoint or + BigQueryReadClient._get_api_endpoint( + self._client_options.api_endpoint, + self._client_cert_source, + self._universe_domain, + self._use_mtls_endpoint)) + + if not transport_provided: + import google.auth._default # type: ignore + + if api_key_value and hasattr(google.auth._default, "get_api_key_credentials"): + credentials = google.auth._default.get_api_key_credentials(api_key_value) + + transport_init: Union[Type[BigQueryReadTransport], Callable[..., BigQueryReadTransport]] = ( + BigQueryReadClient.get_transport_class(transport) + if isinstance(transport, str) or transport is None + else cast(Callable[..., BigQueryReadTransport], transport) + ) + # initialize with the provided callable or the passed in class + self._transport = transport_init( + credentials=credentials, + credentials_file=self._client_options.credentials_file, + host=self._api_endpoint, + scopes=self._client_options.scopes, + client_cert_source_for_mtls=self._client_cert_source, + quota_project_id=self._client_options.quota_project_id, + client_info=client_info, + always_use_jwt_access=True, + api_audience=self._client_options.api_audience, + ) + + def create_read_session(self, + request: Optional[Union[storage.CreateReadSessionRequest, dict]] = None, + *, + parent: Optional[str] = None, + read_session: Optional[stream.ReadSession] = None, + max_stream_count: Optional[int] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.ReadSession: + r"""Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a + push-down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + def sample_create_read_session(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.CreateReadSessionRequest( + parent="parent_value", + ) + + # Make the request + response = client.create_read_session(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1beta2.types.CreateReadSessionRequest, dict]): + The request object. Request message for ``CreateReadSession``. + parent (str): + Required. The request project that owns the session, in + the form of ``projects/{project_id}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + read_session (google.cloud.bigquery_storage_v1beta2.types.ReadSession): + Required. Session to be created. + This corresponds to the ``read_session`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + max_stream_count (int): + Max initial number of streams. If + unset or zero, the server will provide a + value of streams so as to produce + reasonable throughput. Must be + non-negative. The number of streams may + be lower than the requested number, + depending on the amount parallelism that + is reasonable for the table. Error will + be returned if the max count is greater + than the current system max limit of + 1,000. + + Streams must be read starting from + offset 0. + + This corresponds to the ``max_stream_count`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.ReadSession: + Information about the ReadSession. + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, read_session, max_stream_count]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.CreateReadSessionRequest): + request = storage.CreateReadSessionRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if read_session is not None: + request.read_session = read_session + if max_stream_count is not None: + request.max_stream_count = max_stream_count + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.create_read_session] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_session.table", request.read_session.table), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def read_rows(self, + request: Optional[Union[storage.ReadRowsRequest, dict]] = None, + *, + read_stream: Optional[str] = None, + offset: Optional[int] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Iterable[storage.ReadRowsResponse]: + r"""Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + def sample_read_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.ReadRowsRequest( + read_stream="read_stream_value", + ) + + # Make the request + stream = client.read_rows(request=request) + + # Handle the response + for response in stream: + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1beta2.types.ReadRowsRequest, dict]): + The request object. Request message for ``ReadRows``. + read_stream (str): + Required. Stream to read rows from. + This corresponds to the ``read_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + offset (int): + The offset requested must be less + than the last row read from Read. + Requesting a larger offset is undefined. + If not specified, start reading from + offset zero. + + This corresponds to the ``offset`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + Iterable[google.cloud.bigquery_storage_v1beta2.types.ReadRowsResponse]: + Response from calling ReadRows may include row data, progress and + throttling information. + + """ + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([read_stream, offset]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.ReadRowsRequest): + request = storage.ReadRowsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if read_stream is not None: + request.read_stream = read_stream + if offset is not None: + request.offset = offset + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.read_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("read_stream", request.read_stream), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def split_read_stream(self, + request: Optional[Union[storage.SplitReadStreamRequest, dict]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.SplitReadStreamResponse: + r"""Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + def sample_split_read_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.SplitReadStreamRequest( + name="name_value", + ) + + # Make the request + response = client.split_read_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamRequest, dict]): + The request object. Request message for ``SplitReadStream``. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamResponse: + + """ + # Create or coerce a protobuf request object. + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.SplitReadStreamRequest): + request = storage.SplitReadStreamRequest(request) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.split_read_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def __enter__(self) -> "BigQueryReadClient": + return self + + def __exit__(self, type, value, traceback): + """Releases underlying transport's resources. + + .. warning:: + ONLY use as a context manager if the transport is NOT shared + with other clients! Exiting the with block will CLOSE the transport + and may cause errors in other clients! + """ + self.transport.close() + + + + + + + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +__all__ = ( + "BigQueryReadClient", +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/README.rst b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/README.rst new file mode 100644 index 00000000..687d374a --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/README.rst @@ -0,0 +1,9 @@ + +transport inheritance structure +_______________________________ + +`BigQueryReadTransport` is the ABC for all transports. +- public child `BigQueryReadGrpcTransport` for sync gRPC transport (defined in `grpc.py`). +- public child `BigQueryReadGrpcAsyncIOTransport` for async gRPC transport (defined in `grpc_asyncio.py`). +- private child `_BaseBigQueryReadRestTransport` for base REST transport with inner classes `_BaseMETHOD` (defined in `rest_base.py`). +- public child `BigQueryReadRestTransport` for sync REST transport with inner classes `METHOD` derived from the parent's corresponding `_BaseMETHOD` classes (defined in `rest.py`). diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/__init__.py new file mode 100644 index 00000000..7547977d --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from typing import Dict, Type + +from .base import BigQueryReadTransport +from .grpc import BigQueryReadGrpcTransport +from .grpc_asyncio import BigQueryReadGrpcAsyncIOTransport + + +# Compile a registry of transports. +_transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryReadTransport]] +_transport_registry['grpc'] = BigQueryReadGrpcTransport +_transport_registry['grpc_asyncio'] = BigQueryReadGrpcAsyncIOTransport + +__all__ = ( + 'BigQueryReadTransport', + 'BigQueryReadGrpcTransport', + 'BigQueryReadGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py new file mode 100644 index 00000000..726d2e78 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/base.py @@ -0,0 +1,213 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union + +from google.cloud.bigquery_storage_v1beta2 import gapic_version as package_version + +import google.auth # type: ignore +import google.api_core +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +class BigQueryReadTransport(abc.ABC): + """Abstract transport class for BigQueryRead.""" + + AUTH_SCOPES = ( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', + ) + + DEFAULT_HOST: str = 'bigquerystorage.googleapis.com' + def __init__( + self, *, + host: str = DEFAULT_HOST, + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + **kwargs, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A list of scopes. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + """ + + scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES} + + # Save the scopes. + self._scopes = scopes + if not hasattr(self, "_ignore_credentials"): + self._ignore_credentials: bool = False + + # If no credentials are provided, then determine the appropriate + # defaults. + if credentials and credentials_file: + raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive") + + if credentials_file is not None: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, + **scopes_kwargs, + quota_project_id=quota_project_id + ) + elif credentials is None and not self._ignore_credentials: + credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id) + # Don't apply audience if the credentials file passed from user. + if hasattr(credentials, "with_gdch_audience"): + credentials = credentials.with_gdch_audience(api_audience if api_audience else host) + + # If the credentials are service account credentials, then always try to use self signed JWT. + if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"): + credentials = credentials.with_always_use_jwt_access(True) + + # Save the credentials. + self._credentials = credentials + + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ':' not in host: + host += ':443' + self._host = host + + @property + def host(self): + return self._host + + def _prep_wrapped_messages(self, client_info): + # Precompute the wrapped methods. + self._wrapped_methods = { + self.create_read_session: gapic_v1.method.wrap_method( + self.create_read_session, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.read_rows: gapic_v1.method.wrap_method( + self.read_rows, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.split_read_stream: gapic_v1.method.wrap_method( + self.split_read_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + def close(self): + """Closes resources associated with the transport. + + .. warning:: + Only call this method if the transport is NOT shared + with other clients - this may cause errors in other clients! + """ + raise NotImplementedError() + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + Union[ + stream.ReadSession, + Awaitable[stream.ReadSession] + ]]: + raise NotImplementedError() + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + Union[ + storage.ReadRowsResponse, + Awaitable[storage.ReadRowsResponse] + ]]: + raise NotImplementedError() + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + Union[ + storage.SplitReadStreamResponse, + Awaitable[storage.SplitReadStreamResponse] + ]]: + raise NotImplementedError() + + @property + def kind(self) -> str: + raise NotImplementedError() + + +__all__ = ( + 'BigQueryReadTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py new file mode 100644 index 00000000..073d2680 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc.py @@ -0,0 +1,369 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import grpc_helpers +from google.api_core import gapic_v1 +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO + + +class BigQueryReadGrpcTransport(BigQueryReadTransport): + """gRPC backend transport for BigQueryRead. + + BigQuery Read API. + + The Read API can be used to read data from BigQuery. + + New code should use the v1 Read API going forward, if they don't + use Write API at the same time. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + _stubs: Dict[str, Callable] + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]] = None, + api_mtls_endpoint: Optional[str] = None, + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + ssl_channel_credentials: Optional[grpc.ChannelCredentials] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if a ``channel`` instance is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if a ``channel`` instance is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if a ``channel`` instance is provided. + channel (Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]]): + A ``Channel`` instance through which to make calls, or a Callable + that constructs and returns one. If set to None, ``self.create_channel`` + is used to create the channel. If a Callable is given, it will be called + with the same arguments as used in ``self.create_channel``. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or application default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for the grpc channel. It is ignored if a ``channel`` instance is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if a ``channel`` instance or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if isinstance(channel, grpc.Channel): + # Ignore credentials if a channel was passed. + credentials = None + self._ignore_credentials = True + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, + ) + + if not self._grpc_channel: + # initialize with the provided callable or the default channel + channel_init = channel or type(self).create_channel + self._grpc_channel = channel_init( + self._host, + # use the credentials which are saved + credentials=self._credentials, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> grpc.Channel: + """Create and return a gRPC channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + grpc.Channel: A gRPC channel object. + + Raises: + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + + return grpc_helpers.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + @property + def grpc_channel(self) -> grpc.Channel: + """Return the channel designed to connect to this service. + """ + return self._grpc_channel + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + stream.ReadSession]: + r"""Return a callable for the create read session method over gRPC. + + Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a + push-down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + Returns: + Callable[[~.CreateReadSessionRequest], + ~.ReadSession]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_read_session' not in self._stubs: + self._stubs['create_read_session'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/CreateReadSession', + request_serializer=storage.CreateReadSessionRequest.serialize, + response_deserializer=stream.ReadSession.deserialize, + ) + return self._stubs['create_read_session'] + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + storage.ReadRowsResponse]: + r"""Return a callable for the read rows method over gRPC. + + Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Returns: + Callable[[~.ReadRowsRequest], + ~.ReadRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'read_rows' not in self._stubs: + self._stubs['read_rows'] = self.grpc_channel.unary_stream( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/ReadRows', + request_serializer=storage.ReadRowsRequest.serialize, + response_deserializer=storage.ReadRowsResponse.deserialize, + ) + return self._stubs['read_rows'] + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + storage.SplitReadStreamResponse]: + r"""Return a callable for the split read stream method over gRPC. + + Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Returns: + Callable[[~.SplitReadStreamRequest], + ~.SplitReadStreamResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'split_read_stream' not in self._stubs: + self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/SplitReadStream', + request_serializer=storage.SplitReadStreamRequest.serialize, + response_deserializer=storage.SplitReadStreamResponse.deserialize, + ) + return self._stubs['split_read_stream'] + + def close(self): + self.grpc_channel.close() + + @property + def kind(self) -> str: + return "grpc" + + +__all__ = ( + 'BigQueryReadGrpcTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py new file mode 100644 index 00000000..77f43c47 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_read/transports/grpc_asyncio.py @@ -0,0 +1,429 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import inspect +import warnings +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers_async +from google.api_core import exceptions as core_exceptions +from google.api_core import retry_async as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore +from grpc.experimental import aio # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from .base import BigQueryReadTransport, DEFAULT_CLIENT_INFO +from .grpc import BigQueryReadGrpcTransport + + +class BigQueryReadGrpcAsyncIOTransport(BigQueryReadTransport): + """gRPC AsyncIO backend transport for BigQueryRead. + + BigQuery Read API. + + The Read API can be used to read data from BigQuery. + + New code should use the v1 Read API going forward, if they don't + use Write API at the same time. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + + _grpc_channel: aio.Channel + _stubs: Dict[str, Callable] = {} + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> aio.Channel: + """Create and return a gRPC AsyncIO channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + aio.Channel: A gRPC AsyncIO channel object. + """ + + return grpc_helpers_async.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: Optional[Union[aio.Channel, Callable[..., aio.Channel]]] = None, + api_mtls_endpoint: Optional[str] = None, + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + ssl_channel_credentials: Optional[grpc.ChannelCredentials] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if a ``channel`` instance is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if a ``channel`` instance is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + channel (Optional[Union[aio.Channel, Callable[..., aio.Channel]]]): + A ``Channel`` instance through which to make calls, or a Callable + that constructs and returns one. If set to None, ``self.create_channel`` + is used to create the channel. If a Callable is given, it will be called + with the same arguments as used in ``self.create_channel``. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or application default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for the grpc channel. It is ignored if a ``channel`` instance is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if a ``channel`` instance or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if isinstance(channel, aio.Channel): + # Ignore credentials if a channel was passed. + credentials = None + self._ignore_credentials = True + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, + ) + + if not self._grpc_channel: + # initialize with the provided callable or the default channel + channel_init = channel or type(self).create_channel + self._grpc_channel = channel_init( + self._host, + # use the credentials which are saved + credentials=self._credentials, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._wrap_with_kind = "kind" in inspect.signature(gapic_v1.method_async.wrap_method).parameters + self._prep_wrapped_messages(client_info) + + @property + def grpc_channel(self) -> aio.Channel: + """Create the channel designed to connect to this service. + + This property caches on the instance; repeated calls return + the same channel. + """ + # Return the channel from cache. + return self._grpc_channel + + @property + def create_read_session(self) -> Callable[ + [storage.CreateReadSessionRequest], + Awaitable[stream.ReadSession]]: + r"""Return a callable for the create read session method over gRPC. + + Creates a new read session. A read session divides + the contents of a BigQuery table into one or more + streams, which can then be used to read data from the + table. The read session also specifies properties of the + data to be read, such as a list of columns or a + push-down filter describing the rows to be returned. + + A particular row can be read by at most one stream. When + the caller has reached the end of each stream in the + session, then all the data in the table has been read. + + Data is assigned to each stream such that roughly the + same number of rows can be read from each stream. + Because the server-side unit for assigning data is + collections of rows, the API does not guarantee that + each stream will return the same number or rows. + Additionally, the limits are enforced based on the + number of pre-filtered rows, so some filters can lead to + lopsided assignments. + + Read sessions automatically expire 6 hours after they + are created and do not require manual clean-up by the + caller. + + Returns: + Callable[[~.CreateReadSessionRequest], + Awaitable[~.ReadSession]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_read_session' not in self._stubs: + self._stubs['create_read_session'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/CreateReadSession', + request_serializer=storage.CreateReadSessionRequest.serialize, + response_deserializer=stream.ReadSession.deserialize, + ) + return self._stubs['create_read_session'] + + @property + def read_rows(self) -> Callable[ + [storage.ReadRowsRequest], + Awaitable[storage.ReadRowsResponse]]: + r"""Return a callable for the read rows method over gRPC. + + Reads rows from the stream in the format prescribed + by the ReadSession. Each response contains one or more + table rows, up to a maximum of 100 MiB per response; + read requests which attempt to read individual rows + larger than 100 MiB will fail. + + Each request also returns a set of stream statistics + reflecting the current state of the stream. + + Returns: + Callable[[~.ReadRowsRequest], + Awaitable[~.ReadRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'read_rows' not in self._stubs: + self._stubs['read_rows'] = self.grpc_channel.unary_stream( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/ReadRows', + request_serializer=storage.ReadRowsRequest.serialize, + response_deserializer=storage.ReadRowsResponse.deserialize, + ) + return self._stubs['read_rows'] + + @property + def split_read_stream(self) -> Callable[ + [storage.SplitReadStreamRequest], + Awaitable[storage.SplitReadStreamResponse]]: + r"""Return a callable for the split read stream method over gRPC. + + Splits a given ``ReadStream`` into two ``ReadStream`` objects. + These ``ReadStream`` objects are referred to as the primary and + the residual streams of the split. The original ``ReadStream`` + can still be read from in the same manner as before. Both of the + returned ``ReadStream`` objects can also be read from, and the + rows returned by both child streams will be the same as the rows + read from the original stream. + + Moreover, the two child streams will be allocated back-to-back + in the original ``ReadStream``. Concretely, it is guaranteed + that for streams original, primary, and residual, that + original[0-j] = primary[0-j] and original[j-n] = residual[0-m] + once the streams have been read to completion. + + Returns: + Callable[[~.SplitReadStreamRequest], + Awaitable[~.SplitReadStreamResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'split_read_stream' not in self._stubs: + self._stubs['split_read_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryRead/SplitReadStream', + request_serializer=storage.SplitReadStreamRequest.serialize, + response_deserializer=storage.SplitReadStreamResponse.deserialize, + ) + return self._stubs['split_read_stream'] + + def _prep_wrapped_messages(self, client_info): + """ Precompute the wrapped methods, overriding the base class method to use async wrappers.""" + self._wrapped_methods = { + self.create_read_session: self._wrap_method( + self.create_read_session, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.read_rows: self._wrap_method( + self.read_rows, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.split_read_stream: self._wrap_method( + self.split_read_stream, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + def _wrap_method(self, func, *args, **kwargs): + if self._wrap_with_kind: # pragma: NO COVER + kwargs["kind"] = self.kind + return gapic_v1.method_async.wrap_method(func, *args, **kwargs) + + def close(self): + return self.grpc_channel.close() + + @property + def kind(self) -> str: + return "grpc_asyncio" + + +__all__ = ( + 'BigQueryReadGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/__init__.py new file mode 100644 index 00000000..9000288a --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/__init__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .client import BigQueryWriteClient +from .async_client import BigQueryWriteAsyncClient + +__all__ = ( + 'BigQueryWriteClient', + 'BigQueryWriteAsyncClient', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py new file mode 100644 index 00000000..1bc0fe3b --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/async_client.py @@ -0,0 +1,907 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import re +from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, AsyncIterable, Awaitable, AsyncIterator, Sequence, Tuple, Type, Union +import warnings + +from google.cloud.bigquery_storage_v1beta2 import gapic_version as package_version + +from google.api_core.client_options import ClientOptions +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry_async as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + + +try: + OptionalRetry = Union[retries.AsyncRetry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.AsyncRetry, object, None] # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.cloud.bigquery_storage_v1beta2.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +from .transports.base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .transports.grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport +from .client import BigQueryWriteClient + + +class BigQueryWriteAsyncClient: + """BigQuery Write API. + + The Write API can be used to write data to BigQuery. + + The `google.cloud.bigquery.storage.v1 + API `__ + should be used instead of the v1beta2 API for BigQueryWrite + operations. + """ + + _client: BigQueryWriteClient + + # Copy defaults from the synchronous client for use here. + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + DEFAULT_ENDPOINT = BigQueryWriteClient.DEFAULT_ENDPOINT + DEFAULT_MTLS_ENDPOINT = BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + _DEFAULT_ENDPOINT_TEMPLATE = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE + _DEFAULT_UNIVERSE = BigQueryWriteClient._DEFAULT_UNIVERSE + + table_path = staticmethod(BigQueryWriteClient.table_path) + parse_table_path = staticmethod(BigQueryWriteClient.parse_table_path) + write_stream_path = staticmethod(BigQueryWriteClient.write_stream_path) + parse_write_stream_path = staticmethod(BigQueryWriteClient.parse_write_stream_path) + common_billing_account_path = staticmethod(BigQueryWriteClient.common_billing_account_path) + parse_common_billing_account_path = staticmethod(BigQueryWriteClient.parse_common_billing_account_path) + common_folder_path = staticmethod(BigQueryWriteClient.common_folder_path) + parse_common_folder_path = staticmethod(BigQueryWriteClient.parse_common_folder_path) + common_organization_path = staticmethod(BigQueryWriteClient.common_organization_path) + parse_common_organization_path = staticmethod(BigQueryWriteClient.parse_common_organization_path) + common_project_path = staticmethod(BigQueryWriteClient.common_project_path) + parse_common_project_path = staticmethod(BigQueryWriteClient.parse_common_project_path) + common_location_path = staticmethod(BigQueryWriteClient.common_location_path) + parse_common_location_path = staticmethod(BigQueryWriteClient.parse_common_location_path) + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteAsyncClient: The constructed client. + """ + return BigQueryWriteClient.from_service_account_info.__func__(BigQueryWriteAsyncClient, info, *args, **kwargs) # type: ignore + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteAsyncClient: The constructed client. + """ + return BigQueryWriteClient.from_service_account_file.__func__(BigQueryWriteAsyncClient, filename, *args, **kwargs) # type: ignore + + from_service_account_json = from_service_account_file + + @classmethod + def get_mtls_endpoint_and_cert_source(cls, client_options: Optional[ClientOptions] = None): + """Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variable is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + return BigQueryWriteClient.get_mtls_endpoint_and_cert_source(client_options) # type: ignore + + @property + def transport(self) -> BigQueryWriteTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryWriteTransport: The transport used by the client instance. + """ + return self._client.transport + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._client._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used + by the client instance. + """ + return self._client._universe_domain + + get_transport_class = BigQueryWriteClient.get_transport_class + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Optional[Union[str, BigQueryWriteTransport, Callable[..., BigQueryWriteTransport]]] = "grpc_asyncio", + client_options: Optional[ClientOptions] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query write async client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Optional[Union[str,BigQueryWriteTransport,Callable[..., BigQueryWriteTransport]]]): + The transport to use, or a Callable that constructs and returns a new transport to use. + If a Callable is given, it will be called with the same set of initialization + arguments as used in the BigQueryWriteTransport constructor. + If set to None, a transport is chosen automatically. + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide a client certificate for mTLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client = BigQueryWriteClient( + credentials=credentials, + transport=transport, + client_options=client_options, + client_info=client_info, + + ) + + async def create_write_stream(self, + request: Optional[Union[storage.CreateWriteStreamRequest, dict]] = None, + *, + parent: Optional[str] = None, + write_stream: Optional[stream.WriteStream] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Creates a write stream to the given table. Additionally, every + table has a special COMMITTED stream named '_default' to which + data can be written. This stream doesn't need to be created + using CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + async def sample_create_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.CreateWriteStreamRequest( + parent="parent_value", + ) + + # Make the request + response = await client.create_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1beta2.types.CreateWriteStreamRequest, dict]]): + The request object. Request message for ``CreateWriteStream``. + parent (:class:`str`): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + write_stream (:class:`google.cloud.bigquery_storage_v1beta2.types.WriteStream`): + Required. Stream to be created. + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + warnings.warn("BigQueryWriteAsyncClient.create_write_stream is deprecated", + DeprecationWarning) + + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, write_stream]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.CreateWriteStreamRequest): + request = storage.CreateWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.create_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def append_rows(self, + requests: Optional[AsyncIterator[storage.AppendRowsRequest]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Awaitable[AsyncIterable[storage.AppendRowsResponse]]: + r"""Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provids an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC stream. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains the offset at which the append happened. + Responses are received in the same order in which requests are + sent. There will be one response for each successful request. If + the ``offset`` is not set in response, it means append didn't + happen due to some errors. If one request fails, all the + subsequent requests will also fail until a success request is + made again. + + If the stream is of ``PENDING`` type, data will only be + available for read operations after the stream is committed. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + async def sample_append_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.AppendRowsRequest( + write_stream="write_stream_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1beta2.AppendRowsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = await client.append_rows(requests=request_generator()) + + # Handle the response + async for response in stream: + print(response) + + Args: + requests (AsyncIterator[`google.cloud.bigquery_storage_v1beta2.types.AppendRowsRequest`]): + The request object AsyncIterator. Request message for ``AppendRows``. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + AsyncIterable[google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse]: + Response message for AppendRows. + """ + warnings.warn("BigQueryWriteAsyncClient.append_rows is deprecated", + DeprecationWarning) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.append_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = rpc( + requests, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def get_write_stream(self, + request: Optional[Union[storage.GetWriteStreamRequest, dict]] = None, + *, + name: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Gets a write stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + async def sample_get_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.GetWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.get_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1beta2.types.GetWriteStreamRequest, dict]]): + The request object. Request message for ``GetWriteStreamRequest``. + name (:class:`str`): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + warnings.warn("BigQueryWriteAsyncClient.get_write_stream is deprecated", + DeprecationWarning) + + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.GetWriteStreamRequest): + request = storage.GetWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.get_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def finalize_write_stream(self, + request: Optional[Union[storage.FinalizeWriteStreamRequest, dict]] = None, + *, + name: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FinalizeWriteStreamResponse: + r"""Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + async def sample_finalize_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.FinalizeWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.finalize_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamRequest, dict]]): + The request object. Request message for invoking ``FinalizeWriteStream``. + name (:class:`str`): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamResponse: + Response message for FinalizeWriteStream. + """ + warnings.warn("BigQueryWriteAsyncClient.finalize_write_stream is deprecated", + DeprecationWarning) + + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.FinalizeWriteStreamRequest): + request = storage.FinalizeWriteStreamRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.finalize_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def batch_commit_write_streams(self, + request: Optional[Union[storage.BatchCommitWriteStreamsRequest, dict]] = None, + *, + parent: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.BatchCommitWriteStreamsResponse: + r"""Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. Streams must be finalized before + commit and cannot be committed multiple times. Once a stream is + committed, data in the stream becomes available for read + operations. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + async def sample_batch_commit_write_streams(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.BatchCommitWriteStreamsRequest( + parent="parent_value", + write_streams=['write_streams_value1', 'write_streams_value2'], + ) + + # Make the request + response = await client.batch_commit_write_streams(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsRequest, dict]]): + The request object. Request message for ``BatchCommitWriteStreams``. + parent (:class:`str`): + Required. Parent table that all the streams should + belong to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsResponse: + Response message for BatchCommitWriteStreams. + """ + warnings.warn("BigQueryWriteAsyncClient.batch_commit_write_streams is deprecated", + DeprecationWarning) + + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.BatchCommitWriteStreamsRequest): + request = storage.BatchCommitWriteStreamsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.batch_commit_write_streams] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def flush_rows(self, + request: Optional[Union[storage.FlushRowsRequest, dict]] = None, + *, + write_stream: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FlushRowsResponse: + r"""Flushes rows to a BUFFERED stream. If users are appending rows + to BUFFERED stream, flush operation is required in order for the + rows to become available for reading. A Flush operation flushes + up to any previously flushed offset in a BUFFERED stream, to the + offset specified in the request. Flush is not supported on the + \_default stream, since it is not BUFFERED. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + async def sample_flush_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.FlushRowsRequest( + write_stream="write_stream_value", + ) + + # Make the request + response = await client.flush_rows(request=request) + + # Handle the response + print(response) + + Args: + request (Optional[Union[google.cloud.bigquery_storage_v1beta2.types.FlushRowsRequest, dict]]): + The request object. Request message for ``FlushRows``. + write_stream (:class:`str`): + Required. The stream that is the + target of the flush operation. + + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry_async.AsyncRetry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.FlushRowsResponse: + Respond message for FlushRows. + """ + warnings.warn("BigQueryWriteAsyncClient.flush_rows is deprecated", + DeprecationWarning) + + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([write_stream]) + if request is not None and has_flattened_params: + raise ValueError("If the `request` argument is set, then none of " + "the individual field arguments should be set.") + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.FlushRowsRequest): + request = storage.FlushRowsRequest(request) + + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._client._transport._wrapped_methods[self._client._transport.flush_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("write_stream", request.write_stream), + )), + ) + + # Validate the universe domain. + self._client._validate_universe_domain() + + # Send the request. + response = await rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + async def __aenter__(self) -> "BigQueryWriteAsyncClient": + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.transport.close() + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +__all__ = ( + "BigQueryWriteAsyncClient", +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/client.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/client.py new file mode 100644 index 00000000..eae701a7 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/client.py @@ -0,0 +1,1265 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +import os +import re +from typing import Dict, Callable, Mapping, MutableMapping, MutableSequence, Optional, Iterable, Iterator, Sequence, Tuple, Type, Union, cast +import warnings + +from google.cloud.bigquery_storage_v1beta2 import gapic_version as package_version + +from google.api_core import client_options as client_options_lib +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport import mtls # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore +from google.auth.exceptions import MutualTLSChannelError # type: ignore +from google.oauth2 import service_account # type: ignore + +try: + OptionalRetry = Union[retries.Retry, gapic_v1.method._MethodDefault, None] +except AttributeError: # pragma: NO COVER + OptionalRetry = Union[retries.Retry, object, None] # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.cloud.bigquery_storage_v1beta2.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +from .transports.base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .transports.grpc import BigQueryWriteGrpcTransport +from .transports.grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport + + +class BigQueryWriteClientMeta(type): + """Metaclass for the BigQueryWrite client. + + This provides class-level methods for building and retrieving + support objects (e.g. transport) without polluting the client instance + objects. + """ + _transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryWriteTransport]] + _transport_registry["grpc"] = BigQueryWriteGrpcTransport + _transport_registry["grpc_asyncio"] = BigQueryWriteGrpcAsyncIOTransport + + def get_transport_class(cls, + label: Optional[str] = None, + ) -> Type[BigQueryWriteTransport]: + """Returns an appropriate transport class. + + Args: + label: The name of the desired transport. If none is + provided, then the first transport in the registry is used. + + Returns: + The transport class to use. + """ + # If a specific transport is requested, return that one. + if label: + return cls._transport_registry[label] + + # No transport is requested; return the default (that is, the first one + # in the dictionary). + return next(iter(cls._transport_registry.values())) + + +class BigQueryWriteClient(metaclass=BigQueryWriteClientMeta): + """BigQuery Write API. + + The Write API can be used to write data to BigQuery. + + The `google.cloud.bigquery.storage.v1 + API `__ + should be used instead of the v1beta2 API for BigQueryWrite + operations. + """ + + @staticmethod + def _get_default_mtls_endpoint(api_endpoint): + """Converts api endpoint to mTLS endpoint. + + Convert "*.sandbox.googleapis.com" and "*.googleapis.com" to + "*.mtls.sandbox.googleapis.com" and "*.mtls.googleapis.com" respectively. + Args: + api_endpoint (Optional[str]): the api endpoint to convert. + Returns: + str: converted mTLS api endpoint. + """ + if not api_endpoint: + return api_endpoint + + mtls_endpoint_re = re.compile( + r"(?P[^.]+)(?P\.mtls)?(?P\.sandbox)?(?P\.googleapis\.com)?" + ) + + m = mtls_endpoint_re.match(api_endpoint) + name, mtls, sandbox, googledomain = m.groups() + if mtls or not googledomain: + return api_endpoint + + if sandbox: + return api_endpoint.replace( + "sandbox.googleapis.com", "mtls.sandbox.googleapis.com" + ) + + return api_endpoint.replace(".googleapis.com", ".mtls.googleapis.com") + + # Note: DEFAULT_ENDPOINT is deprecated. Use _DEFAULT_ENDPOINT_TEMPLATE instead. + DEFAULT_ENDPOINT = "bigquerystorage.googleapis.com" + DEFAULT_MTLS_ENDPOINT = _get_default_mtls_endpoint.__func__( # type: ignore + DEFAULT_ENDPOINT + ) + + _DEFAULT_ENDPOINT_TEMPLATE = "bigquerystorage.{UNIVERSE_DOMAIN}" + _DEFAULT_UNIVERSE = "googleapis.com" + + @classmethod + def from_service_account_info(cls, info: dict, *args, **kwargs): + """Creates an instance of this client using the provided credentials + info. + + Args: + info (dict): The service account private key info. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_info(info) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + @classmethod + def from_service_account_file(cls, filename: str, *args, **kwargs): + """Creates an instance of this client using the provided credentials + file. + + Args: + filename (str): The path to the service account private key json + file. + args: Additional arguments to pass to the constructor. + kwargs: Additional arguments to pass to the constructor. + + Returns: + BigQueryWriteClient: The constructed client. + """ + credentials = service_account.Credentials.from_service_account_file( + filename) + kwargs["credentials"] = credentials + return cls(*args, **kwargs) + + from_service_account_json = from_service_account_file + + @property + def transport(self) -> BigQueryWriteTransport: + """Returns the transport used by the client instance. + + Returns: + BigQueryWriteTransport: The transport used by the client + instance. + """ + return self._transport + + @staticmethod + def table_path(project: str,dataset: str,table: str,) -> str: + """Returns a fully-qualified table string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + + @staticmethod + def parse_table_path(path: str) -> Dict[str,str]: + """Parses a table path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def write_stream_path(project: str,dataset: str,table: str,stream: str,) -> str: + """Returns a fully-qualified write_stream string.""" + return "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}".format(project=project, dataset=dataset, table=table, stream=stream, ) + + @staticmethod + def parse_write_stream_path(path: str) -> Dict[str,str]: + """Parses a write_stream path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/datasets/(?P.+?)/tables/(?P
.+?)/streams/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_billing_account_path(billing_account: str, ) -> str: + """Returns a fully-qualified billing_account string.""" + return "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + + @staticmethod + def parse_common_billing_account_path(path: str) -> Dict[str,str]: + """Parse a billing_account path into its component segments.""" + m = re.match(r"^billingAccounts/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_folder_path(folder: str, ) -> str: + """Returns a fully-qualified folder string.""" + return "folders/{folder}".format(folder=folder, ) + + @staticmethod + def parse_common_folder_path(path: str) -> Dict[str,str]: + """Parse a folder path into its component segments.""" + m = re.match(r"^folders/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_organization_path(organization: str, ) -> str: + """Returns a fully-qualified organization string.""" + return "organizations/{organization}".format(organization=organization, ) + + @staticmethod + def parse_common_organization_path(path: str) -> Dict[str,str]: + """Parse a organization path into its component segments.""" + m = re.match(r"^organizations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_project_path(project: str, ) -> str: + """Returns a fully-qualified project string.""" + return "projects/{project}".format(project=project, ) + + @staticmethod + def parse_common_project_path(path: str) -> Dict[str,str]: + """Parse a project path into its component segments.""" + m = re.match(r"^projects/(?P.+?)$", path) + return m.groupdict() if m else {} + + @staticmethod + def common_location_path(project: str, location: str, ) -> str: + """Returns a fully-qualified location string.""" + return "projects/{project}/locations/{location}".format(project=project, location=location, ) + + @staticmethod + def parse_common_location_path(path: str) -> Dict[str,str]: + """Parse a location path into its component segments.""" + m = re.match(r"^projects/(?P.+?)/locations/(?P.+?)$", path) + return m.groupdict() if m else {} + + @classmethod + def get_mtls_endpoint_and_cert_source(cls, client_options: Optional[client_options_lib.ClientOptions] = None): + """Deprecated. Return the API endpoint and client cert source for mutual TLS. + + The client cert source is determined in the following order: + (1) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is not "true", the + client cert source is None. + (2) if `client_options.client_cert_source` is provided, use the provided one; if the + default client cert source exists, use the default one; otherwise the client cert + source is None. + + The API endpoint is determined in the following order: + (1) if `client_options.api_endpoint` if provided, use the provided one. + (2) if `GOOGLE_API_USE_CLIENT_CERTIFICATE` environment variable is "always", use the + default mTLS endpoint; if the environment variable is "never", use the default API + endpoint; otherwise if client cert source exists, use the default mTLS endpoint, otherwise + use the default API endpoint. + + More details can be found at https://google.aip.dev/auth/4114. + + Args: + client_options (google.api_core.client_options.ClientOptions): Custom options for the + client. Only the `api_endpoint` and `client_cert_source` properties may be used + in this method. + + Returns: + Tuple[str, Callable[[], Tuple[bytes, bytes]]]: returns the API endpoint and the + client cert source to use. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If any errors happen. + """ + + warnings.warn("get_mtls_endpoint_and_cert_source is deprecated. Use the api_endpoint property instead.", + DeprecationWarning) + if client_options is None: + client_options = client_options_lib.ClientOptions() + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto") + if use_client_cert not in ("true", "false"): + raise ValueError("Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`") + + # Figure out the client cert source to use. + client_cert_source = None + if use_client_cert == "true": + if client_options.client_cert_source: + client_cert_source = client_options.client_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + + # Figure out which api endpoint to use. + if client_options.api_endpoint is not None: + api_endpoint = client_options.api_endpoint + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + api_endpoint = cls.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = cls.DEFAULT_ENDPOINT + + return api_endpoint, client_cert_source + + @staticmethod + def _read_environment_variables(): + """Returns the environment variables used by the client. + + Returns: + Tuple[bool, str, str]: returns the GOOGLE_API_USE_CLIENT_CERTIFICATE, + GOOGLE_API_USE_MTLS_ENDPOINT, and GOOGLE_CLOUD_UNIVERSE_DOMAIN environment variables. + + Raises: + ValueError: If GOOGLE_API_USE_CLIENT_CERTIFICATE is not + any of ["true", "false"]. + google.auth.exceptions.MutualTLSChannelError: If GOOGLE_API_USE_MTLS_ENDPOINT + is not any of ["auto", "never", "always"]. + """ + use_client_cert = os.getenv("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false").lower() + use_mtls_endpoint = os.getenv("GOOGLE_API_USE_MTLS_ENDPOINT", "auto").lower() + universe_domain_env = os.getenv("GOOGLE_CLOUD_UNIVERSE_DOMAIN") + if use_client_cert not in ("true", "false"): + raise ValueError("Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`") + if use_mtls_endpoint not in ("auto", "never", "always"): + raise MutualTLSChannelError("Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`") + return use_client_cert == "true", use_mtls_endpoint, universe_domain_env + + @staticmethod + def _get_client_cert_source(provided_cert_source, use_cert_flag): + """Return the client cert source to be used by the client. + + Args: + provided_cert_source (bytes): The client certificate source provided. + use_cert_flag (bool): A flag indicating whether to use the client certificate. + + Returns: + bytes or None: The client cert source to be used by the client. + """ + client_cert_source = None + if use_cert_flag: + if provided_cert_source: + client_cert_source = provided_cert_source + elif mtls.has_default_client_cert_source(): + client_cert_source = mtls.default_client_cert_source() + return client_cert_source + + @staticmethod + def _get_api_endpoint(api_override, client_cert_source, universe_domain, use_mtls_endpoint): + """Return the API endpoint used by the client. + + Args: + api_override (str): The API endpoint override. If specified, this is always + the return value of this function and the other arguments are not used. + client_cert_source (bytes): The client certificate source used by the client. + universe_domain (str): The universe domain used by the client. + use_mtls_endpoint (str): How to use the mTLS endpoint, which depends also on the other parameters. + Possible values are "always", "auto", or "never". + + Returns: + str: The API endpoint to be used by the client. + """ + if api_override is not None: + api_endpoint = api_override + elif use_mtls_endpoint == "always" or (use_mtls_endpoint == "auto" and client_cert_source): + _default_universe = BigQueryWriteClient._DEFAULT_UNIVERSE + if universe_domain != _default_universe: + raise MutualTLSChannelError(f"mTLS is not supported in any universe other than {_default_universe}.") + api_endpoint = BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + else: + api_endpoint = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=universe_domain) + return api_endpoint + + @staticmethod + def _get_universe_domain(client_universe_domain: Optional[str], universe_domain_env: Optional[str]) -> str: + """Return the universe domain used by the client. + + Args: + client_universe_domain (Optional[str]): The universe domain configured via the client options. + universe_domain_env (Optional[str]): The universe domain configured via the "GOOGLE_CLOUD_UNIVERSE_DOMAIN" environment variable. + + Returns: + str: The universe domain to be used by the client. + + Raises: + ValueError: If the universe domain is an empty string. + """ + universe_domain = BigQueryWriteClient._DEFAULT_UNIVERSE + if client_universe_domain is not None: + universe_domain = client_universe_domain + elif universe_domain_env is not None: + universe_domain = universe_domain_env + if len(universe_domain.strip()) == 0: + raise ValueError("Universe Domain cannot be an empty string.") + return universe_domain + + @staticmethod + def _compare_universes(client_universe: str, + credentials: ga_credentials.Credentials) -> bool: + """Returns True iff the universe domains used by the client and credentials match. + + Args: + client_universe (str): The universe domain configured via the client options. + credentials (ga_credentials.Credentials): The credentials being used in the client. + + Returns: + bool: True iff client_universe matches the universe in credentials. + + Raises: + ValueError: when client_universe does not match the universe in credentials. + """ + + default_universe = BigQueryWriteClient._DEFAULT_UNIVERSE + credentials_universe = getattr(credentials, "universe_domain", default_universe) + + if client_universe != credentials_universe: + raise ValueError("The configured universe domain " + f"({client_universe}) does not match the universe domain " + f"found in the credentials ({credentials_universe}). " + "If you haven't configured the universe domain explicitly, " + f"`{default_universe}` is the default.") + return True + + def _validate_universe_domain(self): + """Validates client's and credentials' universe domains are consistent. + + Returns: + bool: True iff the configured universe domain is valid. + + Raises: + ValueError: If the configured universe domain is not valid. + """ + self._is_universe_domain_valid = (self._is_universe_domain_valid or + BigQueryWriteClient._compare_universes(self.universe_domain, self.transport._credentials)) + return self._is_universe_domain_valid + + @property + def api_endpoint(self): + """Return the API endpoint used by the client instance. + + Returns: + str: The API endpoint used by the client instance. + """ + return self._api_endpoint + + @property + def universe_domain(self) -> str: + """Return the universe domain used by the client instance. + + Returns: + str: The universe domain used by the client instance. + """ + return self._universe_domain + + def __init__(self, *, + credentials: Optional[ga_credentials.Credentials] = None, + transport: Optional[Union[str, BigQueryWriteTransport, Callable[..., BigQueryWriteTransport]]] = None, + client_options: Optional[Union[client_options_lib.ClientOptions, dict]] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + ) -> None: + """Instantiates the big query write client. + + Args: + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + transport (Optional[Union[str,BigQueryWriteTransport,Callable[..., BigQueryWriteTransport]]]): + The transport to use, or a Callable that constructs and returns a new transport. + If a Callable is given, it will be called with the same set of initialization + arguments as used in the BigQueryWriteTransport constructor. + If set to None, a transport is chosen automatically. + client_options (Optional[Union[google.api_core.client_options.ClientOptions, dict]]): + Custom options for the client. + + 1. The ``api_endpoint`` property can be used to override the + default endpoint provided by the client when ``transport`` is + not explicitly provided. Only if this property is not set and + ``transport`` was not explicitly provided, the endpoint is + determined by the GOOGLE_API_USE_MTLS_ENDPOINT environment + variable, which have one of the following values: + "always" (always use the default mTLS endpoint), "never" (always + use the default regular endpoint) and "auto" (auto-switch to the + default mTLS endpoint if client certificate is present; this is + the default value). + + 2. If the GOOGLE_API_USE_CLIENT_CERTIFICATE environment variable + is "true", then the ``client_cert_source`` property can be used + to provide a client certificate for mTLS transport. If + not provided, the default SSL client certificate will be used if + present. If GOOGLE_API_USE_CLIENT_CERTIFICATE is "false" or not + set, no client certificate will be used. + + 3. The ``universe_domain`` property can be used to override the + default "googleapis.com" universe. Note that the ``api_endpoint`` + property still takes precedence; and ``universe_domain`` is + currently not supported for mTLS. + + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + """ + self._client_options = client_options + if isinstance(self._client_options, dict): + self._client_options = client_options_lib.from_dict(self._client_options) + if self._client_options is None: + self._client_options = client_options_lib.ClientOptions() + self._client_options = cast(client_options_lib.ClientOptions, self._client_options) + + universe_domain_opt = getattr(self._client_options, 'universe_domain', None) + + self._use_client_cert, self._use_mtls_endpoint, self._universe_domain_env = BigQueryWriteClient._read_environment_variables() + self._client_cert_source = BigQueryWriteClient._get_client_cert_source(self._client_options.client_cert_source, self._use_client_cert) + self._universe_domain = BigQueryWriteClient._get_universe_domain(universe_domain_opt, self._universe_domain_env) + self._api_endpoint = None # updated below, depending on `transport` + + # Initialize the universe domain validation. + self._is_universe_domain_valid = False + + api_key_value = getattr(self._client_options, "api_key", None) + if api_key_value and credentials: + raise ValueError("client_options.api_key and credentials are mutually exclusive") + + # Save or instantiate the transport. + # Ordinarily, we provide the transport, but allowing a custom transport + # instance provides an extensibility point for unusual situations. + transport_provided = isinstance(transport, BigQueryWriteTransport) + if transport_provided: + # transport is a BigQueryWriteTransport instance. + if credentials or self._client_options.credentials_file or api_key_value: + raise ValueError("When providing a transport instance, " + "provide its credentials directly.") + if self._client_options.scopes: + raise ValueError( + "When providing a transport instance, provide its scopes " + "directly." + ) + self._transport = cast(BigQueryWriteTransport, transport) + self._api_endpoint = self._transport.host + + self._api_endpoint = (self._api_endpoint or + BigQueryWriteClient._get_api_endpoint( + self._client_options.api_endpoint, + self._client_cert_source, + self._universe_domain, + self._use_mtls_endpoint)) + + if not transport_provided: + import google.auth._default # type: ignore + + if api_key_value and hasattr(google.auth._default, "get_api_key_credentials"): + credentials = google.auth._default.get_api_key_credentials(api_key_value) + + transport_init: Union[Type[BigQueryWriteTransport], Callable[..., BigQueryWriteTransport]] = ( + BigQueryWriteClient.get_transport_class(transport) + if isinstance(transport, str) or transport is None + else cast(Callable[..., BigQueryWriteTransport], transport) + ) + # initialize with the provided callable or the passed in class + self._transport = transport_init( + credentials=credentials, + credentials_file=self._client_options.credentials_file, + host=self._api_endpoint, + scopes=self._client_options.scopes, + client_cert_source_for_mtls=self._client_cert_source, + quota_project_id=self._client_options.quota_project_id, + client_info=client_info, + always_use_jwt_access=True, + api_audience=self._client_options.api_audience, + ) + + def create_write_stream(self, + request: Optional[Union[storage.CreateWriteStreamRequest, dict]] = None, + *, + parent: Optional[str] = None, + write_stream: Optional[stream.WriteStream] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Creates a write stream to the given table. Additionally, every + table has a special COMMITTED stream named '_default' to which + data can be written. This stream doesn't need to be created + using CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + def sample_create_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.CreateWriteStreamRequest( + parent="parent_value", + ) + + # Make the request + response = client.create_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1beta2.types.CreateWriteStreamRequest, dict]): + The request object. Request message for ``CreateWriteStream``. + parent (str): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + write_stream (google.cloud.bigquery_storage_v1beta2.types.WriteStream): + Required. Stream to be created. + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + warnings.warn("BigQueryWriteClient.create_write_stream is deprecated", + DeprecationWarning) + + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent, write_stream]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.CreateWriteStreamRequest): + request = storage.CreateWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.create_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def append_rows(self, + requests: Optional[Iterator[storage.AppendRowsRequest]] = None, + *, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> Iterable[storage.AppendRowsResponse]: + r"""Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provids an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC stream. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains the offset at which the append happened. + Responses are received in the same order in which requests are + sent. There will be one response for each successful request. If + the ``offset`` is not set in response, it means append didn't + happen due to some errors. If one request fails, all the + subsequent requests will also fail until a success request is + made again. + + If the stream is of ``PENDING`` type, data will only be + available for read operations after the stream is committed. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + def sample_append_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.AppendRowsRequest( + write_stream="write_stream_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1beta2.AppendRowsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = client.append_rows(requests=request_generator()) + + # Handle the response + for response in stream: + print(response) + + Args: + requests (Iterator[google.cloud.bigquery_storage_v1beta2.types.AppendRowsRequest]): + The request object iterator. Request message for ``AppendRows``. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + Iterable[google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse]: + Response message for AppendRows. + """ + warnings.warn("BigQueryWriteClient.append_rows is deprecated", + DeprecationWarning) + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.append_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + requests, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def get_write_stream(self, + request: Optional[Union[storage.GetWriteStreamRequest, dict]] = None, + *, + name: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> stream.WriteStream: + r"""Gets a write stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + def sample_get_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.GetWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = client.get_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1beta2.types.GetWriteStreamRequest, dict]): + The request object. Request message for ``GetWriteStreamRequest``. + name (str): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.WriteStream: + Information about a single stream + that gets data inside the storage + system. + + """ + warnings.warn("BigQueryWriteClient.get_write_stream is deprecated", + DeprecationWarning) + + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.GetWriteStreamRequest): + request = storage.GetWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.get_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def finalize_write_stream(self, + request: Optional[Union[storage.FinalizeWriteStreamRequest, dict]] = None, + *, + name: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FinalizeWriteStreamResponse: + r"""Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + def sample_finalize_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.FinalizeWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = client.finalize_write_stream(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamRequest, dict]): + The request object. Request message for invoking ``FinalizeWriteStream``. + name (str): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + + This corresponds to the ``name`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamResponse: + Response message for FinalizeWriteStream. + """ + warnings.warn("BigQueryWriteClient.finalize_write_stream is deprecated", + DeprecationWarning) + + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([name]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.FinalizeWriteStreamRequest): + request = storage.FinalizeWriteStreamRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if name is not None: + request.name = name + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.finalize_write_stream] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("name", request.name), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def batch_commit_write_streams(self, + request: Optional[Union[storage.BatchCommitWriteStreamsRequest, dict]] = None, + *, + parent: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.BatchCommitWriteStreamsResponse: + r"""Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. Streams must be finalized before + commit and cannot be committed multiple times. Once a stream is + committed, data in the stream becomes available for read + operations. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + def sample_batch_commit_write_streams(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.BatchCommitWriteStreamsRequest( + parent="parent_value", + write_streams=['write_streams_value1', 'write_streams_value2'], + ) + + # Make the request + response = client.batch_commit_write_streams(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsRequest, dict]): + The request object. Request message for ``BatchCommitWriteStreams``. + parent (str): + Required. Parent table that all the streams should + belong to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + + This corresponds to the ``parent`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsResponse: + Response message for BatchCommitWriteStreams. + """ + warnings.warn("BigQueryWriteClient.batch_commit_write_streams is deprecated", + DeprecationWarning) + + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([parent]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.BatchCommitWriteStreamsRequest): + request = storage.BatchCommitWriteStreamsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if parent is not None: + request.parent = parent + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.batch_commit_write_streams] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("parent", request.parent), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def flush_rows(self, + request: Optional[Union[storage.FlushRowsRequest, dict]] = None, + *, + write_stream: Optional[str] = None, + retry: OptionalRetry = gapic_v1.method.DEFAULT, + timeout: Union[float, object] = gapic_v1.method.DEFAULT, + metadata: Sequence[Tuple[str, str]] = (), + ) -> storage.FlushRowsResponse: + r"""Flushes rows to a BUFFERED stream. If users are appending rows + to BUFFERED stream, flush operation is required in order for the + rows to become available for reading. A Flush operation flushes + up to any previously flushed offset in a BUFFERED stream, to the + offset specified in the request. Flush is not supported on the + \_default stream, since it is not BUFFERED. + + .. code-block:: python + + # This snippet has been automatically generated and should be regarded as a + # code template only. + # It will require modifications to work: + # - It may require correct/in-range values for request initialization. + # - It may require specifying regional endpoints when creating the service + # client as shown in: + # https://googleapis.dev/python/google-api-core/latest/client_options.html + from google.cloud import bigquery_storage_v1beta2 + + def sample_flush_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.FlushRowsRequest( + write_stream="write_stream_value", + ) + + # Make the request + response = client.flush_rows(request=request) + + # Handle the response + print(response) + + Args: + request (Union[google.cloud.bigquery_storage_v1beta2.types.FlushRowsRequest, dict]): + The request object. Request message for ``FlushRows``. + write_stream (str): + Required. The stream that is the + target of the flush operation. + + This corresponds to the ``write_stream`` field + on the ``request`` instance; if ``request`` is provided, this + should not be set. + retry (google.api_core.retry.Retry): Designation of what errors, if any, + should be retried. + timeout (float): The timeout for this request. + metadata (Sequence[Tuple[str, str]]): Strings which should be + sent along with the request as metadata. + + Returns: + google.cloud.bigquery_storage_v1beta2.types.FlushRowsResponse: + Respond message for FlushRows. + """ + warnings.warn("BigQueryWriteClient.flush_rows is deprecated", + DeprecationWarning) + + # Create or coerce a protobuf request object. + # - Quick check: If we got a request object, we should *not* have + # gotten any keyword arguments that map to the request. + has_flattened_params = any([write_stream]) + if request is not None and has_flattened_params: + raise ValueError('If the `request` argument is set, then none of ' + 'the individual field arguments should be set.') + + # - Use the request object if provided (there's no risk of modifying the input as + # there are no flattened fields), or create one. + if not isinstance(request, storage.FlushRowsRequest): + request = storage.FlushRowsRequest(request) + # If we have keyword arguments corresponding to fields on the + # request, apply these. + if write_stream is not None: + request.write_stream = write_stream + + # Wrap the RPC method; this adds retry and timeout information, + # and friendly error handling. + rpc = self._transport._wrapped_methods[self._transport.flush_rows] + + # Certain fields should be provided within the metadata header; + # add these here. + metadata = tuple(metadata) + ( + gapic_v1.routing_header.to_grpc_metadata(( + ("write_stream", request.write_stream), + )), + ) + + # Validate the universe domain. + self._validate_universe_domain() + + # Send the request. + response = rpc( + request, + retry=retry, + timeout=timeout, + metadata=metadata, + ) + + # Done; return the response. + return response + + def __enter__(self) -> "BigQueryWriteClient": + return self + + def __exit__(self, type, value, traceback): + """Releases underlying transport's resources. + + .. warning:: + ONLY use as a context manager if the transport is NOT shared + with other clients! Exiting the with block will CLOSE the transport + and may cause errors in other clients! + """ + self.transport.close() + + + + + + + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +__all__ = ( + "BigQueryWriteClient", +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/README.rst b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/README.rst new file mode 100644 index 00000000..d2db1fed --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/README.rst @@ -0,0 +1,9 @@ + +transport inheritance structure +_______________________________ + +`BigQueryWriteTransport` is the ABC for all transports. +- public child `BigQueryWriteGrpcTransport` for sync gRPC transport (defined in `grpc.py`). +- public child `BigQueryWriteGrpcAsyncIOTransport` for async gRPC transport (defined in `grpc_asyncio.py`). +- private child `_BaseBigQueryWriteRestTransport` for base REST transport with inner classes `_BaseMETHOD` (defined in `rest_base.py`). +- public child `BigQueryWriteRestTransport` for sync REST transport with inner classes `METHOD` derived from the parent's corresponding `_BaseMETHOD` classes (defined in `rest.py`). diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/__init__.py new file mode 100644 index 00000000..56e33fc5 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/__init__.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from collections import OrderedDict +from typing import Dict, Type + +from .base import BigQueryWriteTransport +from .grpc import BigQueryWriteGrpcTransport +from .grpc_asyncio import BigQueryWriteGrpcAsyncIOTransport + + +# Compile a registry of transports. +_transport_registry = OrderedDict() # type: Dict[str, Type[BigQueryWriteTransport]] +_transport_registry['grpc'] = BigQueryWriteGrpcTransport +_transport_registry['grpc_asyncio'] = BigQueryWriteGrpcAsyncIOTransport + +__all__ = ( + 'BigQueryWriteTransport', + 'BigQueryWriteGrpcTransport', + 'BigQueryWriteGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py new file mode 100644 index 00000000..d24d1854 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/base.py @@ -0,0 +1,288 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import abc +from typing import Awaitable, Callable, Dict, Optional, Sequence, Union + +from google.cloud.bigquery_storage_v1beta2 import gapic_version as package_version + +import google.auth # type: ignore +import google.api_core +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.oauth2 import service_account # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream + +DEFAULT_CLIENT_INFO = gapic_v1.client_info.ClientInfo(gapic_version=package_version.__version__) + + +class BigQueryWriteTransport(abc.ABC): + """Abstract transport class for BigQueryWrite.""" + + AUTH_SCOPES = ( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', + ) + + DEFAULT_HOST: str = 'bigquerystorage.googleapis.com' + def __init__( + self, *, + host: str = DEFAULT_HOST, + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + **kwargs, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A list of scopes. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + """ + + scopes_kwargs = {"scopes": scopes, "default_scopes": self.AUTH_SCOPES} + + # Save the scopes. + self._scopes = scopes + if not hasattr(self, "_ignore_credentials"): + self._ignore_credentials: bool = False + + # If no credentials are provided, then determine the appropriate + # defaults. + if credentials and credentials_file: + raise core_exceptions.DuplicateCredentialArgs("'credentials_file' and 'credentials' are mutually exclusive") + + if credentials_file is not None: + credentials, _ = google.auth.load_credentials_from_file( + credentials_file, + **scopes_kwargs, + quota_project_id=quota_project_id + ) + elif credentials is None and not self._ignore_credentials: + credentials, _ = google.auth.default(**scopes_kwargs, quota_project_id=quota_project_id) + # Don't apply audience if the credentials file passed from user. + if hasattr(credentials, "with_gdch_audience"): + credentials = credentials.with_gdch_audience(api_audience if api_audience else host) + + # If the credentials are service account credentials, then always try to use self signed JWT. + if always_use_jwt_access and isinstance(credentials, service_account.Credentials) and hasattr(service_account.Credentials, "with_always_use_jwt_access"): + credentials = credentials.with_always_use_jwt_access(True) + + # Save the credentials. + self._credentials = credentials + + # Save the hostname. Default to port 443 (HTTPS) if none is specified. + if ':' not in host: + host += ':443' + self._host = host + + @property + def host(self): + return self._host + + def _prep_wrapped_messages(self, client_info): + # Precompute the wrapped methods. + self._wrapped_methods = { + self.create_write_stream: gapic_v1.method.wrap_method( + self.create_write_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.append_rows: gapic_v1.method.wrap_method( + self.append_rows, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.get_write_stream: gapic_v1.method.wrap_method( + self.get_write_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.finalize_write_stream: gapic_v1.method.wrap_method( + self.finalize_write_stream, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.batch_commit_write_streams: gapic_v1.method.wrap_method( + self.batch_commit_write_streams, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.flush_rows: gapic_v1.method.wrap_method( + self.flush_rows, + default_retry=retries.Retry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + def close(self): + """Closes resources associated with the transport. + + .. warning:: + Only call this method if the transport is NOT shared + with other clients - this may cause errors in other clients! + """ + raise NotImplementedError() + + @property + def create_write_stream(self) -> Callable[ + [storage.CreateWriteStreamRequest], + Union[ + stream.WriteStream, + Awaitable[stream.WriteStream] + ]]: + raise NotImplementedError() + + @property + def append_rows(self) -> Callable[ + [storage.AppendRowsRequest], + Union[ + storage.AppendRowsResponse, + Awaitable[storage.AppendRowsResponse] + ]]: + raise NotImplementedError() + + @property + def get_write_stream(self) -> Callable[ + [storage.GetWriteStreamRequest], + Union[ + stream.WriteStream, + Awaitable[stream.WriteStream] + ]]: + raise NotImplementedError() + + @property + def finalize_write_stream(self) -> Callable[ + [storage.FinalizeWriteStreamRequest], + Union[ + storage.FinalizeWriteStreamResponse, + Awaitable[storage.FinalizeWriteStreamResponse] + ]]: + raise NotImplementedError() + + @property + def batch_commit_write_streams(self) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + Union[ + storage.BatchCommitWriteStreamsResponse, + Awaitable[storage.BatchCommitWriteStreamsResponse] + ]]: + raise NotImplementedError() + + @property + def flush_rows(self) -> Callable[ + [storage.FlushRowsRequest], + Union[ + storage.FlushRowsResponse, + Awaitable[storage.FlushRowsResponse] + ]]: + raise NotImplementedError() + + @property + def kind(self) -> str: + raise NotImplementedError() + + +__all__ = ( + 'BigQueryWriteTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py new file mode 100644 index 00000000..96392a7d --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc.py @@ -0,0 +1,444 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import warnings +from typing import Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import grpc_helpers +from google.api_core import gapic_v1 +import google.auth # type: ignore +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from .base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO + + +class BigQueryWriteGrpcTransport(BigQueryWriteTransport): + """gRPC backend transport for BigQueryWrite. + + BigQuery Write API. + + The Write API can be used to write data to BigQuery. + + The `google.cloud.bigquery.storage.v1 + API `__ + should be used instead of the v1beta2 API for BigQueryWrite + operations. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + _stubs: Dict[str, Callable] + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]] = None, + api_mtls_endpoint: Optional[str] = None, + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + ssl_channel_credentials: Optional[grpc.ChannelCredentials] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if a ``channel`` instance is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if a ``channel`` instance is provided. + scopes (Optional(Sequence[str])): A list of scopes. This argument is + ignored if a ``channel`` instance is provided. + channel (Optional[Union[grpc.Channel, Callable[..., grpc.Channel]]]): + A ``Channel`` instance through which to make calls, or a Callable + that constructs and returns one. If set to None, ``self.create_channel`` + is used to create the channel. If a Callable is given, it will be called + with the same arguments as used in ``self.create_channel``. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or application default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for the grpc channel. It is ignored if a ``channel`` instance is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if a ``channel`` instance or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTLSChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if isinstance(channel, grpc.Channel): + # Ignore credentials if a channel was passed. + credentials = None + self._ignore_credentials = True + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, + ) + + if not self._grpc_channel: + # initialize with the provided callable or the default channel + channel_init = channel or type(self).create_channel + self._grpc_channel = channel_init( + self._host, + # use the credentials which are saved + credentials=self._credentials, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._prep_wrapped_messages(client_info) + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> grpc.Channel: + """Create and return a gRPC channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is mutually exclusive with credentials. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + grpc.Channel: A gRPC channel object. + + Raises: + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + + return grpc_helpers.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + @property + def grpc_channel(self) -> grpc.Channel: + """Return the channel designed to connect to this service. + """ + return self._grpc_channel + + @property + def create_write_stream(self) -> Callable[ + [storage.CreateWriteStreamRequest], + stream.WriteStream]: + r"""Return a callable for the create write stream method over gRPC. + + Creates a write stream to the given table. Additionally, every + table has a special COMMITTED stream named '_default' to which + data can be written. This stream doesn't need to be created + using CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Returns: + Callable[[~.CreateWriteStreamRequest], + ~.WriteStream]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_write_stream' not in self._stubs: + self._stubs['create_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/CreateWriteStream', + request_serializer=storage.CreateWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['create_write_stream'] + + @property + def append_rows(self) -> Callable[ + [storage.AppendRowsRequest], + storage.AppendRowsResponse]: + r"""Return a callable for the append rows method over gRPC. + + Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provids an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC stream. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains the offset at which the append happened. + Responses are received in the same order in which requests are + sent. There will be one response for each successful request. If + the ``offset`` is not set in response, it means append didn't + happen due to some errors. If one request fails, all the + subsequent requests will also fail until a success request is + made again. + + If the stream is of ``PENDING`` type, data will only be + available for read operations after the stream is committed. + + Returns: + Callable[[~.AppendRowsRequest], + ~.AppendRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'append_rows' not in self._stubs: + self._stubs['append_rows'] = self.grpc_channel.stream_stream( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/AppendRows', + request_serializer=storage.AppendRowsRequest.serialize, + response_deserializer=storage.AppendRowsResponse.deserialize, + ) + return self._stubs['append_rows'] + + @property + def get_write_stream(self) -> Callable[ + [storage.GetWriteStreamRequest], + stream.WriteStream]: + r"""Return a callable for the get write stream method over gRPC. + + Gets a write stream. + + Returns: + Callable[[~.GetWriteStreamRequest], + ~.WriteStream]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'get_write_stream' not in self._stubs: + self._stubs['get_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/GetWriteStream', + request_serializer=storage.GetWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['get_write_stream'] + + @property + def finalize_write_stream(self) -> Callable[ + [storage.FinalizeWriteStreamRequest], + storage.FinalizeWriteStreamResponse]: + r"""Return a callable for the finalize write stream method over gRPC. + + Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Returns: + Callable[[~.FinalizeWriteStreamRequest], + ~.FinalizeWriteStreamResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'finalize_write_stream' not in self._stubs: + self._stubs['finalize_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FinalizeWriteStream', + request_serializer=storage.FinalizeWriteStreamRequest.serialize, + response_deserializer=storage.FinalizeWriteStreamResponse.deserialize, + ) + return self._stubs['finalize_write_stream'] + + @property + def batch_commit_write_streams(self) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + storage.BatchCommitWriteStreamsResponse]: + r"""Return a callable for the batch commit write streams method over gRPC. + + Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. Streams must be finalized before + commit and cannot be committed multiple times. Once a stream is + committed, data in the stream becomes available for read + operations. + + Returns: + Callable[[~.BatchCommitWriteStreamsRequest], + ~.BatchCommitWriteStreamsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_commit_write_streams' not in self._stubs: + self._stubs['batch_commit_write_streams'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/BatchCommitWriteStreams', + request_serializer=storage.BatchCommitWriteStreamsRequest.serialize, + response_deserializer=storage.BatchCommitWriteStreamsResponse.deserialize, + ) + return self._stubs['batch_commit_write_streams'] + + @property + def flush_rows(self) -> Callable[ + [storage.FlushRowsRequest], + storage.FlushRowsResponse]: + r"""Return a callable for the flush rows method over gRPC. + + Flushes rows to a BUFFERED stream. If users are appending rows + to BUFFERED stream, flush operation is required in order for the + rows to become available for reading. A Flush operation flushes + up to any previously flushed offset in a BUFFERED stream, to the + offset specified in the request. Flush is not supported on the + \_default stream, since it is not BUFFERED. + + Returns: + Callable[[~.FlushRowsRequest], + ~.FlushRowsResponse]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'flush_rows' not in self._stubs: + self._stubs['flush_rows'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FlushRows', + request_serializer=storage.FlushRowsRequest.serialize, + response_deserializer=storage.FlushRowsResponse.deserialize, + ) + return self._stubs['flush_rows'] + + def close(self): + self.grpc_channel.close() + + @property + def kind(self) -> str: + return "grpc" + + +__all__ = ( + 'BigQueryWriteGrpcTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py new file mode 100644 index 00000000..9c1a2634 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/services/big_query_write/transports/grpc_asyncio.py @@ -0,0 +1,551 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import inspect +import warnings +from typing import Awaitable, Callable, Dict, Optional, Sequence, Tuple, Union + +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers_async +from google.api_core import exceptions as core_exceptions +from google.api_core import retry_async as retries +from google.auth import credentials as ga_credentials # type: ignore +from google.auth.transport.grpc import SslCredentials # type: ignore + +import grpc # type: ignore +from grpc.experimental import aio # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from .base import BigQueryWriteTransport, DEFAULT_CLIENT_INFO +from .grpc import BigQueryWriteGrpcTransport + + +class BigQueryWriteGrpcAsyncIOTransport(BigQueryWriteTransport): + """gRPC AsyncIO backend transport for BigQueryWrite. + + BigQuery Write API. + + The Write API can be used to write data to BigQuery. + + The `google.cloud.bigquery.storage.v1 + API `__ + should be used instead of the v1beta2 API for BigQueryWrite + operations. + + This class defines the same methods as the primary client, so the + primary client can load the underlying transport implementation + and call it. + + It sends protocol buffers over the wire using gRPC (which is built on + top of HTTP/2); the ``grpcio`` package must be installed. + """ + + _grpc_channel: aio.Channel + _stubs: Dict[str, Callable] = {} + + @classmethod + def create_channel(cls, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + quota_project_id: Optional[str] = None, + **kwargs) -> aio.Channel: + """Create and return a gRPC AsyncIO channel object. + Args: + host (Optional[str]): The host for the channel to use. + credentials (Optional[~.Credentials]): The + authorization credentials to attach to requests. These + credentials identify this application to the service. If + none are specified, the client will attempt to ascertain + the credentials from the environment. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + kwargs (Optional[dict]): Keyword arguments, which are passed to the + channel creation. + Returns: + aio.Channel: A gRPC AsyncIO channel object. + """ + + return grpc_helpers_async.create_channel( + host, + credentials=credentials, + credentials_file=credentials_file, + quota_project_id=quota_project_id, + default_scopes=cls.AUTH_SCOPES, + scopes=scopes, + default_host=cls.DEFAULT_HOST, + **kwargs + ) + + def __init__(self, *, + host: str = 'bigquerystorage.googleapis.com', + credentials: Optional[ga_credentials.Credentials] = None, + credentials_file: Optional[str] = None, + scopes: Optional[Sequence[str]] = None, + channel: Optional[Union[aio.Channel, Callable[..., aio.Channel]]] = None, + api_mtls_endpoint: Optional[str] = None, + client_cert_source: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + ssl_channel_credentials: Optional[grpc.ChannelCredentials] = None, + client_cert_source_for_mtls: Optional[Callable[[], Tuple[bytes, bytes]]] = None, + quota_project_id: Optional[str] = None, + client_info: gapic_v1.client_info.ClientInfo = DEFAULT_CLIENT_INFO, + always_use_jwt_access: Optional[bool] = False, + api_audience: Optional[str] = None, + ) -> None: + """Instantiate the transport. + + Args: + host (Optional[str]): + The hostname to connect to (default: 'bigquerystorage.googleapis.com'). + credentials (Optional[google.auth.credentials.Credentials]): The + authorization credentials to attach to requests. These + credentials identify the application to the service; if none + are specified, the client will attempt to ascertain the + credentials from the environment. + This argument is ignored if a ``channel`` instance is provided. + credentials_file (Optional[str]): A file with credentials that can + be loaded with :func:`google.auth.load_credentials_from_file`. + This argument is ignored if a ``channel`` instance is provided. + scopes (Optional[Sequence[str]]): A optional list of scopes needed for this + service. These are only used when credentials are not specified and + are passed to :func:`google.auth.default`. + channel (Optional[Union[aio.Channel, Callable[..., aio.Channel]]]): + A ``Channel`` instance through which to make calls, or a Callable + that constructs and returns one. If set to None, ``self.create_channel`` + is used to create the channel. If a Callable is given, it will be called + with the same arguments as used in ``self.create_channel``. + api_mtls_endpoint (Optional[str]): Deprecated. The mutual TLS endpoint. + If provided, it overrides the ``host`` argument and tries to create + a mutual TLS channel with client SSL credentials from + ``client_cert_source`` or application default SSL credentials. + client_cert_source (Optional[Callable[[], Tuple[bytes, bytes]]]): + Deprecated. A callback to provide client SSL certificate bytes and + private key bytes, both in PEM format. It is ignored if + ``api_mtls_endpoint`` is None. + ssl_channel_credentials (grpc.ChannelCredentials): SSL credentials + for the grpc channel. It is ignored if a ``channel`` instance is provided. + client_cert_source_for_mtls (Optional[Callable[[], Tuple[bytes, bytes]]]): + A callback to provide client certificate bytes and private key bytes, + both in PEM format. It is used to configure a mutual TLS channel. It is + ignored if a ``channel`` instance or ``ssl_channel_credentials`` is provided. + quota_project_id (Optional[str]): An optional project to use for billing + and quota. + client_info (google.api_core.gapic_v1.client_info.ClientInfo): + The client info used to send a user-agent string along with + API requests. If ``None``, then default info will be used. + Generally, you only need to set this if you're developing + your own client library. + always_use_jwt_access (Optional[bool]): Whether self signed JWT should + be used for service account credentials. + + Raises: + google.auth.exceptions.MutualTlsChannelError: If mutual TLS transport + creation failed for any reason. + google.api_core.exceptions.DuplicateCredentialArgs: If both ``credentials`` + and ``credentials_file`` are passed. + """ + self._grpc_channel = None + self._ssl_channel_credentials = ssl_channel_credentials + self._stubs: Dict[str, Callable] = {} + + if api_mtls_endpoint: + warnings.warn("api_mtls_endpoint is deprecated", DeprecationWarning) + if client_cert_source: + warnings.warn("client_cert_source is deprecated", DeprecationWarning) + + if isinstance(channel, aio.Channel): + # Ignore credentials if a channel was passed. + credentials = None + self._ignore_credentials = True + # If a channel was explicitly provided, set it. + self._grpc_channel = channel + self._ssl_channel_credentials = None + else: + if api_mtls_endpoint: + host = api_mtls_endpoint + + # Create SSL credentials with client_cert_source or application + # default SSL credentials. + if client_cert_source: + cert, key = client_cert_source() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + else: + self._ssl_channel_credentials = SslCredentials().ssl_credentials + + else: + if client_cert_source_for_mtls and not ssl_channel_credentials: + cert, key = client_cert_source_for_mtls() + self._ssl_channel_credentials = grpc.ssl_channel_credentials( + certificate_chain=cert, private_key=key + ) + + # The base transport sets the host, credentials and scopes + super().__init__( + host=host, + credentials=credentials, + credentials_file=credentials_file, + scopes=scopes, + quota_project_id=quota_project_id, + client_info=client_info, + always_use_jwt_access=always_use_jwt_access, + api_audience=api_audience, + ) + + if not self._grpc_channel: + # initialize with the provided callable or the default channel + channel_init = channel or type(self).create_channel + self._grpc_channel = channel_init( + self._host, + # use the credentials which are saved + credentials=self._credentials, + # Set ``credentials_file`` to ``None`` here as + # the credentials that we saved earlier should be used. + credentials_file=None, + scopes=self._scopes, + ssl_credentials=self._ssl_channel_credentials, + quota_project_id=quota_project_id, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Wrap messages. This must be done after self._grpc_channel exists + self._wrap_with_kind = "kind" in inspect.signature(gapic_v1.method_async.wrap_method).parameters + self._prep_wrapped_messages(client_info) + + @property + def grpc_channel(self) -> aio.Channel: + """Create the channel designed to connect to this service. + + This property caches on the instance; repeated calls return + the same channel. + """ + # Return the channel from cache. + return self._grpc_channel + + @property + def create_write_stream(self) -> Callable[ + [storage.CreateWriteStreamRequest], + Awaitable[stream.WriteStream]]: + r"""Return a callable for the create write stream method over gRPC. + + Creates a write stream to the given table. Additionally, every + table has a special COMMITTED stream named '_default' to which + data can be written. This stream doesn't need to be created + using CreateWriteStream. It is a stream that can be used + simultaneously by any number of clients. Data written to this + stream is considered committed as soon as an acknowledgement is + received. + + Returns: + Callable[[~.CreateWriteStreamRequest], + Awaitable[~.WriteStream]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'create_write_stream' not in self._stubs: + self._stubs['create_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/CreateWriteStream', + request_serializer=storage.CreateWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['create_write_stream'] + + @property + def append_rows(self) -> Callable[ + [storage.AppendRowsRequest], + Awaitable[storage.AppendRowsResponse]]: + r"""Return a callable for the append rows method over gRPC. + + Appends data to the given stream. + + If ``offset`` is specified, the ``offset`` is checked against + the end of stream. The server returns ``OUT_OF_RANGE`` in + ``AppendRowsResponse`` if an attempt is made to append to an + offset beyond the current end of the stream or + ``ALREADY_EXISTS`` if user provids an ``offset`` that has + already been written to. User can retry with adjusted offset + within the same RPC stream. If ``offset`` is not specified, + append happens at the end of the stream. + + The response contains the offset at which the append happened. + Responses are received in the same order in which requests are + sent. There will be one response for each successful request. If + the ``offset`` is not set in response, it means append didn't + happen due to some errors. If one request fails, all the + subsequent requests will also fail until a success request is + made again. + + If the stream is of ``PENDING`` type, data will only be + available for read operations after the stream is committed. + + Returns: + Callable[[~.AppendRowsRequest], + Awaitable[~.AppendRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'append_rows' not in self._stubs: + self._stubs['append_rows'] = self.grpc_channel.stream_stream( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/AppendRows', + request_serializer=storage.AppendRowsRequest.serialize, + response_deserializer=storage.AppendRowsResponse.deserialize, + ) + return self._stubs['append_rows'] + + @property + def get_write_stream(self) -> Callable[ + [storage.GetWriteStreamRequest], + Awaitable[stream.WriteStream]]: + r"""Return a callable for the get write stream method over gRPC. + + Gets a write stream. + + Returns: + Callable[[~.GetWriteStreamRequest], + Awaitable[~.WriteStream]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'get_write_stream' not in self._stubs: + self._stubs['get_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/GetWriteStream', + request_serializer=storage.GetWriteStreamRequest.serialize, + response_deserializer=stream.WriteStream.deserialize, + ) + return self._stubs['get_write_stream'] + + @property + def finalize_write_stream(self) -> Callable[ + [storage.FinalizeWriteStreamRequest], + Awaitable[storage.FinalizeWriteStreamResponse]]: + r"""Return a callable for the finalize write stream method over gRPC. + + Finalize a write stream so that no new data can be appended to + the stream. Finalize is not supported on the '_default' stream. + + Returns: + Callable[[~.FinalizeWriteStreamRequest], + Awaitable[~.FinalizeWriteStreamResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'finalize_write_stream' not in self._stubs: + self._stubs['finalize_write_stream'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FinalizeWriteStream', + request_serializer=storage.FinalizeWriteStreamRequest.serialize, + response_deserializer=storage.FinalizeWriteStreamResponse.deserialize, + ) + return self._stubs['finalize_write_stream'] + + @property + def batch_commit_write_streams(self) -> Callable[ + [storage.BatchCommitWriteStreamsRequest], + Awaitable[storage.BatchCommitWriteStreamsResponse]]: + r"""Return a callable for the batch commit write streams method over gRPC. + + Atomically commits a group of ``PENDING`` streams that belong to + the same ``parent`` table. Streams must be finalized before + commit and cannot be committed multiple times. Once a stream is + committed, data in the stream becomes available for read + operations. + + Returns: + Callable[[~.BatchCommitWriteStreamsRequest], + Awaitable[~.BatchCommitWriteStreamsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'batch_commit_write_streams' not in self._stubs: + self._stubs['batch_commit_write_streams'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/BatchCommitWriteStreams', + request_serializer=storage.BatchCommitWriteStreamsRequest.serialize, + response_deserializer=storage.BatchCommitWriteStreamsResponse.deserialize, + ) + return self._stubs['batch_commit_write_streams'] + + @property + def flush_rows(self) -> Callable[ + [storage.FlushRowsRequest], + Awaitable[storage.FlushRowsResponse]]: + r"""Return a callable for the flush rows method over gRPC. + + Flushes rows to a BUFFERED stream. If users are appending rows + to BUFFERED stream, flush operation is required in order for the + rows to become available for reading. A Flush operation flushes + up to any previously flushed offset in a BUFFERED stream, to the + offset specified in the request. Flush is not supported on the + \_default stream, since it is not BUFFERED. + + Returns: + Callable[[~.FlushRowsRequest], + Awaitable[~.FlushRowsResponse]]: + A function that, when called, will call the underlying RPC + on the server. + """ + # Generate a "stub function" on-the-fly which will actually make + # the request. + # gRPC handles serialization and deserialization, so we just need + # to pass in the functions for each. + if 'flush_rows' not in self._stubs: + self._stubs['flush_rows'] = self.grpc_channel.unary_unary( + '/google.cloud.bigquery.storage.v1beta2.BigQueryWrite/FlushRows', + request_serializer=storage.FlushRowsRequest.serialize, + response_deserializer=storage.FlushRowsResponse.deserialize, + ) + return self._stubs['flush_rows'] + + def _prep_wrapped_messages(self, client_info): + """ Precompute the wrapped methods, overriding the base class method to use async wrappers.""" + self._wrapped_methods = { + self.create_write_stream: self._wrap_method( + self.create_write_stream, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.append_rows: self._wrap_method( + self.append_rows, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.ResourceExhausted, + core_exceptions.ServiceUnavailable, + ), + deadline=86400.0, + ), + default_timeout=86400.0, + client_info=client_info, + ), + self.get_write_stream: self._wrap_method( + self.get_write_stream, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.finalize_write_stream: self._wrap_method( + self.finalize_write_stream, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.batch_commit_write_streams: self._wrap_method( + self.batch_commit_write_streams, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + self.flush_rows: self._wrap_method( + self.flush_rows, + default_retry=retries.AsyncRetry( + initial=0.1, + maximum=60.0, + multiplier=1.3, + predicate=retries.if_exception_type( + core_exceptions.DeadlineExceeded, + core_exceptions.ServiceUnavailable, + ), + deadline=600.0, + ), + default_timeout=600.0, + client_info=client_info, + ), + } + + def _wrap_method(self, func, *args, **kwargs): + if self._wrap_with_kind: # pragma: NO COVER + kwargs["kind"] = self.kind + return gapic_v1.method_async.wrap_method(func, *args, **kwargs) + + def close(self): + return self.grpc_channel.close() + + @property + def kind(self) -> str: + return "grpc_asyncio" + + +__all__ = ( + 'BigQueryWriteGrpcAsyncIOTransport', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/__init__.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/__init__.py new file mode 100644 index 00000000..239a1c7d --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/__init__.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from .arrow import ( + ArrowRecordBatch, + ArrowSchema, + ArrowSerializationOptions, +) +from .avro import ( + AvroRows, + AvroSchema, +) +from .protobuf import ( + ProtoRows, + ProtoSchema, +) +from .storage import ( + AppendRowsRequest, + AppendRowsResponse, + BatchCommitWriteStreamsRequest, + BatchCommitWriteStreamsResponse, + CreateReadSessionRequest, + CreateWriteStreamRequest, + FinalizeWriteStreamRequest, + FinalizeWriteStreamResponse, + FlushRowsRequest, + FlushRowsResponse, + GetWriteStreamRequest, + ReadRowsRequest, + ReadRowsResponse, + SplitReadStreamRequest, + SplitReadStreamResponse, + StorageError, + StreamStats, + ThrottleState, +) +from .stream import ( + ReadSession, + ReadStream, + WriteStream, + DataFormat, +) +from .table import ( + TableFieldSchema, + TableSchema, +) + +__all__ = ( + 'ArrowRecordBatch', + 'ArrowSchema', + 'ArrowSerializationOptions', + 'AvroRows', + 'AvroSchema', + 'ProtoRows', + 'ProtoSchema', + 'AppendRowsRequest', + 'AppendRowsResponse', + 'BatchCommitWriteStreamsRequest', + 'BatchCommitWriteStreamsResponse', + 'CreateReadSessionRequest', + 'CreateWriteStreamRequest', + 'FinalizeWriteStreamRequest', + 'FinalizeWriteStreamResponse', + 'FlushRowsRequest', + 'FlushRowsResponse', + 'GetWriteStreamRequest', + 'ReadRowsRequest', + 'ReadRowsResponse', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + 'StorageError', + 'StreamStats', + 'ThrottleState', + 'ReadSession', + 'ReadStream', + 'WriteStream', + 'DataFormat', + 'TableFieldSchema', + 'TableSchema', +) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/arrow.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/arrow.py new file mode 100644 index 00000000..5f93a831 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/arrow.py @@ -0,0 +1,99 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'ArrowSchema', + 'ArrowRecordBatch', + 'ArrowSerializationOptions', + }, +) + + +class ArrowSchema(proto.Message): + r"""Arrow schema as specified in + https://arrow.apache.org/docs/python/api/datatypes.html and + serialized to bytes using IPC: + + https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc + + See code samples on how this message can be deserialized. + + Attributes: + serialized_schema (bytes): + IPC serialized Arrow schema. + """ + + serialized_schema: bytes = proto.Field( + proto.BYTES, + number=1, + ) + + +class ArrowRecordBatch(proto.Message): + r"""Arrow RecordBatch. + + Attributes: + serialized_record_batch (bytes): + IPC-serialized Arrow RecordBatch. + """ + + serialized_record_batch: bytes = proto.Field( + proto.BYTES, + number=1, + ) + + +class ArrowSerializationOptions(proto.Message): + r"""Contains options specific to Arrow Serialization. + + Attributes: + format_ (google.cloud.bigquery_storage_v1beta2.types.ArrowSerializationOptions.Format): + The Arrow IPC format to use. + """ + class Format(proto.Enum): + r"""The IPC format to use when serializing Arrow streams. + + Values: + FORMAT_UNSPECIFIED (0): + If unspecied the IPC format as of 0.15 + release will be used. + ARROW_0_14 (1): + Use the legacy IPC message format as of + Apache Arrow Release 0.14. + ARROW_0_15 (2): + Use the message format as of Apache Arrow + Release 0.15. + """ + FORMAT_UNSPECIFIED = 0 + ARROW_0_14 = 1 + ARROW_0_15 = 2 + + format_: Format = proto.Field( + proto.ENUM, + number=1, + enum=Format, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/avro.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/avro.py new file mode 100644 index 00000000..662a19ea --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/avro.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'AvroSchema', + 'AvroRows', + }, +) + + +class AvroSchema(proto.Message): + r"""Avro schema. + + Attributes: + schema (str): + Json serialized schema, as described at + https://avro.apache.org/docs/1.8.1/spec.html. + """ + + schema: str = proto.Field( + proto.STRING, + number=1, + ) + + +class AvroRows(proto.Message): + r"""Avro rows. + + Attributes: + serialized_binary_rows (bytes): + Binary serialized rows in a block. + """ + + serialized_binary_rows: bytes = proto.Field( + proto.BYTES, + number=1, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/protobuf.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/protobuf.py new file mode 100644 index 00000000..baad21bf --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/protobuf.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + +from google.protobuf import descriptor_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'ProtoSchema', + 'ProtoRows', + }, +) + + +class ProtoSchema(proto.Message): + r"""ProtoSchema describes the schema of the serialized protocol + buffer data rows. + + Attributes: + proto_descriptor (google.protobuf.descriptor_pb2.DescriptorProto): + Descriptor for input message. The descriptor + has to be self contained, including all the + nested types, excepted for proto buffer well + known types + (https://developers.google.com/protocol-buffers/docs/reference/google.protobuf). + """ + + proto_descriptor: descriptor_pb2.DescriptorProto = proto.Field( + proto.MESSAGE, + number=1, + message=descriptor_pb2.DescriptorProto, + ) + + +class ProtoRows(proto.Message): + r""" + + Attributes: + serialized_rows (MutableSequence[bytes]): + A sequence of rows serialized as a Protocol + Buffer. + See + https://developers.google.com/protocol-buffers/docs/overview + for more information on deserializing this + field. + """ + + serialized_rows: MutableSequence[bytes] = proto.RepeatedField( + proto.BYTES, + number=1, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/storage.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/storage.py new file mode 100644 index 00000000..9ea0f79e --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/storage.py @@ -0,0 +1,692 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import arrow +from google.cloud.bigquery_storage_v1beta2.types import avro +from google.cloud.bigquery_storage_v1beta2.types import protobuf +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.cloud.bigquery_storage_v1beta2.types import table +from google.protobuf import timestamp_pb2 # type: ignore +from google.protobuf import wrappers_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'CreateReadSessionRequest', + 'ReadRowsRequest', + 'ThrottleState', + 'StreamStats', + 'ReadRowsResponse', + 'SplitReadStreamRequest', + 'SplitReadStreamResponse', + 'CreateWriteStreamRequest', + 'AppendRowsRequest', + 'AppendRowsResponse', + 'GetWriteStreamRequest', + 'BatchCommitWriteStreamsRequest', + 'BatchCommitWriteStreamsResponse', + 'FinalizeWriteStreamRequest', + 'FinalizeWriteStreamResponse', + 'FlushRowsRequest', + 'FlushRowsResponse', + 'StorageError', + }, +) + + +class CreateReadSessionRequest(proto.Message): + r"""Request message for ``CreateReadSession``. + + Attributes: + parent (str): + Required. The request project that owns the session, in the + form of ``projects/{project_id}``. + read_session (google.cloud.bigquery_storage_v1beta2.types.ReadSession): + Required. Session to be created. + max_stream_count (int): + Max initial number of streams. If unset or + zero, the server will provide a value of streams + so as to produce reasonable throughput. Must be + non-negative. The number of streams may be lower + than the requested number, depending on the + amount parallelism that is reasonable for the + table. Error will be returned if the max count + is greater than the current system max limit of + 1,000. + + Streams must be read starting from offset 0. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + read_session: stream.ReadSession = proto.Field( + proto.MESSAGE, + number=2, + message=stream.ReadSession, + ) + max_stream_count: int = proto.Field( + proto.INT32, + number=3, + ) + + +class ReadRowsRequest(proto.Message): + r"""Request message for ``ReadRows``. + + Attributes: + read_stream (str): + Required. Stream to read rows from. + offset (int): + The offset requested must be less than the + last row read from Read. Requesting a larger + offset is undefined. If not specified, start + reading from offset zero. + """ + + read_stream: str = proto.Field( + proto.STRING, + number=1, + ) + offset: int = proto.Field( + proto.INT64, + number=2, + ) + + +class ThrottleState(proto.Message): + r"""Information on if the current connection is being throttled. + + Attributes: + throttle_percent (int): + How much this connection is being throttled. + Zero means no throttling, 100 means fully + throttled. + """ + + throttle_percent: int = proto.Field( + proto.INT32, + number=1, + ) + + +class StreamStats(proto.Message): + r"""Estimated stream statistics for a given Stream. + + Attributes: + progress (google.cloud.bigquery_storage_v1beta2.types.StreamStats.Progress): + Represents the progress of the current + stream. + """ + + class Progress(proto.Message): + r""" + + Attributes: + at_response_start (float): + The fraction of rows assigned to the stream that have been + processed by the server so far, not including the rows in + the current response message. + + This value, along with ``at_response_end``, can be used to + interpolate the progress made as the rows in the message are + being processed using the following formula: + ``at_response_start + (at_response_end - at_response_start) * rows_processed_from_response / rows_in_response``. + + Note that if a filter is provided, the ``at_response_end`` + value of the previous response may not necessarily be equal + to the ``at_response_start`` value of the current response. + at_response_end (float): + Similar to ``at_response_start``, except that this value + includes the rows in the current response. + """ + + at_response_start: float = proto.Field( + proto.DOUBLE, + number=1, + ) + at_response_end: float = proto.Field( + proto.DOUBLE, + number=2, + ) + + progress: Progress = proto.Field( + proto.MESSAGE, + number=2, + message=Progress, + ) + + +class ReadRowsResponse(proto.Message): + r"""Response from calling ``ReadRows`` may include row data, progress + and throttling information. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + avro_rows (google.cloud.bigquery_storage_v1beta2.types.AvroRows): + Serialized row data in AVRO format. + + This field is a member of `oneof`_ ``rows``. + arrow_record_batch (google.cloud.bigquery_storage_v1beta2.types.ArrowRecordBatch): + Serialized row data in Arrow RecordBatch + format. + + This field is a member of `oneof`_ ``rows``. + row_count (int): + Number of serialized rows in the rows block. + stats (google.cloud.bigquery_storage_v1beta2.types.StreamStats): + Statistics for the stream. + throttle_state (google.cloud.bigquery_storage_v1beta2.types.ThrottleState): + Throttling state. If unset, the latest + response still describes the current throttling + status. + avro_schema (google.cloud.bigquery_storage_v1beta2.types.AvroSchema): + Output only. Avro schema. + + This field is a member of `oneof`_ ``schema``. + arrow_schema (google.cloud.bigquery_storage_v1beta2.types.ArrowSchema): + Output only. Arrow schema. + + This field is a member of `oneof`_ ``schema``. + """ + + avro_rows: avro.AvroRows = proto.Field( + proto.MESSAGE, + number=3, + oneof='rows', + message=avro.AvroRows, + ) + arrow_record_batch: arrow.ArrowRecordBatch = proto.Field( + proto.MESSAGE, + number=4, + oneof='rows', + message=arrow.ArrowRecordBatch, + ) + row_count: int = proto.Field( + proto.INT64, + number=6, + ) + stats: 'StreamStats' = proto.Field( + proto.MESSAGE, + number=2, + message='StreamStats', + ) + throttle_state: 'ThrottleState' = proto.Field( + proto.MESSAGE, + number=5, + message='ThrottleState', + ) + avro_schema: avro.AvroSchema = proto.Field( + proto.MESSAGE, + number=7, + oneof='schema', + message=avro.AvroSchema, + ) + arrow_schema: arrow.ArrowSchema = proto.Field( + proto.MESSAGE, + number=8, + oneof='schema', + message=arrow.ArrowSchema, + ) + + +class SplitReadStreamRequest(proto.Message): + r"""Request message for ``SplitReadStream``. + + Attributes: + name (str): + Required. Name of the stream to split. + fraction (float): + A value in the range (0.0, 1.0) that + specifies the fractional point at which the + original stream should be split. The actual + split point is evaluated on pre-filtered rows, + so if a filter is provided, then there is no + guarantee that the division of the rows between + the new child streams will be proportional to + this fractional value. Additionally, because the + server-side unit for assigning data is + collections of rows, this fraction will always + map to a data storage boundary on the server + side. + """ + + name: str = proto.Field( + proto.STRING, + number=1, + ) + fraction: float = proto.Field( + proto.DOUBLE, + number=2, + ) + + +class SplitReadStreamResponse(proto.Message): + r""" + + Attributes: + primary_stream (google.cloud.bigquery_storage_v1beta2.types.ReadStream): + Primary stream, which contains the beginning portion of + \|original_stream|. An empty value indicates that the + original stream can no longer be split. + remainder_stream (google.cloud.bigquery_storage_v1beta2.types.ReadStream): + Remainder stream, which contains the tail of + \|original_stream|. An empty value indicates that the + original stream can no longer be split. + """ + + primary_stream: stream.ReadStream = proto.Field( + proto.MESSAGE, + number=1, + message=stream.ReadStream, + ) + remainder_stream: stream.ReadStream = proto.Field( + proto.MESSAGE, + number=2, + message=stream.ReadStream, + ) + + +class CreateWriteStreamRequest(proto.Message): + r"""Request message for ``CreateWriteStream``. + + Attributes: + parent (str): + Required. Reference to the table to which the stream + belongs, in the format of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + write_stream (google.cloud.bigquery_storage_v1beta2.types.WriteStream): + Required. Stream to be created. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + write_stream: stream.WriteStream = proto.Field( + proto.MESSAGE, + number=2, + message=stream.WriteStream, + ) + + +class AppendRowsRequest(proto.Message): + r"""Request message for ``AppendRows``. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + write_stream (str): + Required. The stream that is the target of the append + operation. This value must be specified for the initial + request. If subsequent requests specify the stream name, it + must equal to the value provided in the first request. To + write to the \_default stream, populate this field with a + string in the format + ``projects/{project}/datasets/{dataset}/tables/{table}/_default``. + offset (google.protobuf.wrappers_pb2.Int64Value): + If present, the write is only performed if the next append + offset is same as the provided value. If not present, the + write is performed at the current end of stream. Specifying + a value for this field is not allowed when calling + AppendRows for the '_default' stream. + proto_rows (google.cloud.bigquery_storage_v1beta2.types.AppendRowsRequest.ProtoData): + Rows in proto format. + + This field is a member of `oneof`_ ``rows``. + trace_id (str): + Id set by client to annotate its identity. + Only initial request setting is respected. + """ + + class ProtoData(proto.Message): + r"""Proto schema and data. + + Attributes: + writer_schema (google.cloud.bigquery_storage_v1beta2.types.ProtoSchema): + Proto schema used to serialize the data. + rows (google.cloud.bigquery_storage_v1beta2.types.ProtoRows): + Serialized row data in protobuf message + format. + """ + + writer_schema: protobuf.ProtoSchema = proto.Field( + proto.MESSAGE, + number=1, + message=protobuf.ProtoSchema, + ) + rows: protobuf.ProtoRows = proto.Field( + proto.MESSAGE, + number=2, + message=protobuf.ProtoRows, + ) + + write_stream: str = proto.Field( + proto.STRING, + number=1, + ) + offset: wrappers_pb2.Int64Value = proto.Field( + proto.MESSAGE, + number=2, + message=wrappers_pb2.Int64Value, + ) + proto_rows: ProtoData = proto.Field( + proto.MESSAGE, + number=4, + oneof='rows', + message=ProtoData, + ) + trace_id: str = proto.Field( + proto.STRING, + number=6, + ) + + +class AppendRowsResponse(proto.Message): + r"""Response message for ``AppendRows``. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + append_result (google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse.AppendResult): + Result if the append is successful. + + This field is a member of `oneof`_ ``response``. + error (google.rpc.status_pb2.Status): + Error returned when problems were encountered. If present, + it indicates rows were not accepted into the system. Users + can retry or continue with other append requests within the + same connection. + + Additional information about error signalling: + + ALREADY_EXISTS: Happens when an append specified an offset, + and the backend already has received data at this offset. + Typically encountered in retry scenarios, and can be + ignored. + + OUT_OF_RANGE: Returned when the specified offset in the + stream is beyond the current end of the stream. + + INVALID_ARGUMENT: Indicates a malformed request or data. + + ABORTED: Request processing is aborted because of prior + failures. The request can be retried if previous failure is + addressed. + + INTERNAL: Indicates server side error(s) that can be + retried. + + This field is a member of `oneof`_ ``response``. + updated_schema (google.cloud.bigquery_storage_v1beta2.types.TableSchema): + If backend detects a schema update, pass it + to user so that user can use it to input new + type of message. It will be empty when no schema + updates have occurred. + """ + + class AppendResult(proto.Message): + r"""AppendResult is returned for successful append requests. + + Attributes: + offset (google.protobuf.wrappers_pb2.Int64Value): + The row offset at which the last append + occurred. The offset will not be set if + appending using default streams. + """ + + offset: wrappers_pb2.Int64Value = proto.Field( + proto.MESSAGE, + number=1, + message=wrappers_pb2.Int64Value, + ) + + append_result: AppendResult = proto.Field( + proto.MESSAGE, + number=1, + oneof='response', + message=AppendResult, + ) + error: status_pb2.Status = proto.Field( + proto.MESSAGE, + number=2, + oneof='response', + message=status_pb2.Status, + ) + updated_schema: table.TableSchema = proto.Field( + proto.MESSAGE, + number=3, + message=table.TableSchema, + ) + + +class GetWriteStreamRequest(proto.Message): + r"""Request message for ``GetWriteStreamRequest``. + + Attributes: + name (str): + Required. Name of the stream to get, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + """ + + name: str = proto.Field( + proto.STRING, + number=1, + ) + + +class BatchCommitWriteStreamsRequest(proto.Message): + r"""Request message for ``BatchCommitWriteStreams``. + + Attributes: + parent (str): + Required. Parent table that all the streams should belong + to, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}``. + write_streams (MutableSequence[str]): + Required. The group of streams that will be + committed atomically. + """ + + parent: str = proto.Field( + proto.STRING, + number=1, + ) + write_streams: MutableSequence[str] = proto.RepeatedField( + proto.STRING, + number=2, + ) + + +class BatchCommitWriteStreamsResponse(proto.Message): + r"""Response message for ``BatchCommitWriteStreams``. + + Attributes: + commit_time (google.protobuf.timestamp_pb2.Timestamp): + The time at which streams were committed in microseconds + granularity. This field will only exist when there are no + stream errors. **Note** if this field is not set, it means + the commit was not successful. + stream_errors (MutableSequence[google.cloud.bigquery_storage_v1beta2.types.StorageError]): + Stream level error if commit failed. Only + streams with error will be in the list. + If empty, there is no error and all streams are + committed successfully. If non empty, certain + streams have errors and ZERO stream is committed + due to atomicity guarantee. + """ + + commit_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=1, + message=timestamp_pb2.Timestamp, + ) + stream_errors: MutableSequence['StorageError'] = proto.RepeatedField( + proto.MESSAGE, + number=2, + message='StorageError', + ) + + +class FinalizeWriteStreamRequest(proto.Message): + r"""Request message for invoking ``FinalizeWriteStream``. + + Attributes: + name (str): + Required. Name of the stream to finalize, in the form of + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + """ + + name: str = proto.Field( + proto.STRING, + number=1, + ) + + +class FinalizeWriteStreamResponse(proto.Message): + r"""Response message for ``FinalizeWriteStream``. + + Attributes: + row_count (int): + Number of rows in the finalized stream. + """ + + row_count: int = proto.Field( + proto.INT64, + number=1, + ) + + +class FlushRowsRequest(proto.Message): + r"""Request message for ``FlushRows``. + + Attributes: + write_stream (str): + Required. The stream that is the target of + the flush operation. + offset (google.protobuf.wrappers_pb2.Int64Value): + Ending offset of the flush operation. Rows + before this offset(including this offset) will + be flushed. + """ + + write_stream: str = proto.Field( + proto.STRING, + number=1, + ) + offset: wrappers_pb2.Int64Value = proto.Field( + proto.MESSAGE, + number=2, + message=wrappers_pb2.Int64Value, + ) + + +class FlushRowsResponse(proto.Message): + r"""Respond message for ``FlushRows``. + + Attributes: + offset (int): + The rows before this offset (including this + offset) are flushed. + """ + + offset: int = proto.Field( + proto.INT64, + number=1, + ) + + +class StorageError(proto.Message): + r"""Structured custom BigQuery Storage error message. The error + can be attached as error details in the returned rpc Status. In + particular, the use of error codes allows more structured error + handling, and reduces the need to evaluate unstructured error + text strings. + + Attributes: + code (google.cloud.bigquery_storage_v1beta2.types.StorageError.StorageErrorCode): + BigQuery Storage specific error code. + entity (str): + Name of the failed entity. + error_message (str): + Message that describes the error. + """ + class StorageErrorCode(proto.Enum): + r"""Error code for ``StorageError``. + + Values: + STORAGE_ERROR_CODE_UNSPECIFIED (0): + Default error. + TABLE_NOT_FOUND (1): + Table is not found in the system. + STREAM_ALREADY_COMMITTED (2): + Stream is already committed. + STREAM_NOT_FOUND (3): + Stream is not found. + INVALID_STREAM_TYPE (4): + Invalid Stream type. + For example, you try to commit a stream that is + not pending. + INVALID_STREAM_STATE (5): + Invalid Stream state. + For example, you try to commit a stream that is + not finalized or is garbaged. + STREAM_FINALIZED (6): + Stream is finalized. + """ + STORAGE_ERROR_CODE_UNSPECIFIED = 0 + TABLE_NOT_FOUND = 1 + STREAM_ALREADY_COMMITTED = 2 + STREAM_NOT_FOUND = 3 + INVALID_STREAM_TYPE = 4 + INVALID_STREAM_STATE = 5 + STREAM_FINALIZED = 6 + + code: StorageErrorCode = proto.Field( + proto.ENUM, + number=1, + enum=StorageErrorCode, + ) + entity: str = proto.Field( + proto.STRING, + number=2, + ) + error_message: str = proto.Field( + proto.STRING, + number=3, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/stream.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/stream.py new file mode 100644 index 00000000..8a6f19b3 --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/stream.py @@ -0,0 +1,298 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + +from google.cloud.bigquery_storage_v1beta2.types import arrow +from google.cloud.bigquery_storage_v1beta2.types import avro +from google.cloud.bigquery_storage_v1beta2.types import table as gcbs_table +from google.protobuf import timestamp_pb2 # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'DataFormat', + 'ReadSession', + 'ReadStream', + 'WriteStream', + }, +) + + +class DataFormat(proto.Enum): + r"""Data format for input or output data. + + Values: + DATA_FORMAT_UNSPECIFIED (0): + No description available. + AVRO (1): + Avro is a standard open source row based file + format. See https://avro.apache.org/ for more + details. + ARROW (2): + Arrow is a standard open source column-based + message format. See https://arrow.apache.org/ + for more details. + """ + DATA_FORMAT_UNSPECIFIED = 0 + AVRO = 1 + ARROW = 2 + + +class ReadSession(proto.Message): + r"""Information about the ReadSession. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + name (str): + Output only. Unique identifier for the session, in the form + ``projects/{project_id}/locations/{location}/sessions/{session_id}``. + expire_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Time at which the session becomes invalid. + After this time, subsequent requests to read this Session + will return errors. The expire_time is automatically + assigned and currently cannot be specified or updated. + data_format (google.cloud.bigquery_storage_v1beta2.types.DataFormat): + Immutable. Data format of the output data. + avro_schema (google.cloud.bigquery_storage_v1beta2.types.AvroSchema): + Output only. Avro schema. + + This field is a member of `oneof`_ ``schema``. + arrow_schema (google.cloud.bigquery_storage_v1beta2.types.ArrowSchema): + Output only. Arrow schema. + + This field is a member of `oneof`_ ``schema``. + table (str): + Immutable. Table that this ReadSession is reading from, in + the form + \`projects/{project_id}/datasets/{dataset_id}/tables/{table_id} + table_modifiers (google.cloud.bigquery_storage_v1beta2.types.ReadSession.TableModifiers): + Optional. Any modifiers which are applied + when reading from the specified table. + read_options (google.cloud.bigquery_storage_v1beta2.types.ReadSession.TableReadOptions): + Optional. Read options for this session (e.g. + column selection, filters). + streams (MutableSequence[google.cloud.bigquery_storage_v1beta2.types.ReadStream]): + Output only. A list of streams created with the session. + + At least one stream is created with the session. In the + future, larger request_stream_count values *may* result in + this list being unpopulated, in that case, the user will + need to use a List method to get the streams instead, which + is not yet available. + """ + + class TableModifiers(proto.Message): + r"""Additional attributes when reading a table. + + Attributes: + snapshot_time (google.protobuf.timestamp_pb2.Timestamp): + The snapshot time of the table. If not set, + interpreted as now. + """ + + snapshot_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=1, + message=timestamp_pb2.Timestamp, + ) + + class TableReadOptions(proto.Message): + r"""Options dictating how we read a table. + + Attributes: + selected_fields (MutableSequence[str]): + Names of the fields in the table that should be read. If + empty, all fields will be read. If the specified field is a + nested field, all the sub-fields in the field will be + selected. The output field order is unrelated to the order + of fields in selected_fields. + row_restriction (str): + SQL text filtering statement, similar to a WHERE clause in a + query. Aggregates are not supported. + + Examples: "int_field > 5" "date_field = CAST('2014-9-27' as + DATE)" "nullable_field is not NULL" "st_equals(geo_field, + st_geofromtext("POINT(2, 2)"))" "numeric_field BETWEEN 1.0 + AND 5.0" + + Restricted to a maximum length for 1 MB. + arrow_serialization_options (google.cloud.bigquery_storage_v1beta2.types.ArrowSerializationOptions): + Optional. Options specific to the Apache + Arrow output format. + """ + + selected_fields: MutableSequence[str] = proto.RepeatedField( + proto.STRING, + number=1, + ) + row_restriction: str = proto.Field( + proto.STRING, + number=2, + ) + arrow_serialization_options: arrow.ArrowSerializationOptions = proto.Field( + proto.MESSAGE, + number=3, + message=arrow.ArrowSerializationOptions, + ) + + name: str = proto.Field( + proto.STRING, + number=1, + ) + expire_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=2, + message=timestamp_pb2.Timestamp, + ) + data_format: 'DataFormat' = proto.Field( + proto.ENUM, + number=3, + enum='DataFormat', + ) + avro_schema: avro.AvroSchema = proto.Field( + proto.MESSAGE, + number=4, + oneof='schema', + message=avro.AvroSchema, + ) + arrow_schema: arrow.ArrowSchema = proto.Field( + proto.MESSAGE, + number=5, + oneof='schema', + message=arrow.ArrowSchema, + ) + table: str = proto.Field( + proto.STRING, + number=6, + ) + table_modifiers: TableModifiers = proto.Field( + proto.MESSAGE, + number=7, + message=TableModifiers, + ) + read_options: TableReadOptions = proto.Field( + proto.MESSAGE, + number=8, + message=TableReadOptions, + ) + streams: MutableSequence['ReadStream'] = proto.RepeatedField( + proto.MESSAGE, + number=10, + message='ReadStream', + ) + + +class ReadStream(proto.Message): + r"""Information about a single stream that gets data out of the storage + system. Most of the information about ``ReadStream`` instances is + aggregated, making ``ReadStream`` lightweight. + + Attributes: + name (str): + Output only. Name of the stream, in the form + ``projects/{project_id}/locations/{location}/sessions/{session_id}/streams/{stream_id}``. + """ + + name: str = proto.Field( + proto.STRING, + number=1, + ) + + +class WriteStream(proto.Message): + r"""Information about a single stream that gets data inside the + storage system. + + Attributes: + name (str): + Output only. Name of the stream, in the form + ``projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}``. + type_ (google.cloud.bigquery_storage_v1beta2.types.WriteStream.Type): + Immutable. Type of the stream. + create_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Create time of the stream. For the \_default + stream, this is the creation_time of the table. + commit_time (google.protobuf.timestamp_pb2.Timestamp): + Output only. Commit time of the stream. If a stream is of + ``COMMITTED`` type, then it will have a commit_time same as + ``create_time``. If the stream is of ``PENDING`` type, + commit_time being empty means it is not committed. + table_schema (google.cloud.bigquery_storage_v1beta2.types.TableSchema): + Output only. The schema of the destination table. It is only + returned in ``CreateWriteStream`` response. Caller should + generate data that's compatible with this schema to send in + initial ``AppendRowsRequest``. The table schema could go out + of date during the life time of the stream. + """ + class Type(proto.Enum): + r"""Type enum of the stream. + + Values: + TYPE_UNSPECIFIED (0): + Unknown type. + COMMITTED (1): + Data will commit automatically and appear as + soon as the write is acknowledged. + PENDING (2): + Data is invisible until the stream is + committed. + BUFFERED (3): + Data is only visible up to the offset to + which it was flushed. + """ + TYPE_UNSPECIFIED = 0 + COMMITTED = 1 + PENDING = 2 + BUFFERED = 3 + + name: str = proto.Field( + proto.STRING, + number=1, + ) + type_: Type = proto.Field( + proto.ENUM, + number=2, + enum=Type, + ) + create_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=3, + message=timestamp_pb2.Timestamp, + ) + commit_time: timestamp_pb2.Timestamp = proto.Field( + proto.MESSAGE, + number=4, + message=timestamp_pb2.Timestamp, + ) + table_schema: gcbs_table.TableSchema = proto.Field( + proto.MESSAGE, + number=5, + message=gcbs_table.TableSchema, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/table.py b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/table.py new file mode 100644 index 00000000..306077ee --- /dev/null +++ b/owl-bot-staging/v1beta2/google/cloud/bigquery_storage_v1beta2/types/table.py @@ -0,0 +1,166 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from __future__ import annotations + +from typing import MutableMapping, MutableSequence + +import proto # type: ignore + + +__protobuf__ = proto.module( + package='google.cloud.bigquery.storage.v1beta2', + manifest={ + 'TableSchema', + 'TableFieldSchema', + }, +) + + +class TableSchema(proto.Message): + r"""Schema of a table + + Attributes: + fields (MutableSequence[google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema]): + Describes the fields in a table. + """ + + fields: MutableSequence['TableFieldSchema'] = proto.RepeatedField( + proto.MESSAGE, + number=1, + message='TableFieldSchema', + ) + + +class TableFieldSchema(proto.Message): + r"""A field in TableSchema + + Attributes: + name (str): + Required. The field name. The name must contain only letters + (a-z, A-Z), numbers (0-9), or underscores (_), and must + start with a letter or underscore. The maximum length is 128 + characters. + type_ (google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema.Type): + Required. The field data type. + mode (google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema.Mode): + Optional. The field mode. The default value + is NULLABLE. + fields (MutableSequence[google.cloud.bigquery_storage_v1beta2.types.TableFieldSchema]): + Optional. Describes the nested schema fields + if the type property is set to STRUCT. + description (str): + Optional. The field description. The maximum + length is 1,024 characters. + """ + class Type(proto.Enum): + r""" + + Values: + TYPE_UNSPECIFIED (0): + Illegal value + STRING (1): + 64K, UTF8 + INT64 (2): + 64-bit signed + DOUBLE (3): + 64-bit IEEE floating point + STRUCT (4): + Aggregate type + BYTES (5): + 64K, Binary + BOOL (6): + 2-valued + TIMESTAMP (7): + 64-bit signed usec since UTC epoch + DATE (8): + Civil date - Year, Month, Day + TIME (9): + Civil time - Hour, Minute, Second, + Microseconds + DATETIME (10): + Combination of civil date and civil time + GEOGRAPHY (11): + Geography object + NUMERIC (12): + Numeric value + BIGNUMERIC (13): + BigNumeric value + INTERVAL (14): + Interval + JSON (15): + JSON, String + """ + TYPE_UNSPECIFIED = 0 + STRING = 1 + INT64 = 2 + DOUBLE = 3 + STRUCT = 4 + BYTES = 5 + BOOL = 6 + TIMESTAMP = 7 + DATE = 8 + TIME = 9 + DATETIME = 10 + GEOGRAPHY = 11 + NUMERIC = 12 + BIGNUMERIC = 13 + INTERVAL = 14 + JSON = 15 + + class Mode(proto.Enum): + r""" + + Values: + MODE_UNSPECIFIED (0): + Illegal value + NULLABLE (1): + No description available. + REQUIRED (2): + No description available. + REPEATED (3): + No description available. + """ + MODE_UNSPECIFIED = 0 + NULLABLE = 1 + REQUIRED = 2 + REPEATED = 3 + + name: str = proto.Field( + proto.STRING, + number=1, + ) + type_: Type = proto.Field( + proto.ENUM, + number=2, + enum=Type, + ) + mode: Mode = proto.Field( + proto.ENUM, + number=3, + enum=Mode, + ) + fields: MutableSequence['TableFieldSchema'] = proto.RepeatedField( + proto.MESSAGE, + number=4, + message='TableFieldSchema', + ) + description: str = proto.Field( + proto.STRING, + number=6, + ) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/owl-bot-staging/v1beta2/mypy.ini b/owl-bot-staging/v1beta2/mypy.ini new file mode 100644 index 00000000..574c5aed --- /dev/null +++ b/owl-bot-staging/v1beta2/mypy.ini @@ -0,0 +1,3 @@ +[mypy] +python_version = 3.7 +namespace_packages = True diff --git a/owl-bot-staging/v1beta2/noxfile.py b/owl-bot-staging/v1beta2/noxfile.py new file mode 100644 index 00000000..225cb6ed --- /dev/null +++ b/owl-bot-staging/v1beta2/noxfile.py @@ -0,0 +1,278 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +import pathlib +import re +import shutil +import subprocess +import sys + + +import nox # type: ignore + +ALL_PYTHON = [ + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", + "3.12" +] + +CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() + +LOWER_BOUND_CONSTRAINTS_FILE = CURRENT_DIRECTORY / "constraints.txt" +PACKAGE_NAME = 'google-cloud-bigquery-storage' + +BLACK_VERSION = "black==22.3.0" +BLACK_PATHS = ["docs", "google", "tests", "samples", "noxfile.py", "setup.py"] +DEFAULT_PYTHON_VERSION = "3.12" + +nox.sessions = [ + "unit", + "cover", + "mypy", + "check_lower_bounds" + # exclude update_lower_bounds from default + "docs", + "blacken", + "lint", + "prerelease_deps", +] + +@nox.session(python=ALL_PYTHON) +@nox.parametrize( + "protobuf_implementation", + [ "python", "upb", "cpp" ], +) +def unit(session, protobuf_implementation): + """Run the unit test suite.""" + + if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"): + session.skip("cpp implementation is not supported in python 3.11+") + + session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"') + session.install('-e', '.', "-c", f"testing/constraints-{session.python}.txt") + + # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. + # The 'cpp' implementation requires Protobuf<4. + if protobuf_implementation == "cpp": + session.install("protobuf<4") + + session.run( + 'py.test', + '--quiet', + '--cov=google/cloud/bigquery_storage_v1beta2/', + '--cov=tests/', + '--cov-config=.coveragerc', + '--cov-report=term', + '--cov-report=html', + os.path.join('tests', 'unit', ''.join(session.posargs)), + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + +@nox.session(python=ALL_PYTHON[-1]) +@nox.parametrize( + "protobuf_implementation", + [ "python", "upb", "cpp" ], +) +def prerelease_deps(session, protobuf_implementation): + """Run the unit test suite against pre-release versions of dependencies.""" + + if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"): + session.skip("cpp implementation is not supported in python 3.11+") + + # Install test environment dependencies + session.install('coverage', 'pytest', 'pytest-cov', 'pytest-asyncio', 'asyncmock; python_version < "3.8"') + + # Install the package without dependencies + session.install('-e', '.', '--no-deps') + + # We test the minimum dependency versions using the minimum Python + # version so the lowest python runtime that we test has a corresponding constraints + # file, located at `testing/constraints--.txt`, which contains all of the + # dependencies and extras. + with open( + CURRENT_DIRECTORY + / "testing" + / f"constraints-{ALL_PYTHON[0]}.txt", + encoding="utf-8", + ) as constraints_file: + constraints_text = constraints_file.read() + + # Ignore leading whitespace and comment lines. + constraints_deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + + session.install(*constraints_deps) + + prerel_deps = [ + "googleapis-common-protos", + "google-api-core", + "google-auth", + "grpcio", + "grpcio-status", + "protobuf", + "proto-plus", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = [ + "requests", + ] + session.install(*other_deps) + + # Print out prerelease package versions + + session.run("python", "-c", "import google.api_core; print(google.api_core.__version__)") + session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + session.run("python", "-c", "import grpc; print(grpc.__version__)") + session.run( + "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" + ) + session.run( + "python", "-c", "import proto; print(proto.__version__)" + ) + + session.run( + 'py.test', + '--quiet', + '--cov=google/cloud/bigquery_storage_v1beta2/', + '--cov=tests/', + '--cov-config=.coveragerc', + '--cov-report=term', + '--cov-report=html', + os.path.join('tests', 'unit', ''.join(session.posargs)), + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def cover(session): + """Run the final coverage report. + This outputs the coverage report aggregating coverage from the unit + test runs (not system test runs), and then erases coverage data. + """ + session.install("coverage", "pytest-cov") + session.run("coverage", "report", "--show-missing", "--fail-under=100") + + session.run("coverage", "erase") + + +@nox.session(python=ALL_PYTHON) +def mypy(session): + """Run the type checker.""" + session.install( + 'mypy', + 'types-requests', + 'types-protobuf' + ) + session.install('.') + session.run( + 'mypy', + '-p', + 'google', + ) + + +@nox.session +def update_lower_bounds(session): + """Update lower bounds in constraints.txt to match setup.py""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'update', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + + +@nox.session +def check_lower_bounds(session): + """Check lower bounds in setup.py are reflected in constraints file""" + session.install('google-cloud-testutils') + session.install('.') + + session.run( + 'lower-bound-checker', + 'check', + '--package-name', + PACKAGE_NAME, + '--constraints-file', + str(LOWER_BOUND_CONSTRAINTS_FILE), + ) + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def docs(session): + """Build the docs for this library.""" + + session.install("-e", ".") + session.install("sphinx==7.0.1", "alabaster", "recommonmark") + + shutil.rmtree(os.path.join("docs", "_build"), ignore_errors=True) + session.run( + "sphinx-build", + "-W", # warnings as errors + "-T", # show full traceback on exception + "-N", # no colors + "-b", + "html", + "-d", + os.path.join("docs", "_build", "doctrees", ""), + os.path.join("docs", ""), + os.path.join("docs", "_build", "html", ""), + ) + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def lint(session): + """Run linters. + + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + session.install("flake8", BLACK_VERSION) + session.run( + "black", + "--check", + *BLACK_PATHS, + ) + session.run("flake8", "google", "tests", "samples") + + +@nox.session(python=DEFAULT_PYTHON_VERSION) +def blacken(session): + """Run black. Format code to uniform standard.""" + session.install(BLACK_VERSION) + session.run( + "black", + *BLACK_PATHS, + ) diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_create_read_session_async.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_create_read_session_async.py new file mode 100644 index 00000000..f508fcdc --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_create_read_session_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for CreateReadSession +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryRead_CreateReadSession_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +async def sample_create_read_session(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.CreateReadSessionRequest( + parent="parent_value", + ) + + # Make the request + response = await client.create_read_session(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryRead_CreateReadSession_async] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_create_read_session_sync.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_create_read_session_sync.py new file mode 100644 index 00000000..a18a9edb --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_create_read_session_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for CreateReadSession +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryRead_CreateReadSession_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +def sample_create_read_session(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.CreateReadSessionRequest( + parent="parent_value", + ) + + # Make the request + response = client.create_read_session(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryRead_CreateReadSession_sync] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_read_rows_async.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_read_rows_async.py new file mode 100644 index 00000000..69126e10 --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_read_rows_async.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for ReadRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryRead_ReadRows_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +async def sample_read_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.ReadRowsRequest( + read_stream="read_stream_value", + ) + + # Make the request + stream = await client.read_rows(request=request) + + # Handle the response + async for response in stream: + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryRead_ReadRows_async] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_read_rows_sync.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_read_rows_sync.py new file mode 100644 index 00000000..c345d549 --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_read_rows_sync.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for ReadRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryRead_ReadRows_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +def sample_read_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.ReadRowsRequest( + read_stream="read_stream_value", + ) + + # Make the request + stream = client.read_rows(request=request) + + # Handle the response + for response in stream: + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryRead_ReadRows_sync] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_split_read_stream_async.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_split_read_stream_async.py new file mode 100644 index 00000000..bbbe296e --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_split_read_stream_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for SplitReadStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryRead_SplitReadStream_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +async def sample_split_read_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.SplitReadStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.split_read_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryRead_SplitReadStream_async] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_split_read_stream_sync.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_split_read_stream_sync.py new file mode 100644 index 00000000..c26ad7dc --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_read_split_read_stream_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for SplitReadStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryRead_SplitReadStream_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +def sample_split_read_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryReadClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.SplitReadStreamRequest( + name="name_value", + ) + + # Make the request + response = client.split_read_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryRead_SplitReadStream_sync] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_append_rows_async.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_append_rows_async.py new file mode 100644 index 00000000..0df55c4a --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_append_rows_async.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for AppendRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_AppendRows_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +async def sample_append_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.AppendRowsRequest( + write_stream="write_stream_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1beta2.AppendRowsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = await client.append_rows(requests=request_generator()) + + # Handle the response + async for response in stream: + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_AppendRows_async] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_append_rows_sync.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_append_rows_sync.py new file mode 100644 index 00000000..46e1bd60 --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_append_rows_sync.py @@ -0,0 +1,63 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for AppendRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_AppendRows_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +def sample_append_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.AppendRowsRequest( + write_stream="write_stream_value", + ) + + # This method expects an iterator which contains + # 'bigquery_storage_v1beta2.AppendRowsRequest' objects + # Here we create a generator that yields a single `request` for + # demonstrative purposes. + requests = [request] + + def request_generator(): + for request in requests: + yield request + + # Make the request + stream = client.append_rows(requests=request_generator()) + + # Handle the response + for response in stream: + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_AppendRows_sync] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_batch_commit_write_streams_async.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_batch_commit_write_streams_async.py new file mode 100644 index 00000000..1718c127 --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_batch_commit_write_streams_async.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for BatchCommitWriteStreams +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_BatchCommitWriteStreams_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +async def sample_batch_commit_write_streams(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.BatchCommitWriteStreamsRequest( + parent="parent_value", + write_streams=['write_streams_value1', 'write_streams_value2'], + ) + + # Make the request + response = await client.batch_commit_write_streams(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_BatchCommitWriteStreams_async] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_batch_commit_write_streams_sync.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_batch_commit_write_streams_sync.py new file mode 100644 index 00000000..c1e45270 --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_batch_commit_write_streams_sync.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for BatchCommitWriteStreams +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_BatchCommitWriteStreams_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +def sample_batch_commit_write_streams(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.BatchCommitWriteStreamsRequest( + parent="parent_value", + write_streams=['write_streams_value1', 'write_streams_value2'], + ) + + # Make the request + response = client.batch_commit_write_streams(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_BatchCommitWriteStreams_sync] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_create_write_stream_async.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_create_write_stream_async.py new file mode 100644 index 00000000..0d75122f --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_create_write_stream_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for CreateWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_CreateWriteStream_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +async def sample_create_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.CreateWriteStreamRequest( + parent="parent_value", + ) + + # Make the request + response = await client.create_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_CreateWriteStream_async] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_create_write_stream_sync.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_create_write_stream_sync.py new file mode 100644 index 00000000..c169e7fd --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_create_write_stream_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for CreateWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_CreateWriteStream_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +def sample_create_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.CreateWriteStreamRequest( + parent="parent_value", + ) + + # Make the request + response = client.create_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_CreateWriteStream_sync] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_finalize_write_stream_async.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_finalize_write_stream_async.py new file mode 100644 index 00000000..034bb364 --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_finalize_write_stream_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for FinalizeWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_FinalizeWriteStream_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +async def sample_finalize_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.FinalizeWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.finalize_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_FinalizeWriteStream_async] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_finalize_write_stream_sync.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_finalize_write_stream_sync.py new file mode 100644 index 00000000..d593f545 --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_finalize_write_stream_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for FinalizeWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_FinalizeWriteStream_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +def sample_finalize_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.FinalizeWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = client.finalize_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_FinalizeWriteStream_sync] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_flush_rows_async.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_flush_rows_async.py new file mode 100644 index 00000000..23d90e11 --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_flush_rows_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for FlushRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_FlushRows_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +async def sample_flush_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.FlushRowsRequest( + write_stream="write_stream_value", + ) + + # Make the request + response = await client.flush_rows(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_FlushRows_async] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_flush_rows_sync.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_flush_rows_sync.py new file mode 100644 index 00000000..201f1d3c --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_flush_rows_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for FlushRows +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_FlushRows_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +def sample_flush_rows(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.FlushRowsRequest( + write_stream="write_stream_value", + ) + + # Make the request + response = client.flush_rows(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_FlushRows_sync] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_get_write_stream_async.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_get_write_stream_async.py new file mode 100644 index 00000000..09718d35 --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_get_write_stream_async.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for GetWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_GetWriteStream_async] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +async def sample_get_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteAsyncClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.GetWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = await client.get_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_GetWriteStream_async] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_get_write_stream_sync.py b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_get_write_stream_sync.py new file mode 100644 index 00000000..aa2f6aa4 --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/bigquerystorage_v1beta2_generated_big_query_write_get_write_stream_sync.py @@ -0,0 +1,52 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Generated code. DO NOT EDIT! +# +# Snippet for GetWriteStream +# NOTE: This snippet has been automatically generated for illustrative purposes only. +# It may require modifications to work in your environment. + +# To install the latest published package dependency, execute the following: +# python3 -m pip install google-cloud-bigquery-storage + + +# [START bigquerystorage_v1beta2_generated_BigQueryWrite_GetWriteStream_sync] +# This snippet has been automatically generated and should be regarded as a +# code template only. +# It will require modifications to work: +# - It may require correct/in-range values for request initialization. +# - It may require specifying regional endpoints when creating the service +# client as shown in: +# https://googleapis.dev/python/google-api-core/latest/client_options.html +from google.cloud import bigquery_storage_v1beta2 + + +def sample_get_write_stream(): + # Create a client + client = bigquery_storage_v1beta2.BigQueryWriteClient() + + # Initialize request argument(s) + request = bigquery_storage_v1beta2.GetWriteStreamRequest( + name="name_value", + ) + + # Make the request + response = client.get_write_stream(request=request) + + # Handle the response + print(response) + +# [END bigquerystorage_v1beta2_generated_BigQueryWrite_GetWriteStream_sync] diff --git a/owl-bot-staging/v1beta2/samples/generated_samples/snippet_metadata_google.cloud.bigquery.storage.v1beta2.json b/owl-bot-staging/v1beta2/samples/generated_samples/snippet_metadata_google.cloud.bigquery.storage.v1beta2.json new file mode 100644 index 00000000..302b7815 --- /dev/null +++ b/owl-bot-staging/v1beta2/samples/generated_samples/snippet_metadata_google.cloud.bigquery.storage.v1beta2.json @@ -0,0 +1,1480 @@ +{ + "clientLibrary": { + "apis": [ + { + "id": "google.cloud.bigquery.storage.v1beta2", + "version": "v1beta2" + } + ], + "language": "PYTHON", + "name": "google-cloud-bigquery-storage", + "version": "0.1.0" + }, + "snippets": [ + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadAsyncClient", + "shortName": "BigQueryReadAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadAsyncClient.create_read_session", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead.CreateReadSession", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "CreateReadSession" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.CreateReadSessionRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "read_session", + "type": "google.cloud.bigquery_storage_v1beta2.types.ReadSession" + }, + { + "name": "max_stream_count", + "type": "int" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.ReadSession", + "shortName": "create_read_session" + }, + "description": "Sample for CreateReadSession", + "file": "bigquerystorage_v1beta2_generated_big_query_read_create_read_session_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryRead_CreateReadSession_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_read_create_read_session_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadClient", + "shortName": "BigQueryReadClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadClient.create_read_session", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead.CreateReadSession", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "CreateReadSession" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.CreateReadSessionRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "read_session", + "type": "google.cloud.bigquery_storage_v1beta2.types.ReadSession" + }, + { + "name": "max_stream_count", + "type": "int" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.ReadSession", + "shortName": "create_read_session" + }, + "description": "Sample for CreateReadSession", + "file": "bigquerystorage_v1beta2_generated_big_query_read_create_read_session_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryRead_CreateReadSession_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_read_create_read_session_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadAsyncClient", + "shortName": "BigQueryReadAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadAsyncClient.read_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead.ReadRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "ReadRows" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.ReadRowsRequest" + }, + { + "name": "read_stream", + "type": "str" + }, + { + "name": "offset", + "type": "int" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "Iterable[google.cloud.bigquery_storage_v1beta2.types.ReadRowsResponse]", + "shortName": "read_rows" + }, + "description": "Sample for ReadRows", + "file": "bigquerystorage_v1beta2_generated_big_query_read_read_rows_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryRead_ReadRows_async", + "segments": [ + { + "end": 52, + "start": 27, + "type": "FULL" + }, + { + "end": 52, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 53, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_read_read_rows_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadClient", + "shortName": "BigQueryReadClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadClient.read_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead.ReadRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "ReadRows" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.ReadRowsRequest" + }, + { + "name": "read_stream", + "type": "str" + }, + { + "name": "offset", + "type": "int" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "Iterable[google.cloud.bigquery_storage_v1beta2.types.ReadRowsResponse]", + "shortName": "read_rows" + }, + "description": "Sample for ReadRows", + "file": "bigquerystorage_v1beta2_generated_big_query_read_read_rows_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryRead_ReadRows_sync", + "segments": [ + { + "end": 52, + "start": 27, + "type": "FULL" + }, + { + "end": 52, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 53, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_read_read_rows_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadAsyncClient", + "shortName": "BigQueryReadAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadAsyncClient.split_read_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead.SplitReadStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "SplitReadStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamRequest" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamResponse", + "shortName": "split_read_stream" + }, + "description": "Sample for SplitReadStream", + "file": "bigquerystorage_v1beta2_generated_big_query_read_split_read_stream_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryRead_SplitReadStream_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_read_split_read_stream_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadClient", + "shortName": "BigQueryReadClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryReadClient.split_read_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead.SplitReadStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryRead", + "shortName": "BigQueryRead" + }, + "shortName": "SplitReadStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamRequest" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.SplitReadStreamResponse", + "shortName": "split_read_stream" + }, + "description": "Sample for SplitReadStream", + "file": "bigquerystorage_v1beta2_generated_big_query_read_split_read_stream_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryRead_SplitReadStream_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_read_split_read_stream_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient.append_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.AppendRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "AppendRows" + }, + "parameters": [ + { + "name": "requests", + "type": "Iterator[google.cloud.bigquery_storage_v1beta2.types.AppendRowsRequest]" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "Iterable[google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse]", + "shortName": "append_rows" + }, + "description": "Sample for AppendRows", + "file": "bigquerystorage_v1beta2_generated_big_query_write_append_rows_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_AppendRows_async", + "segments": [ + { + "end": 62, + "start": 27, + "type": "FULL" + }, + { + "end": 62, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 55, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 58, + "start": 56, + "type": "REQUEST_EXECUTION" + }, + { + "end": 63, + "start": 59, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_append_rows_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient.append_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.AppendRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "AppendRows" + }, + "parameters": [ + { + "name": "requests", + "type": "Iterator[google.cloud.bigquery_storage_v1beta2.types.AppendRowsRequest]" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "Iterable[google.cloud.bigquery_storage_v1beta2.types.AppendRowsResponse]", + "shortName": "append_rows" + }, + "description": "Sample for AppendRows", + "file": "bigquerystorage_v1beta2_generated_big_query_write_append_rows_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_AppendRows_sync", + "segments": [ + { + "end": 62, + "start": 27, + "type": "FULL" + }, + { + "end": 62, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 55, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 58, + "start": 56, + "type": "REQUEST_EXECUTION" + }, + { + "end": 63, + "start": 59, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_append_rows_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient.batch_commit_write_streams", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.BatchCommitWriteStreams", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "BatchCommitWriteStreams" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsResponse", + "shortName": "batch_commit_write_streams" + }, + "description": "Sample for BatchCommitWriteStreams", + "file": "bigquerystorage_v1beta2_generated_big_query_write_batch_commit_write_streams_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_BatchCommitWriteStreams_async", + "segments": [ + { + "end": 52, + "start": 27, + "type": "FULL" + }, + { + "end": 52, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 46, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 49, + "start": 47, + "type": "REQUEST_EXECUTION" + }, + { + "end": 53, + "start": 50, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_batch_commit_write_streams_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient.batch_commit_write_streams", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.BatchCommitWriteStreams", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "BatchCommitWriteStreams" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.BatchCommitWriteStreamsResponse", + "shortName": "batch_commit_write_streams" + }, + "description": "Sample for BatchCommitWriteStreams", + "file": "bigquerystorage_v1beta2_generated_big_query_write_batch_commit_write_streams_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_BatchCommitWriteStreams_sync", + "segments": [ + { + "end": 52, + "start": 27, + "type": "FULL" + }, + { + "end": 52, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 46, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 49, + "start": 47, + "type": "REQUEST_EXECUTION" + }, + { + "end": 53, + "start": 50, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_batch_commit_write_streams_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient.create_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.CreateWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "CreateWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.CreateWriteStreamRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "write_stream", + "type": "google.cloud.bigquery_storage_v1beta2.types.WriteStream" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.WriteStream", + "shortName": "create_write_stream" + }, + "description": "Sample for CreateWriteStream", + "file": "bigquerystorage_v1beta2_generated_big_query_write_create_write_stream_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_CreateWriteStream_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_create_write_stream_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient.create_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.CreateWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "CreateWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.CreateWriteStreamRequest" + }, + { + "name": "parent", + "type": "str" + }, + { + "name": "write_stream", + "type": "google.cloud.bigquery_storage_v1beta2.types.WriteStream" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.WriteStream", + "shortName": "create_write_stream" + }, + "description": "Sample for CreateWriteStream", + "file": "bigquerystorage_v1beta2_generated_big_query_write_create_write_stream_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_CreateWriteStream_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_create_write_stream_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient.finalize_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.FinalizeWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "FinalizeWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamRequest" + }, + { + "name": "name", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamResponse", + "shortName": "finalize_write_stream" + }, + "description": "Sample for FinalizeWriteStream", + "file": "bigquerystorage_v1beta2_generated_big_query_write_finalize_write_stream_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_FinalizeWriteStream_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_finalize_write_stream_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient.finalize_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.FinalizeWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "FinalizeWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamRequest" + }, + { + "name": "name", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.FinalizeWriteStreamResponse", + "shortName": "finalize_write_stream" + }, + "description": "Sample for FinalizeWriteStream", + "file": "bigquerystorage_v1beta2_generated_big_query_write_finalize_write_stream_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_FinalizeWriteStream_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_finalize_write_stream_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient.flush_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.FlushRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "FlushRows" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.FlushRowsRequest" + }, + { + "name": "write_stream", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.FlushRowsResponse", + "shortName": "flush_rows" + }, + "description": "Sample for FlushRows", + "file": "bigquerystorage_v1beta2_generated_big_query_write_flush_rows_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_FlushRows_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_flush_rows_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient.flush_rows", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.FlushRows", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "FlushRows" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.FlushRowsRequest" + }, + { + "name": "write_stream", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.FlushRowsResponse", + "shortName": "flush_rows" + }, + "description": "Sample for FlushRows", + "file": "bigquerystorage_v1beta2_generated_big_query_write_flush_rows_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_FlushRows_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_flush_rows_sync.py" + }, + { + "canonical": true, + "clientMethod": { + "async": true, + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient", + "shortName": "BigQueryWriteAsyncClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteAsyncClient.get_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.GetWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "GetWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.GetWriteStreamRequest" + }, + { + "name": "name", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.WriteStream", + "shortName": "get_write_stream" + }, + "description": "Sample for GetWriteStream", + "file": "bigquerystorage_v1beta2_generated_big_query_write_get_write_stream_async.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_GetWriteStream_async", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_get_write_stream_async.py" + }, + { + "canonical": true, + "clientMethod": { + "client": { + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient", + "shortName": "BigQueryWriteClient" + }, + "fullName": "google.cloud.bigquery_storage_v1beta2.BigQueryWriteClient.get_write_stream", + "method": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite.GetWriteStream", + "service": { + "fullName": "google.cloud.bigquery.storage.v1beta2.BigQueryWrite", + "shortName": "BigQueryWrite" + }, + "shortName": "GetWriteStream" + }, + "parameters": [ + { + "name": "request", + "type": "google.cloud.bigquery_storage_v1beta2.types.GetWriteStreamRequest" + }, + { + "name": "name", + "type": "str" + }, + { + "name": "retry", + "type": "google.api_core.retry.Retry" + }, + { + "name": "timeout", + "type": "float" + }, + { + "name": "metadata", + "type": "Sequence[Tuple[str, str]" + } + ], + "resultType": "google.cloud.bigquery_storage_v1beta2.types.WriteStream", + "shortName": "get_write_stream" + }, + "description": "Sample for GetWriteStream", + "file": "bigquerystorage_v1beta2_generated_big_query_write_get_write_stream_sync.py", + "language": "PYTHON", + "origin": "API_DEFINITION", + "regionTag": "bigquerystorage_v1beta2_generated_BigQueryWrite_GetWriteStream_sync", + "segments": [ + { + "end": 51, + "start": 27, + "type": "FULL" + }, + { + "end": 51, + "start": 27, + "type": "SHORT" + }, + { + "end": 40, + "start": 38, + "type": "CLIENT_INITIALIZATION" + }, + { + "end": 45, + "start": 41, + "type": "REQUEST_INITIALIZATION" + }, + { + "end": 48, + "start": 46, + "type": "REQUEST_EXECUTION" + }, + { + "end": 52, + "start": 49, + "type": "RESPONSE_HANDLING" + } + ], + "title": "bigquerystorage_v1beta2_generated_big_query_write_get_write_stream_sync.py" + } + ] +} diff --git a/owl-bot-staging/v1beta2/scripts/fixup_bigquery_storage_v1beta2_keywords.py b/owl-bot-staging/v1beta2/scripts/fixup_bigquery_storage_v1beta2_keywords.py new file mode 100644 index 00000000..4a63ffab --- /dev/null +++ b/owl-bot-staging/v1beta2/scripts/fixup_bigquery_storage_v1beta2_keywords.py @@ -0,0 +1,184 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import argparse +import os +import libcst as cst +import pathlib +import sys +from typing import (Any, Callable, Dict, List, Sequence, Tuple) + + +def partition( + predicate: Callable[[Any], bool], + iterator: Sequence[Any] +) -> Tuple[List[Any], List[Any]]: + """A stable, out-of-place partition.""" + results = ([], []) + + for i in iterator: + results[int(predicate(i))].append(i) + + # Returns trueList, falseList + return results[1], results[0] + + +class bigquery_storageCallTransformer(cst.CSTTransformer): + CTRL_PARAMS: Tuple[str] = ('retry', 'timeout', 'metadata') + METHOD_TO_PARAMS: Dict[str, Tuple[str]] = { + 'append_rows': ('write_stream', 'offset', 'proto_rows', 'trace_id', ), + 'batch_commit_write_streams': ('parent', 'write_streams', ), + 'create_read_session': ('parent', 'read_session', 'max_stream_count', ), + 'create_write_stream': ('parent', 'write_stream', ), + 'finalize_write_stream': ('name', ), + 'flush_rows': ('write_stream', 'offset', ), + 'get_write_stream': ('name', ), + 'read_rows': ('read_stream', 'offset', ), + 'split_read_stream': ('name', 'fraction', ), + } + + def leave_Call(self, original: cst.Call, updated: cst.Call) -> cst.CSTNode: + try: + key = original.func.attr.value + kword_params = self.METHOD_TO_PARAMS[key] + except (AttributeError, KeyError): + # Either not a method from the API or too convoluted to be sure. + return updated + + # If the existing code is valid, keyword args come after positional args. + # Therefore, all positional args must map to the first parameters. + args, kwargs = partition(lambda a: not bool(a.keyword), updated.args) + if any(k.keyword.value == "request" for k in kwargs): + # We've already fixed this file, don't fix it again. + return updated + + kwargs, ctrl_kwargs = partition( + lambda a: a.keyword.value not in self.CTRL_PARAMS, + kwargs + ) + + args, ctrl_args = args[:len(kword_params)], args[len(kword_params):] + ctrl_kwargs.extend(cst.Arg(value=a.value, keyword=cst.Name(value=ctrl)) + for a, ctrl in zip(ctrl_args, self.CTRL_PARAMS)) + + request_arg = cst.Arg( + value=cst.Dict([ + cst.DictElement( + cst.SimpleString("'{}'".format(name)), +cst.Element(value=arg.value) + ) + # Note: the args + kwargs looks silly, but keep in mind that + # the control parameters had to be stripped out, and that + # those could have been passed positionally or by keyword. + for name, arg in zip(kword_params, args + kwargs)]), + keyword=cst.Name("request") + ) + + return updated.with_changes( + args=[request_arg] + ctrl_kwargs + ) + + +def fix_files( + in_dir: pathlib.Path, + out_dir: pathlib.Path, + *, + transformer=bigquery_storageCallTransformer(), +): + """Duplicate the input dir to the output dir, fixing file method calls. + + Preconditions: + * in_dir is a real directory + * out_dir is a real, empty directory + """ + pyfile_gen = ( + pathlib.Path(os.path.join(root, f)) + for root, _, files in os.walk(in_dir) + for f in files if os.path.splitext(f)[1] == ".py" + ) + + for fpath in pyfile_gen: + with open(fpath, 'r') as f: + src = f.read() + + # Parse the code and insert method call fixes. + tree = cst.parse_module(src) + updated = tree.visit(transformer) + + # Create the path and directory structure for the new file. + updated_path = out_dir.joinpath(fpath.relative_to(in_dir)) + updated_path.parent.mkdir(parents=True, exist_ok=True) + + # Generate the updated source file at the corresponding path. + with open(updated_path, 'w') as f: + f.write(updated.code) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description="""Fix up source that uses the bigquery_storage client library. + +The existing sources are NOT overwritten but are copied to output_dir with changes made. + +Note: This tool operates at a best-effort level at converting positional + parameters in client method calls to keyword based parameters. + Cases where it WILL FAIL include + A) * or ** expansion in a method call. + B) Calls via function or method alias (includes free function calls) + C) Indirect or dispatched calls (e.g. the method is looked up dynamically) + + These all constitute false negatives. The tool will also detect false + positives when an API method shares a name with another method. +""") + parser.add_argument( + '-d', + '--input-directory', + required=True, + dest='input_dir', + help='the input directory to walk for python files to fix up', + ) + parser.add_argument( + '-o', + '--output-directory', + required=True, + dest='output_dir', + help='the directory to output files fixed via un-flattening', + ) + args = parser.parse_args() + input_dir = pathlib.Path(args.input_dir) + output_dir = pathlib.Path(args.output_dir) + if not input_dir.is_dir(): + print( + f"input directory '{input_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if not output_dir.is_dir(): + print( + f"output directory '{output_dir}' does not exist or is not a directory", + file=sys.stderr, + ) + sys.exit(-1) + + if os.listdir(output_dir): + print( + f"output directory '{output_dir}' is not empty", + file=sys.stderr, + ) + sys.exit(-1) + + fix_files(input_dir, output_dir) diff --git a/owl-bot-staging/v1beta2/setup.py b/owl-bot-staging/v1beta2/setup.py new file mode 100644 index 00000000..b688ea84 --- /dev/null +++ b/owl-bot-staging/v1beta2/setup.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import io +import os +import re + +import setuptools # type: ignore + +package_root = os.path.abspath(os.path.dirname(__file__)) + +name = 'google-cloud-bigquery-storage' + + +description = "Google Cloud Bigquery Storage API client library" + +version = None + +with open(os.path.join(package_root, 'google/cloud/bigquery_storage/gapic_version.py')) as fp: + version_candidates = re.findall(r"(?<=\")\d+.\d+.\d+(?=\")", fp.read()) + assert (len(version_candidates) == 1) + version = version_candidates[0] + +if version[0] == "0": + release_status = "Development Status :: 4 - Beta" +else: + release_status = "Development Status :: 5 - Production/Stable" + +dependencies = [ + "google-api-core[grpc] >= 1.34.1, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", + # Exclude incompatible versions of `google-auth` + # See https://github.com/googleapis/google-cloud-python/issues/12364 + "google-auth >= 2.14.1, <3.0.0dev,!=2.24.0,!=2.25.0", + "proto-plus >= 1.22.3, <2.0.0dev", + "protobuf>=3.20.2,<6.0.0dev,!=4.21.0,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", +] +extras = { +} +url = "https://github.com/googleapis/google-cloud-python/tree/main/packages/google-cloud-bigquery-storage" + +package_root = os.path.abspath(os.path.dirname(__file__)) + +readme_filename = os.path.join(package_root, "README.rst") +with io.open(readme_filename, encoding="utf-8") as readme_file: + readme = readme_file.read() + +packages = [ + package + for package in setuptools.find_namespace_packages() + if package.startswith("google") +] + +setuptools.setup( + name=name, + version=version, + description=description, + long_description=readme, + author="Google LLC", + author_email="googleapis-packages@google.com", + license="Apache 2.0", + url=url, + classifiers=[ + release_status, + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Operating System :: OS Independent", + "Topic :: Internet", + ], + platforms="Posix; MacOS X; Windows", + packages=packages, + python_requires=">=3.7", + install_requires=dependencies, + extras_require=extras, + include_package_data=True, + zip_safe=False, +) diff --git a/owl-bot-staging/v1beta2/testing/constraints-3.10.txt b/owl-bot-staging/v1beta2/testing/constraints-3.10.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1beta2/testing/constraints-3.10.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1beta2/testing/constraints-3.11.txt b/owl-bot-staging/v1beta2/testing/constraints-3.11.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1beta2/testing/constraints-3.11.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1beta2/testing/constraints-3.12.txt b/owl-bot-staging/v1beta2/testing/constraints-3.12.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1beta2/testing/constraints-3.12.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1beta2/testing/constraints-3.7.txt b/owl-bot-staging/v1beta2/testing/constraints-3.7.txt new file mode 100644 index 00000000..fc812592 --- /dev/null +++ b/owl-bot-staging/v1beta2/testing/constraints-3.7.txt @@ -0,0 +1,10 @@ +# This constraints file is used to check that lower bounds +# are correct in setup.py +# List all library dependencies and extras in this file. +# Pin the version to the lower bound. +# e.g., if setup.py has "google-cloud-foo >= 1.14.0, < 2.0.0dev", +# Then this file should have google-cloud-foo==1.14.0 +google-api-core==1.34.1 +google-auth==2.14.1 +proto-plus==1.22.3 +protobuf==3.20.2 diff --git a/owl-bot-staging/v1beta2/testing/constraints-3.8.txt b/owl-bot-staging/v1beta2/testing/constraints-3.8.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1beta2/testing/constraints-3.8.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1beta2/testing/constraints-3.9.txt b/owl-bot-staging/v1beta2/testing/constraints-3.9.txt new file mode 100644 index 00000000..ed7f9aed --- /dev/null +++ b/owl-bot-staging/v1beta2/testing/constraints-3.9.txt @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +# This constraints file is required for unit tests. +# List all library dependencies and extras in this file. +google-api-core +proto-plus +protobuf diff --git a/owl-bot-staging/v1beta2/tests/__init__.py b/owl-bot-staging/v1beta2/tests/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1beta2/tests/unit/__init__.py b/owl-bot-staging/v1beta2/tests/unit/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/unit/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/__init__.py b/owl-bot-staging/v1beta2/tests/unit/gapic/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/unit/gapic/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py new file mode 100644 index 00000000..7b3de311 --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/__init__.py @@ -0,0 +1,16 @@ + +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py new file mode 100644 index 00000000..8b20c52f --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_read.py @@ -0,0 +1,2494 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +# try/except added for compatibility with python < 3.8 +try: + from unittest import mock + from unittest.mock import AsyncMock # pragma: NO COVER +except ImportError: # pragma: NO COVER + import mock + +import grpc +from grpc.experimental import aio +import math +import pytest +from google.api_core import api_core_version +from proto.marshal.rules.dates import DurationRule, TimestampRule +from proto.marshal.rules import wrappers + +try: + from google.auth.aio import credentials as ga_credentials_async + HAS_GOOGLE_AUTH_AIO = True +except ImportError: # pragma: NO COVER + HAS_GOOGLE_AUTH_AIO = False + +from google.api_core import client_options +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers +from google.api_core import grpc_helpers_async +from google.api_core import path_template +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials +from google.auth.exceptions import MutualTLSChannelError +from google.cloud.bigquery_storage_v1beta2.services.big_query_read import BigQueryReadAsyncClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_read import BigQueryReadClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_read import transports +from google.cloud.bigquery_storage_v1beta2.types import arrow +from google.cloud.bigquery_storage_v1beta2.types import avro +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.oauth2 import service_account +from google.protobuf import timestamp_pb2 # type: ignore +import google.auth + + +async def mock_async_gen(data, chunk_size=1): + for i in range(0, len(data)): # pragma: NO COVER + chunk = data[i : i + chunk_size] + yield chunk.encode("utf-8") + +def client_cert_source_callback(): + return b"cert bytes", b"key bytes" + +# TODO: use async auth anon credentials by default once the minimum version of google-auth is upgraded. +# See related issue: https://github.com/googleapis/gapic-generator-python/issues/2107. +def async_anonymous_credentials(): + if HAS_GOOGLE_AUTH_AIO: + return ga_credentials_async.AnonymousCredentials() + return ga_credentials.AnonymousCredentials() + +# If default endpoint is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint(client): + return "foo.googleapis.com" if ("localhost" in client.DEFAULT_ENDPOINT) else client.DEFAULT_ENDPOINT + +# If default endpoint template is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint template so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint_template(client): + return "test.{UNIVERSE_DOMAIN}" if ("localhost" in client._DEFAULT_ENDPOINT_TEMPLATE) else client._DEFAULT_ENDPOINT_TEMPLATE + + +def test__get_default_mtls_endpoint(): + api_endpoint = "example.googleapis.com" + api_mtls_endpoint = "example.mtls.googleapis.com" + sandbox_endpoint = "example.sandbox.googleapis.com" + sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" + non_googleapi = "api.example.com" + + assert BigQueryReadClient._get_default_mtls_endpoint(None) is None + assert BigQueryReadClient._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint + assert BigQueryReadClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi + +def test__read_environment_variables(): + assert BigQueryReadClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert BigQueryReadClient._read_environment_variables() == (True, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + assert BigQueryReadClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + BigQueryReadClient._read_environment_variables() + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + assert BigQueryReadClient._read_environment_variables() == (False, "never", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + assert BigQueryReadClient._read_environment_variables() == (False, "always", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}): + assert BigQueryReadClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + BigQueryReadClient._read_environment_variables() + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + with mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}): + assert BigQueryReadClient._read_environment_variables() == (False, "auto", "foo.com") + +def test__get_client_cert_source(): + mock_provided_cert_source = mock.Mock() + mock_default_cert_source = mock.Mock() + + assert BigQueryReadClient._get_client_cert_source(None, False) is None + assert BigQueryReadClient._get_client_cert_source(mock_provided_cert_source, False) is None + assert BigQueryReadClient._get_client_cert_source(mock_provided_cert_source, True) == mock_provided_cert_source + + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=mock_default_cert_source): + assert BigQueryReadClient._get_client_cert_source(None, True) is mock_default_cert_source + assert BigQueryReadClient._get_client_cert_source(mock_provided_cert_source, "true") is mock_provided_cert_source + +@mock.patch.object(BigQueryReadClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadAsyncClient)) +def test__get_api_endpoint(): + api_override = "foo.com" + mock_client_cert_source = mock.Mock() + default_universe = BigQueryReadClient._DEFAULT_UNIVERSE + default_endpoint = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=default_universe) + mock_universe = "bar.com" + mock_endpoint = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=mock_universe) + + assert BigQueryReadClient._get_api_endpoint(api_override, mock_client_cert_source, default_universe, "always") == api_override + assert BigQueryReadClient._get_api_endpoint(None, mock_client_cert_source, default_universe, "auto") == BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryReadClient._get_api_endpoint(None, None, default_universe, "auto") == default_endpoint + assert BigQueryReadClient._get_api_endpoint(None, None, default_universe, "always") == BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryReadClient._get_api_endpoint(None, mock_client_cert_source, default_universe, "always") == BigQueryReadClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryReadClient._get_api_endpoint(None, None, mock_universe, "never") == mock_endpoint + assert BigQueryReadClient._get_api_endpoint(None, None, default_universe, "never") == default_endpoint + + with pytest.raises(MutualTLSChannelError) as excinfo: + BigQueryReadClient._get_api_endpoint(None, mock_client_cert_source, mock_universe, "auto") + assert str(excinfo.value) == "mTLS is not supported in any universe other than googleapis.com." + + +def test__get_universe_domain(): + client_universe_domain = "foo.com" + universe_domain_env = "bar.com" + + assert BigQueryReadClient._get_universe_domain(client_universe_domain, universe_domain_env) == client_universe_domain + assert BigQueryReadClient._get_universe_domain(None, universe_domain_env) == universe_domain_env + assert BigQueryReadClient._get_universe_domain(None, None) == BigQueryReadClient._DEFAULT_UNIVERSE + + with pytest.raises(ValueError) as excinfo: + BigQueryReadClient._get_universe_domain("", None) + assert str(excinfo.value) == "Universe Domain cannot be an empty string." + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), +]) +def test__validate_universe_domain(client_class, transport_class, transport_name): + client = client_class( + transport=transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + ) + assert client._validate_universe_domain() == True + + # Test the case when universe is already validated. + assert client._validate_universe_domain() == True + + if transport_name == "grpc": + # Test the case where credentials are provided by the + # `local_channel_credentials`. The default universes in both match. + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + client = client_class(transport=transport_class(channel=channel)) + assert client._validate_universe_domain() == True + + # Test the case where credentials do not exist: e.g. a transport is provided + # with no credentials. Validation should still succeed because there is no + # mismatch with non-existent credentials. + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + transport=transport_class(channel=channel) + transport._credentials = None + client = client_class(transport=transport) + assert client._validate_universe_domain() == True + + # TODO: This is needed to cater for older versions of google-auth + # Make this test unconditional once the minimum supported version of + # google-auth becomes 2.23.0 or higher. + google_auth_major, google_auth_minor = [int(part) for part in google.auth.__version__.split(".")[0:2]] + if google_auth_major > 2 or (google_auth_major == 2 and google_auth_minor >= 23): + credentials = ga_credentials.AnonymousCredentials() + credentials._universe_domain = "foo.com" + # Test the case when there is a universe mismatch from the credentials. + client = client_class( + transport=transport_class(credentials=credentials) + ) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert str(excinfo.value) == "The configured universe domain (googleapis.com) does not match the universe domain found in the credentials (foo.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + + # Test the case when there is a universe mismatch from the client. + # + # TODO: Make this test unconditional once the minimum supported version of + # google-api-core becomes 2.15.0 or higher. + api_core_major, api_core_minor = [int(part) for part in api_core_version.__version__.split(".")[0:2]] + if api_core_major > 2 or (api_core_major == 2 and api_core_minor >= 15): + client = client_class(client_options={"universe_domain": "bar.com"}, transport=transport_class(credentials=ga_credentials.AnonymousCredentials(),)) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert str(excinfo.value) == "The configured universe domain (bar.com) does not match the universe domain found in the credentials (googleapis.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + + # Test that ValueError is raised if universe_domain is provided via client options and credentials is None + with pytest.raises(ValueError): + client._compare_universes("foo.bar", None) + + +@pytest.mark.parametrize("client_class,transport_name", [ + (BigQueryReadClient, "grpc"), + (BigQueryReadAsyncClient, "grpc_asyncio"), +]) +def test_big_query_read_client_from_service_account_info(client_class, transport_name): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_info') as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info, transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + + +@pytest.mark.parametrize("transport_class,transport_name", [ + (transports.BigQueryReadGrpcTransport, "grpc"), + (transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_read_client_service_account_always_use_jwt(transport_class, transport_name): + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize("client_class,transport_name", [ + (BigQueryReadClient, "grpc"), + (BigQueryReadAsyncClient, "grpc_asyncio"), +]) +def test_big_query_read_client_from_service_account_file(client_class, transport_name): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_file') as factory: + factory.return_value = creds + client = client_class.from_service_account_file("dummy/file/path.json", transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + client = client_class.from_service_account_json("dummy/file/path.json", transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + + +def test_big_query_read_client_get_transport_class(): + transport = BigQueryReadClient.get_transport_class() + available_transports = [ + transports.BigQueryReadGrpcTransport, + ] + assert transport in available_transports + + transport = BigQueryReadClient.get_transport_class("grpc") + assert transport == transports.BigQueryReadGrpcTransport + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +@mock.patch.object(BigQueryReadClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadAsyncClient)) +def test_big_query_read_client_client_options(client_class, transport_class, transport_name): + # Check that if channel is provided we won't create a new one. + with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + client = client_class(transport=transport) + gtc.assert_not_called() + + # Check that if channel is provided via str we will create a new one. + with mock.patch.object(BigQueryReadClient, 'get_transport_class') as gtc: + client = client_class(transport=transport_name) + gtc.assert_called() + + # Check the case api_endpoint is provided. + options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name, client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_MTLS_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client = client_class(transport=transport_name) + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + client = client_class(transport=transport_name) + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + # Check the case api_endpoint is provided + options = client_options.ClientOptions(api_audience="https://language.googleapis.com") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience="https://language.googleapis.com" + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,use_client_cert_env", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "true"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "true"), + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", "false"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", "false"), +]) +@mock.patch.object(BigQueryReadClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadAsyncClient)) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_big_query_read_client_mtls_env_auto(client_class, transport_class, transport_name, use_client_cert_env): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + options = client_options.ClientOptions(client_cert_source=client_cert_source_callback) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE) + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT + + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=client_cert_source_callback): + if use_client_cert_env == "false": + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE) + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback + + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=False): + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + +@pytest.mark.parametrize("client_class", [ + BigQueryReadClient, BigQueryReadAsyncClient +]) +@mock.patch.object(BigQueryReadClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryReadAsyncClient)) +def test_big_query_read_client_get_mtls_endpoint_and_cert_source(client_class): + mock_client_cert_source = mock.Mock() + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "true". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + mock_api_endpoint = "foo" + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source == mock_client_cert_source + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "false". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert doesn't exist. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=False): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert exists. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=mock_client_cert_source): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source == mock_client_cert_source + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + +@pytest.mark.parametrize("client_class", [ + BigQueryReadClient, BigQueryReadAsyncClient +]) +@mock.patch.object(BigQueryReadClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadClient)) +@mock.patch.object(BigQueryReadAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryReadAsyncClient)) +def test_big_query_read_client_client_api_endpoint(client_class): + mock_client_cert_source = client_cert_source_callback + api_override = "foo.com" + default_universe = BigQueryReadClient._DEFAULT_UNIVERSE + default_endpoint = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=default_universe) + mock_universe = "bar.com" + mock_endpoint = BigQueryReadClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=mock_universe) + + # If ClientOptions.api_endpoint is set and GOOGLE_API_USE_CLIENT_CERTIFICATE="true", + # use ClientOptions.api_endpoint as the api endpoint regardless. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch("google.auth.transport.requests.AuthorizedSession.configure_mtls_channel"): + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=api_override) + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == api_override + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="always", + # use the DEFAULT_MTLS_ENDPOINT as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + + # If ClientOptions.api_endpoint is not set, GOOGLE_API_USE_MTLS_ENDPOINT="auto" (default), + # GOOGLE_API_USE_CLIENT_CERTIFICATE="false" (default), default cert source doesn't exist, + # and ClientOptions.universe_domain="bar.com", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with universe domain as the api endpoint. + options = client_options.ClientOptions() + universe_exists = hasattr(options, "universe_domain") + if universe_exists: + options = client_options.ClientOptions(universe_domain=mock_universe) + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + else: + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == (mock_endpoint if universe_exists else default_endpoint) + assert client.universe_domain == (mock_universe if universe_exists else default_universe) + + # If ClientOptions does not have a universe domain attribute and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + options = client_options.ClientOptions() + if hasattr(options, "universe_domain"): + delattr(options, "universe_domain") + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc"), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_read_client_client_options_scopes(client_class, transport_class, transport_name): + # Check the case scopes are provided. + options = client_options.ClientOptions( + scopes=["1", "2"], + ) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=["1", "2"], + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,grpc_helpers", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", grpc_helpers), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", grpc_helpers_async), +]) +def test_big_query_read_client_client_options_credentials_file(client_class, transport_class, transport_name, grpc_helpers): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + +def test_big_query_read_client_client_options_from_dict(): + with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadGrpcTransport.__init__') as grpc_transport: + grpc_transport.return_value = None + client = BigQueryReadClient( + client_options={'api_endpoint': 'squid.clam.whelk'} + ) + grpc_transport.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + +@pytest.mark.parametrize("client_class,transport_class,transport_name,grpc_helpers", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport, "grpc", grpc_helpers), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport, "grpc_asyncio", grpc_helpers_async), +]) +def test_big_query_read_client_create_channel_credentials_file(client_class, transport_class, transport_name, grpc_helpers): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=None, + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("request_type", [ + storage.CreateReadSessionRequest, + dict, +]) +def test_create_read_session(request_type, transport: str = 'grpc'): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession( + name='name_value', + data_format=stream.DataFormat.AVRO, + table='table_value', + ) + response = client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.CreateReadSessionRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.ReadSession) + assert response.name == 'name_value' + assert response.data_format == stream.DataFormat.AVRO + assert response.table == 'table_value' + + +def test_create_read_session_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.CreateReadSessionRequest( + parent='parent_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.create_read_session(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateReadSessionRequest( + parent='parent_value', + ) + +def test_create_read_session_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.create_read_session in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.create_read_session] = mock_rpc + request = {} + client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.create_read_session(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_create_read_session_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.create_read_session in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.create_read_session] = mock_rpc + + request = {} + await client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.create_read_session(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_create_read_session_async(transport: str = 'grpc_asyncio', request_type=storage.CreateReadSessionRequest): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession( + name='name_value', + data_format=stream.DataFormat.AVRO, + table='table_value', + )) + response = await client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.CreateReadSessionRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.ReadSession) + assert response.name == 'name_value' + assert response.data_format == stream.DataFormat.AVRO + assert response.table == 'table_value' + + +@pytest.mark.asyncio +async def test_create_read_session_async_from_dict(): + await test_create_read_session_async(request_type=dict) + +def test_create_read_session_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateReadSessionRequest() + + request.read_session.table = 'table_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value = stream.ReadSession() + client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_session.table=table_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_create_read_session_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateReadSessionRequest() + + request.read_session.table = 'table_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) + await client.create_read_session(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_session.table=table_value', + ) in kw['metadata'] + + +def test_create_read_session_flattened(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.create_read_session( + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + arg = args[0].read_session + mock_val = stream.ReadSession(name='name_value') + assert arg == mock_val + arg = args[0].max_stream_count + mock_val = 1721 + assert arg == mock_val + + +def test_create_read_session_flattened_error(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.create_read_session( + storage.CreateReadSessionRequest(), + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + +@pytest.mark.asyncio +async def test_create_read_session_flattened_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.ReadSession() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.create_read_session( + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + arg = args[0].read_session + mock_val = stream.ReadSession(name='name_value') + assert arg == mock_val + arg = args[0].max_stream_count + mock_val = 1721 + assert arg == mock_val + +@pytest.mark.asyncio +async def test_create_read_session_flattened_error_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.create_read_session( + storage.CreateReadSessionRequest(), + parent='parent_value', + read_session=stream.ReadSession(name='name_value'), + max_stream_count=1721, + ) + + +@pytest.mark.parametrize("request_type", [ + storage.ReadRowsRequest, + dict, +]) +def test_read_rows(request_type, transport: str = 'grpc'): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + response = client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.ReadRowsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + for message in response: + assert isinstance(message, storage.ReadRowsResponse) + + +def test_read_rows_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.ReadRowsRequest( + read_stream='read_stream_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.read_rows(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.ReadRowsRequest( + read_stream='read_stream_value', + ) + +def test_read_rows_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.read_rows in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.read_rows] = mock_rpc + request = {} + client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.read_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_read_rows_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.read_rows in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.read_rows] = mock_rpc + + request = {} + await client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.read_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_read_rows_async(transport: str = 'grpc_asyncio', request_type=storage.ReadRowsRequest): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) + response = await client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.ReadRowsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + message = await response.read() + assert isinstance(message, storage.ReadRowsResponse) + + +@pytest.mark.asyncio +async def test_read_rows_async_from_dict(): + await test_read_rows_async(request_type=dict) + +def test_read_rows_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.ReadRowsRequest() + + request.read_stream = 'read_stream_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value = iter([storage.ReadRowsResponse()]) + client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_stream=read_stream_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_read_rows_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.ReadRowsRequest() + + request.read_stream = 'read_stream_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) + await client.read_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'read_stream=read_stream_value', + ) in kw['metadata'] + + +def test_read_rows_flattened(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.read_rows( + read_stream='read_stream_value', + offset=647, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].read_stream + mock_val = 'read_stream_value' + assert arg == mock_val + arg = args[0].offset + mock_val = 647 + assert arg == mock_val + + +def test_read_rows_flattened_error(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.read_rows( + storage.ReadRowsRequest(), + read_stream='read_stream_value', + offset=647, + ) + +@pytest.mark.asyncio +async def test_read_rows_flattened_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.ReadRowsResponse()]) + + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.read_rows( + read_stream='read_stream_value', + offset=647, + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].read_stream + mock_val = 'read_stream_value' + assert arg == mock_val + arg = args[0].offset + mock_val = 647 + assert arg == mock_val + +@pytest.mark.asyncio +async def test_read_rows_flattened_error_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.read_rows( + storage.ReadRowsRequest(), + read_stream='read_stream_value', + offset=647, + ) + + +@pytest.mark.parametrize("request_type", [ + storage.SplitReadStreamRequest, + dict, +]) +def test_split_read_stream(request_type, transport: str = 'grpc'): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.SplitReadStreamResponse( + ) + response = client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.SplitReadStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.SplitReadStreamResponse) + + +def test_split_read_stream_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.SplitReadStreamRequest( + name='name_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.split_read_stream(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.SplitReadStreamRequest( + name='name_value', + ) + +def test_split_read_stream_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.split_read_stream in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.split_read_stream] = mock_rpc + request = {} + client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.split_read_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_split_read_stream_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.split_read_stream in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.split_read_stream] = mock_rpc + + request = {} + await client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.split_read_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_split_read_stream_async(transport: str = 'grpc_asyncio', request_type=storage.SplitReadStreamRequest): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse( + )) + response = await client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.SplitReadStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.SplitReadStreamResponse) + + +@pytest.mark.asyncio +async def test_split_read_stream_async_from_dict(): + await test_split_read_stream_async(request_type=dict) + +def test_split_read_stream_field_headers(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.SplitReadStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value = storage.SplitReadStreamResponse() + client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_split_read_stream_field_headers_async(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.SplitReadStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse()) + await client.split_read_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide an api_key and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options=options, + transport=transport, + ) + + # It is an error to provide an api_key and a credential. + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options=options, + credentials=ga_credentials.AnonymousCredentials() + ) + + # It is an error to provide scopes and a transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryReadClient( + client_options={"scopes": ["1", "2"]}, + transport=transport, + ) + + +def test_transport_instance(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + client = BigQueryReadClient(transport=transport) + assert client.transport is transport + +def test_transport_get_channel(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryReadGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + + transport = transports.BigQueryReadGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + +@pytest.mark.parametrize("transport_class", [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, +]) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + +def test_transport_kind_grpc(): + transport = BigQueryReadClient.get_transport_class("grpc")( + credentials=ga_credentials.AnonymousCredentials() + ) + assert transport.kind == "grpc" + + +def test_initialize_client_w_grpc(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc" + ) + assert client is not None + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_create_read_session_empty_call_grpc(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + call.return_value = stream.ReadSession() + client.create_read_session(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.CreateReadSessionRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_read_rows_empty_call_grpc(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + call.return_value = iter([storage.ReadRowsResponse()]) + client.read_rows(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.ReadRowsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_split_read_stream_empty_call_grpc(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + call.return_value = storage.SplitReadStreamResponse() + client.split_read_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.SplitReadStreamRequest() + + assert args[0] == request_msg + + +def test_transport_kind_grpc_asyncio(): + transport = BigQueryReadAsyncClient.get_transport_class("grpc_asyncio")( + credentials=async_anonymous_credentials() + ) + assert transport.kind == "grpc_asyncio" + + +def test_initialize_client_w_grpc_asyncio(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio" + ) + assert client is not None + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_create_read_session_empty_call_grpc_asyncio(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.create_read_session), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.ReadSession( + name='name_value', + data_format=stream.DataFormat.AVRO, + table='table_value', + )) + await client.create_read_session(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.CreateReadSessionRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_read_rows_empty_call_grpc_asyncio(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.read_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = mock.Mock(aio.UnaryStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.ReadRowsResponse()]) + await client.read_rows(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.ReadRowsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_split_read_stream_empty_call_grpc_asyncio(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.split_read_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.SplitReadStreamResponse( + )) + await client.split_read_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.SplitReadStreamRequest() + + assert args[0] == request_msg + + +def test_transport_grpc_default(): + # A client should use the gRPC transport by default. + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert isinstance( + client.transport, + transports.BigQueryReadGrpcTransport, + ) + +def test_big_query_read_base_transport_error(): + # Passing both a credentials object and credentials_file should raise an error + with pytest.raises(core_exceptions.DuplicateCredentialArgs): + transport = transports.BigQueryReadTransport( + credentials=ga_credentials.AnonymousCredentials(), + credentials_file="credentials.json" + ) + + +def test_big_query_read_base_transport(): + # Instantiate the base transport. + with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadTransport.__init__') as Transport: + Transport.return_value = None + transport = transports.BigQueryReadTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Every method on the transport should just blindly + # raise NotImplementedError. + methods = ( + 'create_read_session', + 'read_rows', + 'split_read_stream', + ) + for method in methods: + with pytest.raises(NotImplementedError): + getattr(transport, method)(request=object()) + + with pytest.raises(NotImplementedError): + transport.close() + + # Catch all for all remaining methods and properties + remainder = [ + 'kind', + ] + for r in remainder: + with pytest.raises(NotImplementedError): + getattr(transport, r)() + + +def test_big_query_read_base_transport_with_credentials_file(): + # Instantiate the base transport with a credentials file + with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryReadTransport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id="octopus", + ) + + +def test_big_query_read_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(google.auth, 'default', autospec=True) as adc, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_read.transports.BigQueryReadTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryReadTransport() + adc.assert_called_once() + + +def test_big_query_read_auth_adc(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryReadClient() + adc.assert_called_once_with( + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id=None, + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, + ], +) +def test_big_query_read_transport_auth_adc(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + adc.assert_called_once_with( + scopes=["1", "2"], + default_scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/cloud-platform',), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryReadGrpcTransport, + transports.BigQueryReadGrpcAsyncIOTransport, + ], +) +def test_big_query_read_transport_auth_gdch_credentials(transport_class): + host = 'https://language.com' + api_audience_tests = [None, 'https://language2.com'] + api_audience_expect = [host, 'https://language2.com'] + for t, e in zip(api_audience_tests, api_audience_expect): + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + gdch_mock = mock.MagicMock() + type(gdch_mock).with_gdch_audience = mock.PropertyMock(return_value=gdch_mock) + adc.return_value = (gdch_mock, None) + transport_class(host=host, api_audience=t) + gdch_mock.with_gdch_audience.assert_called_once_with( + e + ) + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.BigQueryReadGrpcTransport, grpc_helpers), + (transports.BigQueryReadGrpcAsyncIOTransport, grpc_helpers_async) + ], +) +def test_big_query_read_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class( + quota_project_id="octopus", + scopes=["1", "2"] + ) + + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=["1", "2"], + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_grpc_transport_client_cert_source_for_mtls( + transport_class +): + cred = ga_credentials.AnonymousCredentials() + + # Check ssl_channel_credentials is used if provided. + with mock.patch.object(transport_class, "create_channel") as mock_create_channel: + mock_ssl_channel_creds = mock.Mock() + transport_class( + host="squid.clam.whelk", + credentials=cred, + ssl_channel_credentials=mock_ssl_channel_creds + ) + mock_create_channel.assert_called_once_with( + "squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_channel_creds, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls + # is used. + with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): + with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: + transport_class( + credentials=cred, + client_cert_source_for_mtls=client_cert_source_callback + ) + expected_cert, expected_key = client_cert_source_callback() + mock_ssl_cred.assert_called_once_with( + certificate_chain=expected_cert, + private_key=expected_key + ) + + +@pytest.mark.parametrize("transport_name", [ + "grpc", + "grpc_asyncio", +]) +def test_big_query_read_host_no_port(transport_name): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com'), + transport=transport_name, + ) + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + +@pytest.mark.parametrize("transport_name", [ + "grpc", + "grpc_asyncio", +]) +def test_big_query_read_host_with_port(transport_name): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com:8000'), + transport=transport_name, + ) + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:8000' + ) + +def test_big_query_read_grpc_transport_channel(): + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryReadGrpcTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +def test_big_query_read_grpc_asyncio_transport_channel(): + channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryReadGrpcAsyncIOTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_transport_channel_mtls_with_client_cert_source( + transport_class +): + with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred: + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_ssl_cred = mock.Mock() + grpc_ssl_channel_cred.return_value = mock_ssl_cred + + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + + cred = ga_credentials.AnonymousCredentials() + with pytest.warns(DeprecationWarning): + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (cred, None) + transport = transport_class( + host="squid.clam.whelk", + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=client_cert_source_callback, + ) + adc.assert_called_once() + + grpc_ssl_channel_cred.assert_called_once_with( + certificate_chain=b"cert bytes", private_key=b"key bytes" + ) + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + assert transport._ssl_channel_credentials == mock_ssl_cred + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryReadGrpcTransport, transports.BigQueryReadGrpcAsyncIOTransport]) +def test_big_query_read_transport_channel_mtls_with_adc( + transport_class +): + mock_ssl_cred = mock.Mock() + with mock.patch.multiple( + "google.auth.transport.grpc.SslCredentials", + __init__=mock.Mock(return_value=None), + ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), + ): + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + mock_cred = mock.Mock() + + with pytest.warns(DeprecationWarning): + transport = transport_class( + host="squid.clam.whelk", + credentials=mock_cred, + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=None, + ) + + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=mock_cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + + +def test_read_session_path(): + project = "squid" + location = "clam" + session = "whelk" + expected = "projects/{project}/locations/{location}/sessions/{session}".format(project=project, location=location, session=session, ) + actual = BigQueryReadClient.read_session_path(project, location, session) + assert expected == actual + + +def test_parse_read_session_path(): + expected = { + "project": "octopus", + "location": "oyster", + "session": "nudibranch", + } + path = BigQueryReadClient.read_session_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_read_session_path(path) + assert expected == actual + +def test_read_stream_path(): + project = "cuttlefish" + location = "mussel" + session = "winkle" + stream = "nautilus" + expected = "projects/{project}/locations/{location}/sessions/{session}/streams/{stream}".format(project=project, location=location, session=session, stream=stream, ) + actual = BigQueryReadClient.read_stream_path(project, location, session, stream) + assert expected == actual + + +def test_parse_read_stream_path(): + expected = { + "project": "scallop", + "location": "abalone", + "session": "squid", + "stream": "clam", + } + path = BigQueryReadClient.read_stream_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_read_stream_path(path) + assert expected == actual + +def test_table_path(): + project = "whelk" + dataset = "octopus" + table = "oyster" + expected = "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + actual = BigQueryReadClient.table_path(project, dataset, table) + assert expected == actual + + +def test_parse_table_path(): + expected = { + "project": "nudibranch", + "dataset": "cuttlefish", + "table": "mussel", + } + path = BigQueryReadClient.table_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_table_path(path) + assert expected == actual + +def test_common_billing_account_path(): + billing_account = "winkle" + expected = "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + actual = BigQueryReadClient.common_billing_account_path(billing_account) + assert expected == actual + + +def test_parse_common_billing_account_path(): + expected = { + "billing_account": "nautilus", + } + path = BigQueryReadClient.common_billing_account_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_billing_account_path(path) + assert expected == actual + +def test_common_folder_path(): + folder = "scallop" + expected = "folders/{folder}".format(folder=folder, ) + actual = BigQueryReadClient.common_folder_path(folder) + assert expected == actual + + +def test_parse_common_folder_path(): + expected = { + "folder": "abalone", + } + path = BigQueryReadClient.common_folder_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_folder_path(path) + assert expected == actual + +def test_common_organization_path(): + organization = "squid" + expected = "organizations/{organization}".format(organization=organization, ) + actual = BigQueryReadClient.common_organization_path(organization) + assert expected == actual + + +def test_parse_common_organization_path(): + expected = { + "organization": "clam", + } + path = BigQueryReadClient.common_organization_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_organization_path(path) + assert expected == actual + +def test_common_project_path(): + project = "whelk" + expected = "projects/{project}".format(project=project, ) + actual = BigQueryReadClient.common_project_path(project) + assert expected == actual + + +def test_parse_common_project_path(): + expected = { + "project": "octopus", + } + path = BigQueryReadClient.common_project_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_project_path(path) + assert expected == actual + +def test_common_location_path(): + project = "oyster" + location = "nudibranch" + expected = "projects/{project}/locations/{location}".format(project=project, location=location, ) + actual = BigQueryReadClient.common_location_path(project, location) + assert expected == actual + + +def test_parse_common_location_path(): + expected = { + "project": "cuttlefish", + "location": "mussel", + } + path = BigQueryReadClient.common_location_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryReadClient.parse_common_location_path(path) + assert expected == actual + + +def test_client_with_default_client_info(): + client_info = gapic_v1.client_info.ClientInfo() + + with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + with mock.patch.object(transports.BigQueryReadTransport, '_prep_wrapped_messages') as prep: + transport_class = BigQueryReadClient.get_transport_class() + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + +def test_transport_close_grpc(): + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc" + ) + with mock.patch.object(type(getattr(client.transport, "_grpc_channel")), "close") as close: + with client: + close.assert_not_called() + close.assert_called_once() + + +@pytest.mark.asyncio +async def test_transport_close_grpc_asyncio(): + client = BigQueryReadAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio" + ) + with mock.patch.object(type(getattr(client.transport, "_grpc_channel")), "close") as close: + async with client: + close.assert_not_called() + close.assert_called_once() + + +def test_client_ctx(): + transports = [ + 'grpc', + ] + for transport in transports: + client = BigQueryReadClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport + ) + # Test client calls underlying transport. + with mock.patch.object(type(client.transport), "close") as close: + close.assert_not_called() + with client: + pass + close.assert_called() + +@pytest.mark.parametrize("client_class,transport_class", [ + (BigQueryReadClient, transports.BigQueryReadGrpcTransport), + (BigQueryReadAsyncClient, transports.BigQueryReadGrpcAsyncIOTransport), +]) +def test_api_key_credentials(client_class, transport_class): + with mock.patch.object( + google.auth._default, "get_api_key_credentials", create=True + ) as get_api_key_credentials: + mock_cred = mock.Mock() + get_api_key_credentials.return_value = mock_cred + options = client_options.ClientOptions() + options.api_key = "api_key" + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=mock_cred, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) diff --git a/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py new file mode 100644 index 00000000..831043e1 --- /dev/null +++ b/owl-bot-staging/v1beta2/tests/unit/gapic/bigquery_storage_v1beta2/test_big_query_write.py @@ -0,0 +1,3401 @@ +# -*- coding: utf-8 -*- +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import os +# try/except added for compatibility with python < 3.8 +try: + from unittest import mock + from unittest.mock import AsyncMock # pragma: NO COVER +except ImportError: # pragma: NO COVER + import mock + +import grpc +from grpc.experimental import aio +import math +import pytest +from google.api_core import api_core_version +from proto.marshal.rules.dates import DurationRule, TimestampRule +from proto.marshal.rules import wrappers + +try: + from google.auth.aio import credentials as ga_credentials_async + HAS_GOOGLE_AUTH_AIO = True +except ImportError: # pragma: NO COVER + HAS_GOOGLE_AUTH_AIO = False + +from google.api_core import client_options +from google.api_core import exceptions as core_exceptions +from google.api_core import gapic_v1 +from google.api_core import grpc_helpers +from google.api_core import grpc_helpers_async +from google.api_core import path_template +from google.api_core import retry as retries +from google.auth import credentials as ga_credentials +from google.auth.exceptions import MutualTLSChannelError +from google.cloud.bigquery_storage_v1beta2.services.big_query_write import BigQueryWriteAsyncClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_write import BigQueryWriteClient +from google.cloud.bigquery_storage_v1beta2.services.big_query_write import transports +from google.cloud.bigquery_storage_v1beta2.types import protobuf +from google.cloud.bigquery_storage_v1beta2.types import storage +from google.cloud.bigquery_storage_v1beta2.types import stream +from google.cloud.bigquery_storage_v1beta2.types import table +from google.oauth2 import service_account +from google.protobuf import descriptor_pb2 # type: ignore +from google.protobuf import timestamp_pb2 # type: ignore +from google.protobuf import wrappers_pb2 # type: ignore +from google.rpc import status_pb2 # type: ignore +import google.auth + + +async def mock_async_gen(data, chunk_size=1): + for i in range(0, len(data)): # pragma: NO COVER + chunk = data[i : i + chunk_size] + yield chunk.encode("utf-8") + +def client_cert_source_callback(): + return b"cert bytes", b"key bytes" + +# TODO: use async auth anon credentials by default once the minimum version of google-auth is upgraded. +# See related issue: https://github.com/googleapis/gapic-generator-python/issues/2107. +def async_anonymous_credentials(): + if HAS_GOOGLE_AUTH_AIO: + return ga_credentials_async.AnonymousCredentials() + return ga_credentials.AnonymousCredentials() + +# If default endpoint is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint(client): + return "foo.googleapis.com" if ("localhost" in client.DEFAULT_ENDPOINT) else client.DEFAULT_ENDPOINT + +# If default endpoint template is localhost, then default mtls endpoint will be the same. +# This method modifies the default endpoint template so the client can produce a different +# mtls endpoint for endpoint testing purposes. +def modify_default_endpoint_template(client): + return "test.{UNIVERSE_DOMAIN}" if ("localhost" in client._DEFAULT_ENDPOINT_TEMPLATE) else client._DEFAULT_ENDPOINT_TEMPLATE + + +def test__get_default_mtls_endpoint(): + api_endpoint = "example.googleapis.com" + api_mtls_endpoint = "example.mtls.googleapis.com" + sandbox_endpoint = "example.sandbox.googleapis.com" + sandbox_mtls_endpoint = "example.mtls.sandbox.googleapis.com" + non_googleapi = "api.example.com" + + assert BigQueryWriteClient._get_default_mtls_endpoint(None) is None + assert BigQueryWriteClient._get_default_mtls_endpoint(api_endpoint) == api_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(api_mtls_endpoint) == api_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(sandbox_endpoint) == sandbox_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(sandbox_mtls_endpoint) == sandbox_mtls_endpoint + assert BigQueryWriteClient._get_default_mtls_endpoint(non_googleapi) == non_googleapi + +def test__read_environment_variables(): + assert BigQueryWriteClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + assert BigQueryWriteClient._read_environment_variables() == (True, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + assert BigQueryWriteClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + BigQueryWriteClient._read_environment_variables() + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + assert BigQueryWriteClient._read_environment_variables() == (False, "never", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + assert BigQueryWriteClient._read_environment_variables() == (False, "always", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}): + assert BigQueryWriteClient._read_environment_variables() == (False, "auto", None) + + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + BigQueryWriteClient._read_environment_variables() + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + with mock.patch.dict(os.environ, {"GOOGLE_CLOUD_UNIVERSE_DOMAIN": "foo.com"}): + assert BigQueryWriteClient._read_environment_variables() == (False, "auto", "foo.com") + +def test__get_client_cert_source(): + mock_provided_cert_source = mock.Mock() + mock_default_cert_source = mock.Mock() + + assert BigQueryWriteClient._get_client_cert_source(None, False) is None + assert BigQueryWriteClient._get_client_cert_source(mock_provided_cert_source, False) is None + assert BigQueryWriteClient._get_client_cert_source(mock_provided_cert_source, True) == mock_provided_cert_source + + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=mock_default_cert_source): + assert BigQueryWriteClient._get_client_cert_source(None, True) is mock_default_cert_source + assert BigQueryWriteClient._get_client_cert_source(mock_provided_cert_source, "true") is mock_provided_cert_source + +@mock.patch.object(BigQueryWriteClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteAsyncClient)) +def test__get_api_endpoint(): + api_override = "foo.com" + mock_client_cert_source = mock.Mock() + default_universe = BigQueryWriteClient._DEFAULT_UNIVERSE + default_endpoint = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=default_universe) + mock_universe = "bar.com" + mock_endpoint = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=mock_universe) + + assert BigQueryWriteClient._get_api_endpoint(api_override, mock_client_cert_source, default_universe, "always") == api_override + assert BigQueryWriteClient._get_api_endpoint(None, mock_client_cert_source, default_universe, "auto") == BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryWriteClient._get_api_endpoint(None, None, default_universe, "auto") == default_endpoint + assert BigQueryWriteClient._get_api_endpoint(None, None, default_universe, "always") == BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryWriteClient._get_api_endpoint(None, mock_client_cert_source, default_universe, "always") == BigQueryWriteClient.DEFAULT_MTLS_ENDPOINT + assert BigQueryWriteClient._get_api_endpoint(None, None, mock_universe, "never") == mock_endpoint + assert BigQueryWriteClient._get_api_endpoint(None, None, default_universe, "never") == default_endpoint + + with pytest.raises(MutualTLSChannelError) as excinfo: + BigQueryWriteClient._get_api_endpoint(None, mock_client_cert_source, mock_universe, "auto") + assert str(excinfo.value) == "mTLS is not supported in any universe other than googleapis.com." + + +def test__get_universe_domain(): + client_universe_domain = "foo.com" + universe_domain_env = "bar.com" + + assert BigQueryWriteClient._get_universe_domain(client_universe_domain, universe_domain_env) == client_universe_domain + assert BigQueryWriteClient._get_universe_domain(None, universe_domain_env) == universe_domain_env + assert BigQueryWriteClient._get_universe_domain(None, None) == BigQueryWriteClient._DEFAULT_UNIVERSE + + with pytest.raises(ValueError) as excinfo: + BigQueryWriteClient._get_universe_domain("", None) + assert str(excinfo.value) == "Universe Domain cannot be an empty string." + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), +]) +def test__validate_universe_domain(client_class, transport_class, transport_name): + client = client_class( + transport=transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + ) + assert client._validate_universe_domain() == True + + # Test the case when universe is already validated. + assert client._validate_universe_domain() == True + + if transport_name == "grpc": + # Test the case where credentials are provided by the + # `local_channel_credentials`. The default universes in both match. + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + client = client_class(transport=transport_class(channel=channel)) + assert client._validate_universe_domain() == True + + # Test the case where credentials do not exist: e.g. a transport is provided + # with no credentials. Validation should still succeed because there is no + # mismatch with non-existent credentials. + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + transport=transport_class(channel=channel) + transport._credentials = None + client = client_class(transport=transport) + assert client._validate_universe_domain() == True + + # TODO: This is needed to cater for older versions of google-auth + # Make this test unconditional once the minimum supported version of + # google-auth becomes 2.23.0 or higher. + google_auth_major, google_auth_minor = [int(part) for part in google.auth.__version__.split(".")[0:2]] + if google_auth_major > 2 or (google_auth_major == 2 and google_auth_minor >= 23): + credentials = ga_credentials.AnonymousCredentials() + credentials._universe_domain = "foo.com" + # Test the case when there is a universe mismatch from the credentials. + client = client_class( + transport=transport_class(credentials=credentials) + ) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert str(excinfo.value) == "The configured universe domain (googleapis.com) does not match the universe domain found in the credentials (foo.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + + # Test the case when there is a universe mismatch from the client. + # + # TODO: Make this test unconditional once the minimum supported version of + # google-api-core becomes 2.15.0 or higher. + api_core_major, api_core_minor = [int(part) for part in api_core_version.__version__.split(".")[0:2]] + if api_core_major > 2 or (api_core_major == 2 and api_core_minor >= 15): + client = client_class(client_options={"universe_domain": "bar.com"}, transport=transport_class(credentials=ga_credentials.AnonymousCredentials(),)) + with pytest.raises(ValueError) as excinfo: + client._validate_universe_domain() + assert str(excinfo.value) == "The configured universe domain (bar.com) does not match the universe domain found in the credentials (googleapis.com). If you haven't configured the universe domain explicitly, `googleapis.com` is the default." + + # Test that ValueError is raised if universe_domain is provided via client options and credentials is None + with pytest.raises(ValueError): + client._compare_universes("foo.bar", None) + + +@pytest.mark.parametrize("client_class,transport_name", [ + (BigQueryWriteClient, "grpc"), + (BigQueryWriteAsyncClient, "grpc_asyncio"), +]) +def test_big_query_write_client_from_service_account_info(client_class, transport_name): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_info') as factory: + factory.return_value = creds + info = {"valid": True} + client = client_class.from_service_account_info(info, transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + + +@pytest.mark.parametrize("transport_class,transport_name", [ + (transports.BigQueryWriteGrpcTransport, "grpc"), + (transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_write_client_service_account_always_use_jwt(transport_class, transport_name): + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=True) + use_jwt.assert_called_once_with(True) + + with mock.patch.object(service_account.Credentials, 'with_always_use_jwt_access', create=True) as use_jwt: + creds = service_account.Credentials(None, None, None) + transport = transport_class(credentials=creds, always_use_jwt_access=False) + use_jwt.assert_not_called() + + +@pytest.mark.parametrize("client_class,transport_name", [ + (BigQueryWriteClient, "grpc"), + (BigQueryWriteAsyncClient, "grpc_asyncio"), +]) +def test_big_query_write_client_from_service_account_file(client_class, transport_name): + creds = ga_credentials.AnonymousCredentials() + with mock.patch.object(service_account.Credentials, 'from_service_account_file') as factory: + factory.return_value = creds + client = client_class.from_service_account_file("dummy/file/path.json", transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + client = client_class.from_service_account_json("dummy/file/path.json", transport=transport_name) + assert client.transport._credentials == creds + assert isinstance(client, client_class) + + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + + +def test_big_query_write_client_get_transport_class(): + transport = BigQueryWriteClient.get_transport_class() + available_transports = [ + transports.BigQueryWriteGrpcTransport, + ] + assert transport in available_transports + + transport = BigQueryWriteClient.get_transport_class("grpc") + assert transport == transports.BigQueryWriteGrpcTransport + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), +]) +@mock.patch.object(BigQueryWriteClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteAsyncClient)) +def test_big_query_write_client_client_options(client_class, transport_class, transport_name): + # Check that if channel is provided we won't create a new one. + with mock.patch.object(BigQueryWriteClient, 'get_transport_class') as gtc: + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials() + ) + client = client_class(transport=transport) + gtc.assert_not_called() + + # Check that if channel is provided via str we will create a new one. + with mock.patch.object(BigQueryWriteClient, 'get_transport_class') as gtc: + client = client_class(transport=transport_name) + gtc.assert_called() + + # Check the case api_endpoint is provided. + options = client_options.ClientOptions(api_endpoint="squid.clam.whelk") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name, client_options=options) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT is + # "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client.DEFAULT_MTLS_ENDPOINT, + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client = client_class(transport=transport_name) + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + client = client_class(transport=transport_name) + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + + # Check the case quota_project_id is provided + options = client_options.ClientOptions(quota_project_id="octopus") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id="octopus", + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + # Check the case api_endpoint is provided + options = client_options.ClientOptions(api_audience="https://language.googleapis.com") + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience="https://language.googleapis.com" + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,use_client_cert_env", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", "true"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", "true"), + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", "false"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", "false"), +]) +@mock.patch.object(BigQueryWriteClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteAsyncClient)) +@mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "auto"}) +def test_big_query_write_client_mtls_env_auto(client_class, transport_class, transport_name, use_client_cert_env): + # This tests the endpoint autoswitch behavior. Endpoint is autoswitched to the default + # mtls endpoint, if GOOGLE_API_USE_CLIENT_CERTIFICATE is "true" and client cert exists. + + # Check the case client_cert_source is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + options = client_options.ClientOptions(client_cert_source=client_cert_source_callback) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + + if use_client_cert_env == "false": + expected_client_cert_source = None + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE) + else: + expected_client_cert_source = client_cert_source_callback + expected_host = client.DEFAULT_MTLS_ENDPOINT + + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case ADC client cert is provided. Whether client cert is used depends on + # GOOGLE_API_USE_CLIENT_CERTIFICATE value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=client_cert_source_callback): + if use_client_cert_env == "false": + expected_host = client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE) + expected_client_cert_source = None + else: + expected_host = client.DEFAULT_MTLS_ENDPOINT + expected_client_cert_source = client_cert_source_callback + + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=expected_host, + scopes=None, + client_cert_source_for_mtls=expected_client_cert_source, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # Check the case client_cert_source and ADC client cert are not provided. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": use_client_cert_env}): + with mock.patch.object(transport_class, '__init__') as patched: + with mock.patch("google.auth.transport.mtls.has_default_client_cert_source", return_value=False): + patched.return_value = None + client = client_class(transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + +@pytest.mark.parametrize("client_class", [ + BigQueryWriteClient, BigQueryWriteAsyncClient +]) +@mock.patch.object(BigQueryWriteClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "DEFAULT_ENDPOINT", modify_default_endpoint(BigQueryWriteAsyncClient)) +def test_big_query_write_client_get_mtls_endpoint_and_cert_source(client_class): + mock_client_cert_source = mock.Mock() + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "true". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + mock_api_endpoint = "foo" + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source == mock_client_cert_source + + # Test the case GOOGLE_API_USE_CLIENT_CERTIFICATE is "false". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "false"}): + mock_client_cert_source = mock.Mock() + mock_api_endpoint = "foo" + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=mock_api_endpoint) + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source(options) + assert api_endpoint == mock_api_endpoint + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "never". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "always". + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert doesn't exist. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=False): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_ENDPOINT + assert cert_source is None + + # Test the case GOOGLE_API_USE_MTLS_ENDPOINT is "auto" and default cert exists. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch('google.auth.transport.mtls.has_default_client_cert_source', return_value=True): + with mock.patch('google.auth.transport.mtls.default_client_cert_source', return_value=mock_client_cert_source): + api_endpoint, cert_source = client_class.get_mtls_endpoint_and_cert_source() + assert api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + assert cert_source == mock_client_cert_source + + # Check the case api_endpoint is not provided and GOOGLE_API_USE_MTLS_ENDPOINT has + # unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "Unsupported"}): + with pytest.raises(MutualTLSChannelError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_MTLS_ENDPOINT` must be `never`, `auto` or `always`" + + # Check the case GOOGLE_API_USE_CLIENT_CERTIFICATE has unsupported value. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "Unsupported"}): + with pytest.raises(ValueError) as excinfo: + client_class.get_mtls_endpoint_and_cert_source() + + assert str(excinfo.value) == "Environment variable `GOOGLE_API_USE_CLIENT_CERTIFICATE` must be either `true` or `false`" + +@pytest.mark.parametrize("client_class", [ + BigQueryWriteClient, BigQueryWriteAsyncClient +]) +@mock.patch.object(BigQueryWriteClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteClient)) +@mock.patch.object(BigQueryWriteAsyncClient, "_DEFAULT_ENDPOINT_TEMPLATE", modify_default_endpoint_template(BigQueryWriteAsyncClient)) +def test_big_query_write_client_client_api_endpoint(client_class): + mock_client_cert_source = client_cert_source_callback + api_override = "foo.com" + default_universe = BigQueryWriteClient._DEFAULT_UNIVERSE + default_endpoint = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=default_universe) + mock_universe = "bar.com" + mock_endpoint = BigQueryWriteClient._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=mock_universe) + + # If ClientOptions.api_endpoint is set and GOOGLE_API_USE_CLIENT_CERTIFICATE="true", + # use ClientOptions.api_endpoint as the api endpoint regardless. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_CLIENT_CERTIFICATE": "true"}): + with mock.patch("google.auth.transport.requests.AuthorizedSession.configure_mtls_channel"): + options = client_options.ClientOptions(client_cert_source=mock_client_cert_source, api_endpoint=api_override) + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == api_override + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + # If ClientOptions.api_endpoint is not set and GOOGLE_API_USE_MTLS_ENDPOINT="always", + # use the DEFAULT_MTLS_ENDPOINT as the api endpoint. + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "always"}): + client = client_class(credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == client_class.DEFAULT_MTLS_ENDPOINT + + # If ClientOptions.api_endpoint is not set, GOOGLE_API_USE_MTLS_ENDPOINT="auto" (default), + # GOOGLE_API_USE_CLIENT_CERTIFICATE="false" (default), default cert source doesn't exist, + # and ClientOptions.universe_domain="bar.com", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with universe domain as the api endpoint. + options = client_options.ClientOptions() + universe_exists = hasattr(options, "universe_domain") + if universe_exists: + options = client_options.ClientOptions(universe_domain=mock_universe) + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + else: + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == (mock_endpoint if universe_exists else default_endpoint) + assert client.universe_domain == (mock_universe if universe_exists else default_universe) + + # If ClientOptions does not have a universe domain attribute and GOOGLE_API_USE_MTLS_ENDPOINT="never", + # use the _DEFAULT_ENDPOINT_TEMPLATE populated with GDU as the api endpoint. + options = client_options.ClientOptions() + if hasattr(options, "universe_domain"): + delattr(options, "universe_domain") + with mock.patch.dict(os.environ, {"GOOGLE_API_USE_MTLS_ENDPOINT": "never"}): + client = client_class(client_options=options, credentials=ga_credentials.AnonymousCredentials()) + assert client.api_endpoint == default_endpoint + + +@pytest.mark.parametrize("client_class,transport_class,transport_name", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc"), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio"), +]) +def test_big_query_write_client_client_options_scopes(client_class, transport_class, transport_name): + # Check the case scopes are provided. + options = client_options.ClientOptions( + scopes=["1", "2"], + ) + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=["1", "2"], + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + +@pytest.mark.parametrize("client_class,transport_class,transport_name,grpc_helpers", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", grpc_helpers), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", grpc_helpers_async), +]) +def test_big_query_write_client_client_options_credentials_file(client_class, transport_class, transport_name, grpc_helpers): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + +def test_big_query_write_client_client_options_from_dict(): + with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteGrpcTransport.__init__') as grpc_transport: + grpc_transport.return_value = None + client = BigQueryWriteClient( + client_options={'api_endpoint': 'squid.clam.whelk'} + ) + grpc_transport.assert_called_once_with( + credentials=None, + credentials_file=None, + host="squid.clam.whelk", + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + +@pytest.mark.parametrize("client_class,transport_class,transport_name,grpc_helpers", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport, "grpc", grpc_helpers), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport, "grpc_asyncio", grpc_helpers_async), +]) +def test_big_query_write_client_create_channel_credentials_file(client_class, transport_class, transport_name, grpc_helpers): + # Check the case credentials file is provided. + options = client_options.ClientOptions( + credentials_file="credentials.json" + ) + + with mock.patch.object(transport_class, '__init__') as patched: + patched.return_value = None + client = client_class(client_options=options, transport=transport_name) + patched.assert_called_once_with( + credentials=None, + credentials_file="credentials.json", + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + ) + + # test that the credentials from file are saved and used as the credentials. + with mock.patch.object( + google.auth, "load_credentials_from_file", autospec=True + ) as load_creds, mock.patch.object( + google.auth, "default", autospec=True + ) as adc, mock.patch.object( + grpc_helpers, "create_channel" + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + file_creds = ga_credentials.AnonymousCredentials() + load_creds.return_value = (file_creds, None) + adc.return_value = (creds, None) + client = client_class(client_options=options, transport=transport_name) + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=file_creds, + credentials_file=None, + quota_project_id=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=None, + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("request_type", [ + storage.CreateWriteStreamRequest, + dict, +]) +def test_create_write_stream(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + ) + response = client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.CreateWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +def test_create_write_stream_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.CreateWriteStreamRequest( + parent='parent_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.create_write_stream(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.CreateWriteStreamRequest( + parent='parent_value', + ) + +def test_create_write_stream_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.create_write_stream in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.create_write_stream] = mock_rpc + request = {} + client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.create_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_create_write_stream_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.create_write_stream in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.create_write_stream] = mock_rpc + + request = {} + await client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.create_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_create_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.CreateWriteStreamRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + )) + response = await client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.CreateWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +@pytest.mark.asyncio +async def test_create_write_stream_async_from_dict(): + await test_create_write_stream_async(request_type=dict) + +def test_create_write_stream_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateWriteStreamRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + call.return_value = stream.WriteStream() + client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_create_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.CreateWriteStreamRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + await client.create_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +def test_create_write_stream_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.create_write_stream( + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + arg = args[0].write_stream + mock_val = stream.WriteStream(name='name_value') + assert arg == mock_val + + +def test_create_write_stream_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.create_write_stream( + storage.CreateWriteStreamRequest(), + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + +@pytest.mark.asyncio +async def test_create_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.create_write_stream( + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + arg = args[0].write_stream + mock_val = stream.WriteStream(name='name_value') + assert arg == mock_val + +@pytest.mark.asyncio +async def test_create_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.create_write_stream( + storage.CreateWriteStreamRequest(), + parent='parent_value', + write_stream=stream.WriteStream(name='name_value'), + ) + + +@pytest.mark.parametrize("request_type", [ + storage.AppendRowsRequest, + dict, +]) +def test_append_rows(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + requests = [request] + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.append_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = iter([storage.AppendRowsResponse()]) + response = client.append_rows(iter(requests)) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert next(args[0]) == request + + # Establish that the response is the type that we expect. + for message in response: + assert isinstance(message, storage.AppendRowsResponse) + + +def test_append_rows_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.append_rows in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.append_rows] = mock_rpc + request = [{}] + client.append_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.append_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_append_rows_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.append_rows in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.append_rows] = mock_rpc + + request = [{}] + await client.append_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.append_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_append_rows_async(transport: str = 'grpc_asyncio', request_type=storage.AppendRowsRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + requests = [request] + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.append_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = mock.Mock(aio.StreamStreamCall, autospec=True) + call.return_value.read = mock.AsyncMock(side_effect=[storage.AppendRowsResponse()]) + response = await client.append_rows(iter(requests)) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert next(args[0]) == request + + # Establish that the response is the type that we expect. + message = await response.read() + assert isinstance(message, storage.AppendRowsResponse) + + +@pytest.mark.asyncio +async def test_append_rows_async_from_dict(): + await test_append_rows_async(request_type=dict) + + +@pytest.mark.parametrize("request_type", [ + storage.GetWriteStreamRequest, + dict, +]) +def test_get_write_stream(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + ) + response = client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.GetWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +def test_get_write_stream_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.GetWriteStreamRequest( + name='name_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.get_write_stream(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.GetWriteStreamRequest( + name='name_value', + ) + +def test_get_write_stream_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.get_write_stream in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.get_write_stream] = mock_rpc + request = {} + client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.get_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_get_write_stream_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.get_write_stream in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.get_write_stream] = mock_rpc + + request = {} + await client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.get_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_get_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.GetWriteStreamRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + )) + response = await client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.GetWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, stream.WriteStream) + assert response.name == 'name_value' + assert response.type_ == stream.WriteStream.Type.COMMITTED + + +@pytest.mark.asyncio +async def test_get_write_stream_async_from_dict(): + await test_get_write_stream_async(request_type=dict) + +def test_get_write_stream_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.GetWriteStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + call.return_value = stream.WriteStream() + client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_get_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.GetWriteStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + await client.get_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +def test_get_write_stream_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.get_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].name + mock_val = 'name_value' + assert arg == mock_val + + +def test_get_write_stream_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.get_write_stream( + storage.GetWriteStreamRequest(), + name='name_value', + ) + +@pytest.mark.asyncio +async def test_get_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = stream.WriteStream() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.get_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].name + mock_val = 'name_value' + assert arg == mock_val + +@pytest.mark.asyncio +async def test_get_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.get_write_stream( + storage.GetWriteStreamRequest(), + name='name_value', + ) + + +@pytest.mark.parametrize("request_type", [ + storage.FinalizeWriteStreamRequest, + dict, +]) +def test_finalize_write_stream(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse( + row_count=992, + ) + response = client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.FinalizeWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FinalizeWriteStreamResponse) + assert response.row_count == 992 + + +def test_finalize_write_stream_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.FinalizeWriteStreamRequest( + name='name_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.finalize_write_stream(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FinalizeWriteStreamRequest( + name='name_value', + ) + +def test_finalize_write_stream_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.finalize_write_stream in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.finalize_write_stream] = mock_rpc + request = {} + client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.finalize_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_finalize_write_stream_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.finalize_write_stream in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.finalize_write_stream] = mock_rpc + + request = {} + await client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.finalize_write_stream(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_finalize_write_stream_async(transport: str = 'grpc_asyncio', request_type=storage.FinalizeWriteStreamRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse( + row_count=992, + )) + response = await client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.FinalizeWriteStreamRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FinalizeWriteStreamResponse) + assert response.row_count == 992 + + +@pytest.mark.asyncio +async def test_finalize_write_stream_async_from_dict(): + await test_finalize_write_stream_async(request_type=dict) + +def test_finalize_write_stream_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FinalizeWriteStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + call.return_value = storage.FinalizeWriteStreamResponse() + client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_finalize_write_stream_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FinalizeWriteStreamRequest() + + request.name = 'name_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse()) + await client.finalize_write_stream(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'name=name_value', + ) in kw['metadata'] + + +def test_finalize_write_stream_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.finalize_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].name + mock_val = 'name_value' + assert arg == mock_val + + +def test_finalize_write_stream_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.finalize_write_stream( + storage.FinalizeWriteStreamRequest(), + name='name_value', + ) + +@pytest.mark.asyncio +async def test_finalize_write_stream_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FinalizeWriteStreamResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.finalize_write_stream( + name='name_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].name + mock_val = 'name_value' + assert arg == mock_val + +@pytest.mark.asyncio +async def test_finalize_write_stream_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.finalize_write_stream( + storage.FinalizeWriteStreamRequest(), + name='name_value', + ) + + +@pytest.mark.parametrize("request_type", [ + storage.BatchCommitWriteStreamsRequest, + dict, +]) +def test_batch_commit_write_streams(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse( + ) + response = client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.BatchCommitWriteStreamsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.BatchCommitWriteStreamsResponse) + + +def test_batch_commit_write_streams_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.BatchCommitWriteStreamsRequest( + parent='parent_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.batch_commit_write_streams(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.BatchCommitWriteStreamsRequest( + parent='parent_value', + ) + +def test_batch_commit_write_streams_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.batch_commit_write_streams in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.batch_commit_write_streams] = mock_rpc + request = {} + client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.batch_commit_write_streams(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.batch_commit_write_streams in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.batch_commit_write_streams] = mock_rpc + + request = {} + await client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.batch_commit_write_streams(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_async(transport: str = 'grpc_asyncio', request_type=storage.BatchCommitWriteStreamsRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse( + )) + response = await client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.BatchCommitWriteStreamsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.BatchCommitWriteStreamsResponse) + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_async_from_dict(): + await test_batch_commit_write_streams_async(request_type=dict) + +def test_batch_commit_write_streams_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.BatchCommitWriteStreamsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + call.return_value = storage.BatchCommitWriteStreamsResponse() + client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.BatchCommitWriteStreamsRequest() + + request.parent = 'parent_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse()) + await client.batch_commit_write_streams(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'parent=parent_value', + ) in kw['metadata'] + + +def test_batch_commit_write_streams_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.batch_commit_write_streams( + parent='parent_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + + +def test_batch_commit_write_streams_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.batch_commit_write_streams( + storage.BatchCommitWriteStreamsRequest(), + parent='parent_value', + ) + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.BatchCommitWriteStreamsResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.batch_commit_write_streams( + parent='parent_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].parent + mock_val = 'parent_value' + assert arg == mock_val + +@pytest.mark.asyncio +async def test_batch_commit_write_streams_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.batch_commit_write_streams( + storage.BatchCommitWriteStreamsRequest(), + parent='parent_value', + ) + + +@pytest.mark.parametrize("request_type", [ + storage.FlushRowsRequest, + dict, +]) +def test_flush_rows(request_type, transport: str = 'grpc'): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse( + offset=647, + ) + response = client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + request = storage.FlushRowsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FlushRowsResponse) + assert response.offset == 647 + + +def test_flush_rows_non_empty_request_with_auto_populated_field(): + # This test is a coverage failsafe to make sure that UUID4 fields are + # automatically populated, according to AIP-4235, with non-empty requests. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport='grpc', + ) + + # Populate all string fields in the request which are not UUID4 + # since we want to check that UUID4 are populated automatically + # if they meet the requirements of AIP 4235. + request = storage.FlushRowsRequest( + write_stream='write_stream_value', + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + call.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client.flush_rows(request=request) + call.assert_called() + _, args, _ = call.mock_calls[0] + assert args[0] == storage.FlushRowsRequest( + write_stream='write_stream_value', + ) + +def test_flush_rows_use_cached_wrapped_rpc(): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method.wrap_method") as wrapper_fn: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._transport.flush_rows in client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.Mock() + mock_rpc.return_value.name = "foo" # operation_request.operation in compute client(s) expect a string. + client._transport._wrapped_methods[client._transport.flush_rows] = mock_rpc + request = {} + client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + client.flush_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_flush_rows_async_use_cached_wrapped_rpc(transport: str = "grpc_asyncio"): + # Clients should use _prep_wrapped_messages to create cached wrapped rpcs, + # instead of constructing them on each call + with mock.patch("google.api_core.gapic_v1.method_async.wrap_method") as wrapper_fn: + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Should wrap all calls on client creation + assert wrapper_fn.call_count > 0 + wrapper_fn.reset_mock() + + # Ensure method has been cached + assert client._client._transport.flush_rows in client._client._transport._wrapped_methods + + # Replace cached wrapped function with mock + mock_rpc = mock.AsyncMock() + mock_rpc.return_value = mock.Mock() + client._client._transport._wrapped_methods[client._client._transport.flush_rows] = mock_rpc + + request = {} + await client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert mock_rpc.call_count == 1 + + await client.flush_rows(request) + + # Establish that a new wrapper was not created for this call + assert wrapper_fn.call_count == 0 + assert mock_rpc.call_count == 2 + +@pytest.mark.asyncio +async def test_flush_rows_async(transport: str = 'grpc_asyncio', request_type=storage.FlushRowsRequest): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport=transport, + ) + + # Everything is optional in proto3 as far as the runtime is concerned, + # and we are mocking out the actual API, so just send an empty request. + request = request_type() + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value =grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse( + offset=647, + )) + response = await client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + request = storage.FlushRowsRequest() + assert args[0] == request + + # Establish that the response is the type that we expect. + assert isinstance(response, storage.FlushRowsResponse) + assert response.offset == 647 + + +@pytest.mark.asyncio +async def test_flush_rows_async_from_dict(): + await test_flush_rows_async(request_type=dict) + +def test_flush_rows_field_headers(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FlushRowsRequest() + + request.write_stream = 'write_stream_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + call.return_value = storage.FlushRowsResponse() + client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'write_stream=write_stream_value', + ) in kw['metadata'] + + +@pytest.mark.asyncio +async def test_flush_rows_field_headers_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Any value that is part of the HTTP/1.1 URI should be sent as + # a field header. Set these to a non-empty value. + request = storage.FlushRowsRequest() + + request.write_stream = 'write_stream_value' + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse()) + await client.flush_rows(request) + + # Establish that the underlying gRPC stub method was called. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + assert args[0] == request + + # Establish that the field header was sent. + _, _, kw = call.mock_calls[0] + assert ( + 'x-goog-request-params', + 'write_stream=write_stream_value', + ) in kw['metadata'] + + +def test_flush_rows_flattened(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse() + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + client.flush_rows( + write_stream='write_stream_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) == 1 + _, args, _ = call.mock_calls[0] + arg = args[0].write_stream + mock_val = 'write_stream_value' + assert arg == mock_val + + +def test_flush_rows_flattened_error(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + client.flush_rows( + storage.FlushRowsRequest(), + write_stream='write_stream_value', + ) + +@pytest.mark.asyncio +async def test_flush_rows_flattened_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Mock the actual call within the gRPC stub, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = storage.FlushRowsResponse() + + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse()) + # Call the method with a truthy value for each flattened field, + # using the keyword arguments to the method. + response = await client.flush_rows( + write_stream='write_stream_value', + ) + + # Establish that the underlying call was made with the expected + # request object values. + assert len(call.mock_calls) + _, args, _ = call.mock_calls[0] + arg = args[0].write_stream + mock_val = 'write_stream_value' + assert arg == mock_val + +@pytest.mark.asyncio +async def test_flush_rows_flattened_error_async(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + ) + + # Attempting to call a method with both a request object and flattened + # fields is an error. + with pytest.raises(ValueError): + await client.flush_rows( + storage.FlushRowsRequest(), + write_stream='write_stream_value', + ) + + +def test_credentials_transport_error(): + # It is an error to provide credentials and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport, + ) + + # It is an error to provide a credentials file and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options={"credentials_file": "credentials.json"}, + transport=transport, + ) + + # It is an error to provide an api_key and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options=options, + transport=transport, + ) + + # It is an error to provide an api_key and a credential. + options = client_options.ClientOptions() + options.api_key = "api_key" + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options=options, + credentials=ga_credentials.AnonymousCredentials() + ) + + # It is an error to provide scopes and a transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + with pytest.raises(ValueError): + client = BigQueryWriteClient( + client_options={"scopes": ["1", "2"]}, + transport=transport, + ) + + +def test_transport_instance(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + client = BigQueryWriteClient(transport=transport) + assert client.transport is transport + +def test_transport_get_channel(): + # A client may be instantiated with a custom transport instance. + transport = transports.BigQueryWriteGrpcTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + + transport = transports.BigQueryWriteGrpcAsyncIOTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + channel = transport.grpc_channel + assert channel + +@pytest.mark.parametrize("transport_class", [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, +]) +def test_transport_adc(transport_class): + # Test default credentials are used if not provided. + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class() + adc.assert_called_once() + +def test_transport_kind_grpc(): + transport = BigQueryWriteClient.get_transport_class("grpc")( + credentials=ga_credentials.AnonymousCredentials() + ) + assert transport.kind == "grpc" + + +def test_initialize_client_w_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc" + ) + assert client is not None + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_create_write_stream_empty_call_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + call.return_value = stream.WriteStream() + client.create_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.CreateWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_get_write_stream_empty_call_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + call.return_value = stream.WriteStream() + client.get_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.GetWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_finalize_write_stream_empty_call_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + call.return_value = storage.FinalizeWriteStreamResponse() + client.finalize_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.FinalizeWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_batch_commit_write_streams_empty_call_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + call.return_value = storage.BatchCommitWriteStreamsResponse() + client.batch_commit_write_streams(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.BatchCommitWriteStreamsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +def test_flush_rows_empty_call_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + call.return_value = storage.FlushRowsResponse() + client.flush_rows(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.FlushRowsRequest() + + assert args[0] == request_msg + + +def test_transport_kind_grpc_asyncio(): + transport = BigQueryWriteAsyncClient.get_transport_class("grpc_asyncio")( + credentials=async_anonymous_credentials() + ) + assert transport.kind == "grpc_asyncio" + + +def test_initialize_client_w_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio" + ) + assert client is not None + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_create_write_stream_empty_call_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.create_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + )) + await client.create_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.CreateWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_get_write_stream_empty_call_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.get_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(stream.WriteStream( + name='name_value', + type_=stream.WriteStream.Type.COMMITTED, + )) + await client.get_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.GetWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_finalize_write_stream_empty_call_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.finalize_write_stream), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FinalizeWriteStreamResponse( + row_count=992, + )) + await client.finalize_write_stream(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.FinalizeWriteStreamRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_batch_commit_write_streams_empty_call_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.batch_commit_write_streams), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.BatchCommitWriteStreamsResponse( + )) + await client.batch_commit_write_streams(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.BatchCommitWriteStreamsRequest() + + assert args[0] == request_msg + + +# This test is a coverage failsafe to make sure that totally empty calls, +# i.e. request == None and no flattened fields passed, work. +@pytest.mark.asyncio +async def test_flush_rows_empty_call_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio", + ) + + # Mock the actual call, and fake the request. + with mock.patch.object( + type(client.transport.flush_rows), + '__call__') as call: + # Designate an appropriate return value for the call. + call.return_value = grpc_helpers_async.FakeUnaryUnaryCall(storage.FlushRowsResponse( + offset=647, + )) + await client.flush_rows(request=None) + + # Establish that the underlying stub method was called. + call.assert_called() + _, args, _ = call.mock_calls[0] + request_msg = storage.FlushRowsRequest() + + assert args[0] == request_msg + + +def test_transport_grpc_default(): + # A client should use the gRPC transport by default. + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + ) + assert isinstance( + client.transport, + transports.BigQueryWriteGrpcTransport, + ) + +def test_big_query_write_base_transport_error(): + # Passing both a credentials object and credentials_file should raise an error + with pytest.raises(core_exceptions.DuplicateCredentialArgs): + transport = transports.BigQueryWriteTransport( + credentials=ga_credentials.AnonymousCredentials(), + credentials_file="credentials.json" + ) + + +def test_big_query_write_base_transport(): + # Instantiate the base transport. + with mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteTransport.__init__') as Transport: + Transport.return_value = None + transport = transports.BigQueryWriteTransport( + credentials=ga_credentials.AnonymousCredentials(), + ) + + # Every method on the transport should just blindly + # raise NotImplementedError. + methods = ( + 'create_write_stream', + 'append_rows', + 'get_write_stream', + 'finalize_write_stream', + 'batch_commit_write_streams', + 'flush_rows', + ) + for method in methods: + with pytest.raises(NotImplementedError): + getattr(transport, method)(request=object()) + + with pytest.raises(NotImplementedError): + transport.close() + + # Catch all for all remaining methods and properties + remainder = [ + 'kind', + ] + for r in remainder: + with pytest.raises(NotImplementedError): + getattr(transport, r)() + + +def test_big_query_write_base_transport_with_credentials_file(): + # Instantiate the base transport with a credentials file + with mock.patch.object(google.auth, 'load_credentials_from_file', autospec=True) as load_creds, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + load_creds.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryWriteTransport( + credentials_file="credentials.json", + quota_project_id="octopus", + ) + load_creds.assert_called_once_with("credentials.json", + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id="octopus", + ) + + +def test_big_query_write_base_transport_with_adc(): + # Test the default credentials are used if credentials and credentials_file are None. + with mock.patch.object(google.auth, 'default', autospec=True) as adc, mock.patch('google.cloud.bigquery_storage_v1beta2.services.big_query_write.transports.BigQueryWriteTransport._prep_wrapped_messages') as Transport: + Transport.return_value = None + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport = transports.BigQueryWriteTransport() + adc.assert_called_once() + + +def test_big_query_write_auth_adc(): + # If no credentials are provided, we should use ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + BigQueryWriteClient() + adc.assert_called_once_with( + scopes=None, + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + quota_project_id=None, + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +def test_big_query_write_transport_auth_adc(transport_class): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + adc.return_value = (ga_credentials.AnonymousCredentials(), None) + transport_class(quota_project_id="octopus", scopes=["1", "2"]) + adc.assert_called_once_with( + scopes=["1", "2"], + default_scopes=( 'https://www.googleapis.com/auth/bigquery', 'https://www.googleapis.com/auth/bigquery.insertdata', 'https://www.googleapis.com/auth/cloud-platform',), + quota_project_id="octopus", + ) + + +@pytest.mark.parametrize( + "transport_class", + [ + transports.BigQueryWriteGrpcTransport, + transports.BigQueryWriteGrpcAsyncIOTransport, + ], +) +def test_big_query_write_transport_auth_gdch_credentials(transport_class): + host = 'https://language.com' + api_audience_tests = [None, 'https://language2.com'] + api_audience_expect = [host, 'https://language2.com'] + for t, e in zip(api_audience_tests, api_audience_expect): + with mock.patch.object(google.auth, 'default', autospec=True) as adc: + gdch_mock = mock.MagicMock() + type(gdch_mock).with_gdch_audience = mock.PropertyMock(return_value=gdch_mock) + adc.return_value = (gdch_mock, None) + transport_class(host=host, api_audience=t) + gdch_mock.with_gdch_audience.assert_called_once_with( + e + ) + + +@pytest.mark.parametrize( + "transport_class,grpc_helpers", + [ + (transports.BigQueryWriteGrpcTransport, grpc_helpers), + (transports.BigQueryWriteGrpcAsyncIOTransport, grpc_helpers_async) + ], +) +def test_big_query_write_transport_create_channel(transport_class, grpc_helpers): + # If credentials and host are not provided, the transport class should use + # ADC credentials. + with mock.patch.object(google.auth, "default", autospec=True) as adc, mock.patch.object( + grpc_helpers, "create_channel", autospec=True + ) as create_channel: + creds = ga_credentials.AnonymousCredentials() + adc.return_value = (creds, None) + transport_class( + quota_project_id="octopus", + scopes=["1", "2"] + ) + + create_channel.assert_called_with( + "bigquerystorage.googleapis.com:443", + credentials=creds, + credentials_file=None, + quota_project_id="octopus", + default_scopes=( + 'https://www.googleapis.com/auth/bigquery', + 'https://www.googleapis.com/auth/bigquery.insertdata', + 'https://www.googleapis.com/auth/cloud-platform', +), + scopes=["1", "2"], + default_host="bigquerystorage.googleapis.com", + ssl_credentials=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + +@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) +def test_big_query_write_grpc_transport_client_cert_source_for_mtls( + transport_class +): + cred = ga_credentials.AnonymousCredentials() + + # Check ssl_channel_credentials is used if provided. + with mock.patch.object(transport_class, "create_channel") as mock_create_channel: + mock_ssl_channel_creds = mock.Mock() + transport_class( + host="squid.clam.whelk", + credentials=cred, + ssl_channel_credentials=mock_ssl_channel_creds + ) + mock_create_channel.assert_called_once_with( + "squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_channel_creds, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + + # Check if ssl_channel_credentials is not provided, then client_cert_source_for_mtls + # is used. + with mock.patch.object(transport_class, "create_channel", return_value=mock.Mock()): + with mock.patch("grpc.ssl_channel_credentials") as mock_ssl_cred: + transport_class( + credentials=cred, + client_cert_source_for_mtls=client_cert_source_callback + ) + expected_cert, expected_key = client_cert_source_callback() + mock_ssl_cred.assert_called_once_with( + certificate_chain=expected_cert, + private_key=expected_key + ) + + +@pytest.mark.parametrize("transport_name", [ + "grpc", + "grpc_asyncio", +]) +def test_big_query_write_host_no_port(transport_name): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com'), + transport=transport_name, + ) + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:443' + ) + +@pytest.mark.parametrize("transport_name", [ + "grpc", + "grpc_asyncio", +]) +def test_big_query_write_host_with_port(transport_name): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + client_options=client_options.ClientOptions(api_endpoint='bigquerystorage.googleapis.com:8000'), + transport=transport_name, + ) + assert client.transport._host == ( + 'bigquerystorage.googleapis.com:8000' + ) + +def test_big_query_write_grpc_transport_channel(): + channel = grpc.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryWriteGrpcTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +def test_big_query_write_grpc_asyncio_transport_channel(): + channel = aio.secure_channel('http://localhost/', grpc.local_channel_credentials()) + + # Check that channel is used if provided. + transport = transports.BigQueryWriteGrpcAsyncIOTransport( + host="squid.clam.whelk", + channel=channel, + ) + assert transport.grpc_channel == channel + assert transport._host == "squid.clam.whelk:443" + assert transport._ssl_channel_credentials == None + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) +def test_big_query_write_transport_channel_mtls_with_client_cert_source( + transport_class +): + with mock.patch("grpc.ssl_channel_credentials", autospec=True) as grpc_ssl_channel_cred: + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_ssl_cred = mock.Mock() + grpc_ssl_channel_cred.return_value = mock_ssl_cred + + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + + cred = ga_credentials.AnonymousCredentials() + with pytest.warns(DeprecationWarning): + with mock.patch.object(google.auth, 'default') as adc: + adc.return_value = (cred, None) + transport = transport_class( + host="squid.clam.whelk", + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=client_cert_source_callback, + ) + adc.assert_called_once() + + grpc_ssl_channel_cred.assert_called_once_with( + certificate_chain=b"cert bytes", private_key=b"key bytes" + ) + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + assert transport._ssl_channel_credentials == mock_ssl_cred + + +# Remove this test when deprecated arguments (api_mtls_endpoint, client_cert_source) are +# removed from grpc/grpc_asyncio transport constructor. +@pytest.mark.parametrize("transport_class", [transports.BigQueryWriteGrpcTransport, transports.BigQueryWriteGrpcAsyncIOTransport]) +def test_big_query_write_transport_channel_mtls_with_adc( + transport_class +): + mock_ssl_cred = mock.Mock() + with mock.patch.multiple( + "google.auth.transport.grpc.SslCredentials", + __init__=mock.Mock(return_value=None), + ssl_credentials=mock.PropertyMock(return_value=mock_ssl_cred), + ): + with mock.patch.object(transport_class, "create_channel") as grpc_create_channel: + mock_grpc_channel = mock.Mock() + grpc_create_channel.return_value = mock_grpc_channel + mock_cred = mock.Mock() + + with pytest.warns(DeprecationWarning): + transport = transport_class( + host="squid.clam.whelk", + credentials=mock_cred, + api_mtls_endpoint="mtls.squid.clam.whelk", + client_cert_source=None, + ) + + grpc_create_channel.assert_called_once_with( + "mtls.squid.clam.whelk:443", + credentials=mock_cred, + credentials_file=None, + scopes=None, + ssl_credentials=mock_ssl_cred, + quota_project_id=None, + options=[ + ("grpc.max_send_message_length", -1), + ("grpc.max_receive_message_length", -1), + ], + ) + assert transport.grpc_channel == mock_grpc_channel + + +def test_table_path(): + project = "squid" + dataset = "clam" + table = "whelk" + expected = "projects/{project}/datasets/{dataset}/tables/{table}".format(project=project, dataset=dataset, table=table, ) + actual = BigQueryWriteClient.table_path(project, dataset, table) + assert expected == actual + + +def test_parse_table_path(): + expected = { + "project": "octopus", + "dataset": "oyster", + "table": "nudibranch", + } + path = BigQueryWriteClient.table_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_table_path(path) + assert expected == actual + +def test_write_stream_path(): + project = "cuttlefish" + dataset = "mussel" + table = "winkle" + stream = "nautilus" + expected = "projects/{project}/datasets/{dataset}/tables/{table}/streams/{stream}".format(project=project, dataset=dataset, table=table, stream=stream, ) + actual = BigQueryWriteClient.write_stream_path(project, dataset, table, stream) + assert expected == actual + + +def test_parse_write_stream_path(): + expected = { + "project": "scallop", + "dataset": "abalone", + "table": "squid", + "stream": "clam", + } + path = BigQueryWriteClient.write_stream_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_write_stream_path(path) + assert expected == actual + +def test_common_billing_account_path(): + billing_account = "whelk" + expected = "billingAccounts/{billing_account}".format(billing_account=billing_account, ) + actual = BigQueryWriteClient.common_billing_account_path(billing_account) + assert expected == actual + + +def test_parse_common_billing_account_path(): + expected = { + "billing_account": "octopus", + } + path = BigQueryWriteClient.common_billing_account_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_billing_account_path(path) + assert expected == actual + +def test_common_folder_path(): + folder = "oyster" + expected = "folders/{folder}".format(folder=folder, ) + actual = BigQueryWriteClient.common_folder_path(folder) + assert expected == actual + + +def test_parse_common_folder_path(): + expected = { + "folder": "nudibranch", + } + path = BigQueryWriteClient.common_folder_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_folder_path(path) + assert expected == actual + +def test_common_organization_path(): + organization = "cuttlefish" + expected = "organizations/{organization}".format(organization=organization, ) + actual = BigQueryWriteClient.common_organization_path(organization) + assert expected == actual + + +def test_parse_common_organization_path(): + expected = { + "organization": "mussel", + } + path = BigQueryWriteClient.common_organization_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_organization_path(path) + assert expected == actual + +def test_common_project_path(): + project = "winkle" + expected = "projects/{project}".format(project=project, ) + actual = BigQueryWriteClient.common_project_path(project) + assert expected == actual + + +def test_parse_common_project_path(): + expected = { + "project": "nautilus", + } + path = BigQueryWriteClient.common_project_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_project_path(path) + assert expected == actual + +def test_common_location_path(): + project = "scallop" + location = "abalone" + expected = "projects/{project}/locations/{location}".format(project=project, location=location, ) + actual = BigQueryWriteClient.common_location_path(project, location) + assert expected == actual + + +def test_parse_common_location_path(): + expected = { + "project": "squid", + "location": "clam", + } + path = BigQueryWriteClient.common_location_path(**expected) + + # Check that the path construction is reversible. + actual = BigQueryWriteClient.parse_common_location_path(path) + assert expected == actual + + +def test_client_with_default_client_info(): + client_info = gapic_v1.client_info.ClientInfo() + + with mock.patch.object(transports.BigQueryWriteTransport, '_prep_wrapped_messages') as prep: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + with mock.patch.object(transports.BigQueryWriteTransport, '_prep_wrapped_messages') as prep: + transport_class = BigQueryWriteClient.get_transport_class() + transport = transport_class( + credentials=ga_credentials.AnonymousCredentials(), + client_info=client_info, + ) + prep.assert_called_once_with(client_info) + + +def test_transport_close_grpc(): + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport="grpc" + ) + with mock.patch.object(type(getattr(client.transport, "_grpc_channel")), "close") as close: + with client: + close.assert_not_called() + close.assert_called_once() + + +@pytest.mark.asyncio +async def test_transport_close_grpc_asyncio(): + client = BigQueryWriteAsyncClient( + credentials=async_anonymous_credentials(), + transport="grpc_asyncio" + ) + with mock.patch.object(type(getattr(client.transport, "_grpc_channel")), "close") as close: + async with client: + close.assert_not_called() + close.assert_called_once() + + +def test_client_ctx(): + transports = [ + 'grpc', + ] + for transport in transports: + client = BigQueryWriteClient( + credentials=ga_credentials.AnonymousCredentials(), + transport=transport + ) + # Test client calls underlying transport. + with mock.patch.object(type(client.transport), "close") as close: + close.assert_not_called() + with client: + pass + close.assert_called() + +@pytest.mark.parametrize("client_class,transport_class", [ + (BigQueryWriteClient, transports.BigQueryWriteGrpcTransport), + (BigQueryWriteAsyncClient, transports.BigQueryWriteGrpcAsyncIOTransport), +]) +def test_api_key_credentials(client_class, transport_class): + with mock.patch.object( + google.auth._default, "get_api_key_credentials", create=True + ) as get_api_key_credentials: + mock_cred = mock.Mock() + get_api_key_credentials.return_value = mock_cred + options = client_options.ClientOptions() + options.api_key = "api_key" + with mock.patch.object(transport_class, "__init__") as patched: + patched.return_value = None + client = client_class(client_options=options) + patched.assert_called_once_with( + credentials=mock_cred, + credentials_file=None, + host=client._DEFAULT_ENDPOINT_TEMPLATE.format(UNIVERSE_DOMAIN=client._DEFAULT_UNIVERSE), + scopes=None, + client_cert_source_for_mtls=None, + quota_project_id=None, + client_info=transports.base.DEFAULT_CLIENT_INFO, + always_use_jwt_access=True, + api_audience=None, + )