From 750c15d8c7449776955237d01c24be7b4b66d0b5 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 16 Dec 2021 20:22:35 +0100 Subject: [PATCH 01/14] chore(deps): update all dependencies (#1089) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [![WhiteSource Renovate](https://app.renovatebot.com/images/banner.svg)](https://renovatebot.com) This PR contains the following updates: | Package | Change | Age | Adoption | Passing | Confidence | |---|---|---|---|---|---| | [google-api-core](https://togithub.com/googleapis/python-api-core) | `==2.3.0` -> `==2.3.2` | [![age](https://badges.renovateapi.com/packages/pypi/google-api-core/2.3.2/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/google-api-core/2.3.2/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/google-api-core/2.3.2/compatibility-slim/2.3.0)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/google-api-core/2.3.2/confidence-slim/2.3.0)](https://docs.renovatebot.com/merge-confidence/) | | [importlib-metadata](https://togithub.com/python/importlib_metadata) | `==4.8.2` -> `==4.8.3` | [![age](https://badges.renovateapi.com/packages/pypi/importlib-metadata/4.8.3/age-slim)](https://docs.renovatebot.com/merge-confidence/) | [![adoption](https://badges.renovateapi.com/packages/pypi/importlib-metadata/4.8.3/adoption-slim)](https://docs.renovatebot.com/merge-confidence/) | [![passing](https://badges.renovateapi.com/packages/pypi/importlib-metadata/4.8.3/compatibility-slim/4.8.2)](https://docs.renovatebot.com/merge-confidence/) | [![confidence](https://badges.renovateapi.com/packages/pypi/importlib-metadata/4.8.3/confidence-slim/4.8.2)](https://docs.renovatebot.com/merge-confidence/) | --- ### Release Notes
googleapis/python-api-core ### [`v2.3.2`](https://togithub.com/googleapis/python-api-core/releases/v2.3.2) [Compare Source](https://togithub.com/googleapis/python-api-core/compare/v2.3.1...v2.3.2) ##### Bug Fixes - fix: exclude function target from retry deadline exceeded exception message ([#​318](https://www.togithub.com/googleapis/python-api-core/issues/318)) ([34ebdcc](https://togithub.com/googleapis/python-api-core/commit/34ebdcc251d4f3d7d496e8e0b78847645a06650b)) ### [`v2.3.1`](https://togithub.com/googleapis/python-api-core/compare/v2.3.0...v2.3.1) [Compare Source](https://togithub.com/googleapis/python-api-core/compare/v2.3.0...v2.3.1)
python/importlib_metadata ### [`v4.8.3`](https://togithub.com/python/importlib_metadata/blob/HEAD/CHANGES.rst#v483) [Compare Source](https://togithub.com/python/importlib_metadata/compare/v4.8.2...v4.8.3) \====== - [#​357](https://togithub.com/python/importlib_metadata/issues/357): Fixed requirement generation from egg-info when a URL requirement is given.
--- ### Configuration 📅 **Schedule**: At any time (no schedule defined). 🚦 **Automerge**: Disabled by config. Please merge this manually once you are satisfied. ♻ **Rebasing**: Whenever PR becomes conflicted, or you tick the rebase/retry checkbox. 👻 **Immortal**: This PR will be recreated if closed unmerged. Get [config help](https://togithub.com/renovatebot/renovate/discussions) if that's undesired. --- - [ ] If you want to rebase/retry this PR, click this checkbox. --- This PR has been generated by [WhiteSource Renovate](https://renovate.whitesourcesoftware.com). View repository job log [here](https://app.renovatebot.com/dashboard#github/googleapis/python-bigquery). --- samples/geography/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index a10f01e72..4f7956f4b 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -11,7 +11,7 @@ Fiona==1.8.20 geojson==2.5.0 geopandas==0.9.0; python_version < '3.7' geopandas==0.10.2; python_version >= '3.7' -google-api-core==2.3.0 +google-api-core==2.3.2 google-auth==2.3.3 google-cloud-bigquery==2.31.0 google-cloud-bigquery-storage==2.10.1 @@ -21,7 +21,7 @@ google-resumable-media==2.1.0 googleapis-common-protos==1.54.0 grpcio==1.42.0 idna==3.3 -importlib-metadata==4.8.2 +importlib-metadata==4.8.3 libcst==0.3.23 munch==2.5.0 mypy-extensions==0.4.3 From 881c081feb248e6a2b0d7ad72667924b959f8120 Mon Sep 17 00:00:00 2001 From: Anthonios Partheniou Date: Tue, 28 Dec 2021 13:33:13 -0500 Subject: [PATCH 02/14] chore: update .repo-metadata.json (#1093) --- .repo-metadata.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.repo-metadata.json b/.repo-metadata.json index 124b40eb9..670aba793 100644 --- a/.repo-metadata.json +++ b/.repo-metadata.json @@ -2,9 +2,9 @@ "name": "bigquery", "name_pretty": "Google Cloud BigQuery", "product_documentation": "https://cloud.google.com/bigquery", - "client_documentation": "https://googleapis.dev/python/bigquery/latest", + "client_documentation": "https://cloud.google.com/python/docs/reference/bigquery/latest", "issue_tracker": "https://issuetracker.google.com/savedsearches/559654", - "release_level": "ga", + "release_level": "stable", "language": "python", "library_type": "GAPIC_COMBO", "repo": "googleapis/python-bigquery", @@ -12,5 +12,6 @@ "api_id": "bigquery.googleapis.com", "requires_billing": false, "default_version": "v2", - "codeowner_team": "@googleapis/api-bigquery" + "codeowner_team": "@googleapis/api-bigquery", + "api_shortname": "bigquery" } From 9e0ce1f4092e04b681fae4fe8ae4170f3df00e4e Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Tue, 4 Jan 2022 15:57:07 -0600 Subject: [PATCH 03/14] test: avoid flake if connections were already open at start (#1096) Addresses flakes such as https://source.cloud.google.com/results/invocations/bb7ce2a8-d410-45bd-af12-7c04bb8cbb25/targets/cloud-devrel%2Fclient-libraries%2Fpython%2Fgoogleapis%2Fpython-bigquery%2Fpresubmit%2Fsystem-3.10/log --- tests/system/test_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/system/test_client.py b/tests/system/test_client.py index 43b3b6ae7..077bb2886 100644 --- a/tests/system/test_client.py +++ b/tests/system/test_client.py @@ -1725,7 +1725,7 @@ def test_dbapi_connection_does_not_leak_sockets(self): connection.close() conn_count_end = len(current_process.connections()) - self.assertEqual(conn_count_end, conn_count_start) + self.assertLessEqual(conn_count_end, conn_count_start) def _load_table_for_dml(self, rows, dataset_id, table_id): from google.cloud._testing import _NamedTemporaryFile From dcb8728c12f5ab0d7809a1b6cf72755dff973772 Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Wed, 5 Jan 2022 20:44:10 +0100 Subject: [PATCH 04/14] chore(deps): update all dependencies (#1091) * chore(deps): update all dependencies * don't directly depend on cachetools * remove importlib metadata as direct dependency * remove zipp which comes from transitive dependency Co-authored-by: Tim Swast --- samples/geography/requirements.txt | 9 +++------ samples/magics/requirements.txt | 2 +- samples/snippets/requirements.txt | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 4f7956f4b..543076cdc 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,5 +1,4 @@ -attrs==21.2.0 -cachetools==4.2.4 +attrs==21.4.0 certifi==2021.10.8 cffi==1.15.0 charset-normalizer==2.0.9 @@ -19,9 +18,8 @@ google-cloud-core==2.2.1 google-crc32c==1.3.0 google-resumable-media==2.1.0 googleapis-common-protos==1.54.0 -grpcio==1.42.0 +grpcio==1.43.0 idna==3.3 -importlib-metadata==4.8.3 libcst==0.3.23 munch==2.5.0 mypy-extensions==0.4.3 @@ -40,11 +38,10 @@ pyproj==3.1.0; python_version > "3.6" python-dateutil==2.8.2 pytz==2021.3 PyYAML==6.0 -requests==2.26.0 +requests==2.27.0 rsa==4.8 Shapely==1.8.0 six==1.16.0 typing-extensions==4.0.1 typing-inspect==0.7.1 urllib3==1.26.7 -zipp==3.6.0 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index 7fc6561b0..f4337e8fb 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -1,6 +1,6 @@ google-cloud-bigquery-storage==2.10.1 google-auth-oauthlib==0.4.6 -grpcio==1.42.0 +grpcio==1.43.0 ipython==7.16.1; python_version < '3.7' ipython==7.29.0; python_version >= '3.7' matplotlib==3.3.4; python_version < '3.7' diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index 0ee32e1fd..bef333720 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -1,6 +1,6 @@ google-cloud-bigquery-storage==2.10.1 google-auth-oauthlib==0.4.6 -grpcio==1.42.0 +grpcio==1.43.0 ipython==7.16.1; python_version < '3.7' ipython==7.29.0; python_version >= '3.7' matplotlib==3.3.4; python_version < '3.7' From 82e0a7c231e823b025f7fa5b81c4695ced0f73ba Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Thu, 6 Jan 2022 20:21:39 +0100 Subject: [PATCH 05/14] chore(deps): update all dependencies (#1098) --- samples/geography/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index 543076cdc..ca7e38f84 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -1,7 +1,7 @@ attrs==21.4.0 certifi==2021.10.8 cffi==1.15.0 -charset-normalizer==2.0.9 +charset-normalizer==2.0.10 click==8.0.3 click-plugins==1.1.1 cligj==0.7.2 @@ -38,7 +38,7 @@ pyproj==3.1.0; python_version > "3.6" python-dateutil==2.8.2 pytz==2021.3 PyYAML==6.0 -requests==2.27.0 +requests==2.27.1 rsa==4.8 Shapely==1.8.0 six==1.16.0 From ef1f2cbdf4e72eb8ab338ce2434ba82f8acd8515 Mon Sep 17 00:00:00 2001 From: Hugo van Kemenade Date: Mon, 10 Jan 2022 11:20:10 +0200 Subject: [PATCH 06/14] README: Update supported Python versions (#1100) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thank you for opening a Pull Request! Before submitting your PR, there are a few things you can do to make sure it goes smoothly: - [ ] Make sure to open an issue as a [bug/issue](https://github.com/googleapis/python-bigquery/issues/new/choose) before writing your code! That way we can discuss the change, evaluate designs, and agree on the general idea - [ ] Ensure the tests and linter pass - [ ] Code coverage does not decrease (if any source code was changed) - [x] Appropriate docs were updated (if necessary) Fixes # 🦕 To match the Trove classifiers on PyPI: ![image](https://user-images.githubusercontent.com/1324225/148646991-cc536f1f-4389-4cd9-9995-276c9aa7245a.png) https://pypi.org/project/google-cloud-bigquery/ --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index d0ad059a2..bafa06693 100644 --- a/README.rst +++ b/README.rst @@ -52,7 +52,7 @@ dependencies. Supported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^ -Python >= 3.6, < 3.10 +Python >= 3.6, < 3.11 Unsupported Python Versions ^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 18ee0b7478a77f24c575882a30eb872d24b2eaa1 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 11 Jan 2022 10:19:40 -0500 Subject: [PATCH 07/14] chore(samples): Add check for tests in directory (#1103) Source-Link: https://github.com/googleapis/synthtool/commit/52aef91f8d25223d9dbdb4aebd94ba8eea2101f3 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:36a95b8f494e4674dc9eee9af98961293b51b86b3649942aac800ae6c1f796d4 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- samples/geography/noxfile.py | 70 ++++++++++++++++++++---------------- samples/magics/noxfile.py | 70 ++++++++++++++++++++---------------- samples/snippets/noxfile.py | 70 ++++++++++++++++++++---------------- 4 files changed, 118 insertions(+), 94 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 0b3c8cd98..6b8a73b31 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:2f90537dd7df70f6b663cd654b1fa5dee483cf6a4edcfd46072b2775be8a23ec + digest: sha256:36a95b8f494e4674dc9eee9af98961293b51b86b3649942aac800ae6c1f796d4 diff --git a/samples/geography/noxfile.py b/samples/geography/noxfile.py index 93a9122cc..3bbef5d54 100644 --- a/samples/geography/noxfile.py +++ b/samples/geography/noxfile.py @@ -14,6 +14,7 @@ from __future__ import print_function +import glob import os from pathlib import Path import sys @@ -184,37 +185,44 @@ def blacken(session: nox.sessions.Session) -> None: def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) + # check for presence of tests + test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + if len(test_list) == 0: + print("No tests found, skipping directory.") + else: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) @nox.session(python=ALL_VERSIONS) diff --git a/samples/magics/noxfile.py b/samples/magics/noxfile.py index 93a9122cc..3bbef5d54 100644 --- a/samples/magics/noxfile.py +++ b/samples/magics/noxfile.py @@ -14,6 +14,7 @@ from __future__ import print_function +import glob import os from pathlib import Path import sys @@ -184,37 +185,44 @@ def blacken(session: nox.sessions.Session) -> None: def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) + # check for presence of tests + test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + if len(test_list) == 0: + print("No tests found, skipping directory.") + else: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) @nox.session(python=ALL_VERSIONS) diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 93a9122cc..3bbef5d54 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -14,6 +14,7 @@ from __future__ import print_function +import glob import os from pathlib import Path import sys @@ -184,37 +185,44 @@ def blacken(session: nox.sessions.Session) -> None: def _session_tests( session: nox.sessions.Session, post_install: Callable = None ) -> None: - if TEST_CONFIG["pip_version_override"]: - pip_version = TEST_CONFIG["pip_version_override"] - session.install(f"pip=={pip_version}") - """Runs py.test for a particular project.""" - if os.path.exists("requirements.txt"): - if os.path.exists("constraints.txt"): - session.install("-r", "requirements.txt", "-c", "constraints.txt") - else: - session.install("-r", "requirements.txt") - - if os.path.exists("requirements-test.txt"): - if os.path.exists("constraints-test.txt"): - session.install("-r", "requirements-test.txt", "-c", "constraints-test.txt") - else: - session.install("-r", "requirements-test.txt") - - if INSTALL_LIBRARY_FROM_SOURCE: - session.install("-e", _get_repo_root()) - - if post_install: - post_install(session) - - session.run( - "pytest", - *(PYTEST_COMMON_ARGS + session.posargs), - # Pytest will return 5 when no tests are collected. This can happen - # on travis where slow and flaky tests are excluded. - # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html - success_codes=[0, 5], - env=get_pytest_env_vars(), - ) + # check for presence of tests + test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + if len(test_list) == 0: + print("No tests found, skipping directory.") + else: + if TEST_CONFIG["pip_version_override"]: + pip_version = TEST_CONFIG["pip_version_override"] + session.install(f"pip=={pip_version}") + """Runs py.test for a particular project.""" + if os.path.exists("requirements.txt"): + if os.path.exists("constraints.txt"): + session.install("-r", "requirements.txt", "-c", "constraints.txt") + else: + session.install("-r", "requirements.txt") + + if os.path.exists("requirements-test.txt"): + if os.path.exists("constraints-test.txt"): + session.install( + "-r", "requirements-test.txt", "-c", "constraints-test.txt" + ) + else: + session.install("-r", "requirements-test.txt") + + if INSTALL_LIBRARY_FROM_SOURCE: + session.install("-e", _get_repo_root()) + + if post_install: + post_install(session) + + session.run( + "pytest", + *(PYTEST_COMMON_ARGS + session.posargs), + # Pytest will return 5 when no tests are collected. This can happen + # on travis where slow and flaky tests are excluded. + # See http://doc.pytest.org/en/latest/_modules/_pytest/main.html + success_codes=[0, 5], + env=get_pytest_env_vars(), + ) @nox.session(python=ALL_VERSIONS) From e23114ce362e09ac72f733a640e53a561cc9ce69 Mon Sep 17 00:00:00 2001 From: Tim Swast Date: Wed, 12 Jan 2022 15:25:16 -0600 Subject: [PATCH 08/14] fix: remove query text from exception message, use `exception.debug_message` instead (#1105) Since query text can potentially contain sensitive information, remove it from the default exception message. This information is useful for debugging, so provide a `debug_message` attribute, which is not included in the exception representation (and thus the logs). Fixes internal issue 211616590 --- google/cloud/bigquery/job/query.py | 19 +++++++++++++------ tests/unit/job/test_query.py | 20 ++++++++++++++++---- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/google/cloud/bigquery/job/query.py b/google/cloud/bigquery/job/query.py index 36e388238..2dd945984 100644 --- a/google/cloud/bigquery/job/query.py +++ b/google/cloud/bigquery/job/query.py @@ -66,6 +66,7 @@ _CONTAINS_ORDER_BY = re.compile(r"ORDER\s+BY", re.IGNORECASE) +_EXCEPTION_FOOTER_TEMPLATE = "{message}\n\nLocation: {location}\nJob ID: {job_id}\n" _TIMEOUT_BUFFER_SECS = 0.1 @@ -1196,17 +1197,17 @@ def _blocking_poll(self, timeout=None, **kwargs): super(QueryJob, self)._blocking_poll(timeout=timeout, **kwargs) @staticmethod - def _format_for_exception(query, job_id): + def _format_for_exception(message: str, query: str): """Format a query for the output in exception message. Args: + message (str): The original exception message. query (str): The SQL query to format. - job_id (str): The ID of the job that ran the query. Returns: str: A formatted query text. """ - template = "\n\n(job ID: {job_id})\n\n{header}\n\n{ruler}\n{body}\n{ruler}" + template = "{message}\n\n{header}\n\n{ruler}\n{body}\n{ruler}" lines = query.splitlines() max_line_len = max(len(line) for line in lines) @@ -1223,7 +1224,7 @@ def _format_for_exception(query, job_id): "{:4}:{}".format(n, line) for n, line in enumerate(lines, start=1) ) - return template.format(job_id=job_id, header=header, ruler=ruler, body=body) + return template.format(message=message, header=header, ruler=ruler, body=body) def _begin(self, client=None, retry=DEFAULT_RETRY, timeout=None): """API call: begin the job via a POST request @@ -1248,7 +1249,10 @@ def _begin(self, client=None, retry=DEFAULT_RETRY, timeout=None): try: super(QueryJob, self)._begin(client=client, retry=retry, timeout=timeout) except exceptions.GoogleAPICallError as exc: - exc.message += self._format_for_exception(self.query, self.job_id) + exc.message = _EXCEPTION_FOOTER_TEMPLATE.format( + message=exc.message, location=self.location, job_id=self.job_id + ) + exc.debug_message = self._format_for_exception(exc.message, self.query) exc.query_job = self raise @@ -1447,7 +1451,10 @@ def do_get_result(): do_get_result() except exceptions.GoogleAPICallError as exc: - exc.message += self._format_for_exception(self.query, self.job_id) + exc.message = _EXCEPTION_FOOTER_TEMPLATE.format( + message=exc.message, location=self.location, job_id=self.job_id + ) + exc.debug_message = self._format_for_exception(exc.message, self.query) # type: ignore exc.query_job = self # type: ignore raise except requests.exceptions.Timeout as exc: diff --git a/tests/unit/job/test_query.py b/tests/unit/job/test_query.py index 4da035b78..5fb76b9e9 100644 --- a/tests/unit/job/test_query.py +++ b/tests/unit/job/test_query.py @@ -1360,13 +1360,19 @@ def test_result_error(self): exc_job_instance = getattr(exc_info.exception, "query_job", None) self.assertIs(exc_job_instance, job) + # Query text could contain sensitive information, so it must not be + # included in logs / exception representation. full_text = str(exc_info.exception) assert job.job_id in full_text - assert "Query Job SQL Follows" in full_text + assert "Query Job SQL Follows" not in full_text + # It is useful to have query text available, so it is provided in a + # debug_message property. + debug_message = exc_info.exception.debug_message + assert "Query Job SQL Follows" in debug_message for i, line in enumerate(query.splitlines(), start=1): expected_line = "{}:{}".format(i, line) - assert expected_line in full_text + assert expected_line in debug_message def test_result_transport_timeout_error(self): query = textwrap.dedent( @@ -1452,13 +1458,19 @@ def test__begin_error(self): exc_job_instance = getattr(exc_info.exception, "query_job", None) self.assertIs(exc_job_instance, job) + # Query text could contain sensitive information, so it must not be + # included in logs / exception representation. full_text = str(exc_info.exception) assert job.job_id in full_text - assert "Query Job SQL Follows" in full_text + assert "Query Job SQL Follows" not in full_text + # It is useful to have query text available, so it is provided in a + # debug_message property. + debug_message = exc_info.exception.debug_message + assert "Query Job SQL Follows" in debug_message for i, line in enumerate(query.splitlines(), start=1): expected_line = "{}:{}".format(i, line) - assert expected_line in full_text + assert expected_line in debug_message def test__begin_w_timeout(self): PATH = "/projects/%s/jobs" % (self.PROJECT,) From 073c9d383eef4f697fbacdf31daf66ccb33d365b Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Wed, 12 Jan 2022 23:08:21 +0000 Subject: [PATCH 09/14] build: switch to release-please for tagging (#1107) --- .github/.OwlBot.lock.yaml | 2 +- .github/release-please.yml | 1 + .github/release-trigger.yml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 .github/release-trigger.yml diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 6b8a73b31..ff5126c18 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:36a95b8f494e4674dc9eee9af98961293b51b86b3649942aac800ae6c1f796d4 + digest: sha256:dfa9b663b32de8b5b327e32c1da665a80de48876558dd58091d8160c60ad7355 diff --git a/.github/release-please.yml b/.github/release-please.yml index 4507ad059..466597e5b 100644 --- a/.github/release-please.yml +++ b/.github/release-please.yml @@ -1 +1,2 @@ releaseType: python +handleGHRelease: true diff --git a/.github/release-trigger.yml b/.github/release-trigger.yml new file mode 100644 index 000000000..d4ca94189 --- /dev/null +++ b/.github/release-trigger.yml @@ -0,0 +1 @@ +enabled: true From 643b333d9da207021a5726b4ad01a1b4f91f530f Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Thu, 13 Jan 2022 16:52:17 +0000 Subject: [PATCH 10/14] chore(main): release 2.32.0 (#1104) :robot: I have created a release *beep* *boop* --- ## [2.32.0](https://github.com/googleapis/python-bigquery/compare/v2.31.0...v2.32.0) (2022-01-12) ### Features * support authorized dataset entity ([#1075](https://github.com/googleapis/python-bigquery/issues/1075)) ([c098cd0](https://github.com/googleapis/python-bigquery/commit/c098cd01c755633bfaba7193dd5c044a489a5b61)) ### Bug Fixes * remove query text from exception message, use `exception.debug_message` instead ([#1105](https://github.com/googleapis/python-bigquery/issues/1105)) ([e23114c](https://github.com/googleapis/python-bigquery/commit/e23114ce362e09ac72f733a640e53a561cc9ce69)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- CHANGELOG.md | 12 ++++++++++++ google/cloud/bigquery/version.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ba219d20..6e69fa621 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ [1]: https://pypi.org/project/google-cloud-bigquery/#history +## [2.32.0](https://github.com/googleapis/python-bigquery/compare/v2.31.0...v2.32.0) (2022-01-12) + + +### Features + +* support authorized dataset entity ([#1075](https://github.com/googleapis/python-bigquery/issues/1075)) ([c098cd0](https://github.com/googleapis/python-bigquery/commit/c098cd01c755633bfaba7193dd5c044a489a5b61)) + + +### Bug Fixes + +* remove query text from exception message, use `exception.debug_message` instead ([#1105](https://github.com/googleapis/python-bigquery/issues/1105)) ([e23114c](https://github.com/googleapis/python-bigquery/commit/e23114ce362e09ac72f733a640e53a561cc9ce69)) + ## [2.31.0](https://www.github.com/googleapis/python-bigquery/compare/v2.30.1...v2.31.0) (2021-11-24) diff --git a/google/cloud/bigquery/version.py b/google/cloud/bigquery/version.py index 6329658af..b8c5af9a2 100644 --- a/google/cloud/bigquery/version.py +++ b/google/cloud/bigquery/version.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "2.31.0" +__version__ = "2.32.0" From 144cbf4ec8f0ba0199901d2128958a68b3b6f7fc Mon Sep 17 00:00:00 2001 From: WhiteSource Renovate Date: Fri, 14 Jan 2022 21:43:58 +0100 Subject: [PATCH 11/14] chore(deps): update all dependencies (#1101) * chore(deps): update all dependencies * test: limit deps versions by python * remove pyproj, as it is only a transitive dependency Co-authored-by: Tim Swast Co-authored-by: Anthonios Partheniou --- samples/geography/requirements.txt | 14 ++++++-------- samples/magics/requirements.txt | 7 ++++--- samples/snippets/requirements.txt | 7 ++++--- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/samples/geography/requirements.txt b/samples/geography/requirements.txt index ca7e38f84..b07ba50bf 100644 --- a/samples/geography/requirements.txt +++ b/samples/geography/requirements.txt @@ -5,12 +5,12 @@ charset-normalizer==2.0.10 click==8.0.3 click-plugins==1.1.1 cligj==0.7.2 -dataclasses==0.6; python_version < '3.7' +dataclasses==0.8; python_version < '3.7' Fiona==1.8.20 geojson==2.5.0 geopandas==0.9.0; python_version < '3.7' geopandas==0.10.2; python_version >= '3.7' -google-api-core==2.3.2 +google-api-core==2.4.0 google-auth==2.3.3 google-cloud-bigquery==2.31.0 google-cloud-bigquery-storage==2.10.1 @@ -20,21 +20,19 @@ google-resumable-media==2.1.0 googleapis-common-protos==1.54.0 grpcio==1.43.0 idna==3.3 -libcst==0.3.23 +libcst==0.4.0 munch==2.5.0 mypy-extensions==0.4.3 packaging==21.3 pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' proto-plus==1.19.8 -protobuf==3.19.1 +protobuf==3.19.3 pyarrow==6.0.1 pyasn1==0.4.8 pyasn1-modules==0.2.8 pycparser==2.21 pyparsing==3.0.6 -pyproj==3.0.1; python_version < "3.7" -pyproj==3.1.0; python_version > "3.6" python-dateutil==2.8.2 pytz==2021.3 PyYAML==6.0 @@ -44,4 +42,4 @@ Shapely==1.8.0 six==1.16.0 typing-extensions==4.0.1 typing-inspect==0.7.1 -urllib3==1.26.7 +urllib3==1.26.8 diff --git a/samples/magics/requirements.txt b/samples/magics/requirements.txt index f4337e8fb..b47dcbc4f 100644 --- a/samples/magics/requirements.txt +++ b/samples/magics/requirements.txt @@ -2,10 +2,11 @@ google-cloud-bigquery-storage==2.10.1 google-auth-oauthlib==0.4.6 grpcio==1.43.0 ipython==7.16.1; python_version < '3.7' -ipython==7.29.0; python_version >= '3.7' +ipython==7.29.0; python_version == '3.7' +ipython==8.0.0; python_version >= '3.8' matplotlib==3.3.4; python_version < '3.7' -matplotlib==3.5.0rc1; python_version >= '3.7' +matplotlib==3.5.1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' pyarrow==6.0.1 pytz==2021.3 diff --git a/samples/snippets/requirements.txt b/samples/snippets/requirements.txt index bef333720..b47dcbc4f 100644 --- a/samples/snippets/requirements.txt +++ b/samples/snippets/requirements.txt @@ -2,10 +2,11 @@ google-cloud-bigquery-storage==2.10.1 google-auth-oauthlib==0.4.6 grpcio==1.43.0 ipython==7.16.1; python_version < '3.7' -ipython==7.29.0; python_version >= '3.7' +ipython==7.29.0; python_version == '3.7' +ipython==8.0.0; python_version >= '3.8' matplotlib==3.3.4; python_version < '3.7' -matplotlib==3.4.1; python_version >= '3.7' +matplotlib==3.5.1; python_version >= '3.7' pandas==1.1.5; python_version < '3.7' -pandas==1.3.4; python_version >= '3.7' +pandas==1.3.5; python_version >= '3.7' pyarrow==6.0.1 pytz==2021.3 From 51aced2e4354a9997ba7a9ebf7e6c72a03e35f6b Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Fri, 14 Jan 2022 16:26:41 -0500 Subject: [PATCH 12/14] chore(python): update release.sh to use keystore (#1111) Source-Link: https://github.com/googleapis/synthtool/commit/69fda12e2994f0b595a397e8bb6e3e9f380524eb Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 Co-authored-by: Owl Bot Co-authored-by: Tim Swast --- .github/.OwlBot.lock.yaml | 2 +- .kokoro/release.sh | 2 +- .kokoro/release/common.cfg | 12 +++++++++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index ff5126c18..eecb84c21 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:dfa9b663b32de8b5b327e32c1da665a80de48876558dd58091d8160c60ad7355 + digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 diff --git a/.kokoro/release.sh b/.kokoro/release.sh index 3abba6e06..b030caeef 100755 --- a/.kokoro/release.sh +++ b/.kokoro/release.sh @@ -26,7 +26,7 @@ python3 -m pip install --upgrade twine wheel setuptools export PYTHONUNBUFFERED=1 # Move into the package, build the distribution and upload. -TWINE_PASSWORD=$(cat "${KOKORO_GFILE_DIR}/secret_manager/google-cloud-pypi-token") +TWINE_PASSWORD=$(cat "${KOKORO_KEYSTORE_DIR}/73713_google-cloud-pypi-token-keystore-1") cd github/python-bigquery python3 setup.py sdist bdist_wheel twine upload --username __token__ --password "${TWINE_PASSWORD}" dist/* diff --git a/.kokoro/release/common.cfg b/.kokoro/release/common.cfg index 922d7fe50..6ae81b743 100644 --- a/.kokoro/release/common.cfg +++ b/.kokoro/release/common.cfg @@ -23,8 +23,18 @@ env_vars: { value: "github/python-bigquery/.kokoro/release.sh" } +# Fetch PyPI password +before_action { + fetch_keystore { + keystore_resource { + keystore_config_id: 73713 + keyname: "google-cloud-pypi-token-keystore-1" + } + } +} + # Tokens needed to report release status back to GitHub env_vars: { key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,google-cloud-pypi-token" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" } From f54d81fc5b4bb834fe5054de369f7158fff9bb25 Mon Sep 17 00:00:00 2001 From: Matthew Suozzo Date: Tue, 18 Jan 2022 11:21:23 -0500 Subject: [PATCH 13/14] test: make GC check resilient to broken weakrefs (#1099) * fix: Make gc check resilient to broken weakrefs In some python deployments (I believe macOS is one), weakref objects aren't guaranteed to exist across gc collections. The proposed check pattern is also used in tensorflow [0] which ran into similar issues a few years ago. [0]: https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/framework/errors_test.py#L33 * fix: add pragma for coverage * Fix whitespace --- tests/unit/test_dbapi_connection.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_dbapi_connection.py b/tests/unit/test_dbapi_connection.py index 0576cad38..11a268c68 100644 --- a/tests/unit/test_dbapi_connection.py +++ b/tests/unit/test_dbapi_connection.py @@ -219,8 +219,14 @@ def test_does_not_keep_cursor_instances_alive(self): # Connections should not hold strong references to the Cursor instances # they created, unnecessarily keeping them alive. gc.collect() - cursors = [obj for obj in gc.get_objects() if isinstance(obj, Cursor)] - self.assertEqual(len(cursors), 2) + cursor_count = 0 + for obj in gc.get_objects(): + try: + if isinstance(obj, Cursor): + cursor_count += 1 + except ReferenceError: # pragma: NO COVER + pass + self.assertEqual(cursor_count, 2) def test_commit(self): connection = self._make_one(client=self._mock_client()) From fa8aea03879363f4d12ef0c2557047b56ae08714 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Tue, 18 Jan 2022 20:24:46 -0500 Subject: [PATCH 14/14] chore(python): Noxfile recognizes that tests can live in a folder (#1115) Source-Link: https://github.com/googleapis/synthtool/commit/4760d8dce1351d93658cb11d02a1b7ceb23ae5d7 Post-Processor: gcr.io/cloud-devrel-public-resources/owlbot-python:latest@sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828 Co-authored-by: Owl Bot --- .github/.OwlBot.lock.yaml | 2 +- samples/geography/noxfile.py | 1 + samples/magics/noxfile.py | 1 + samples/snippets/noxfile.py | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index eecb84c21..52d79c11f 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -1,3 +1,3 @@ docker: image: gcr.io/cloud-devrel-public-resources/owlbot-python:latest - digest: sha256:ae600f36b6bc972b368367b6f83a1d91ec2c82a4a116b383d67d547c56fe6de3 + digest: sha256:f0e4b51deef56bed74d3e2359c583fc104a8d6367da3984fc5c66938db738828 diff --git a/samples/geography/noxfile.py b/samples/geography/noxfile.py index 3bbef5d54..20cdfc620 100644 --- a/samples/geography/noxfile.py +++ b/samples/geography/noxfile.py @@ -187,6 +187,7 @@ def _session_tests( ) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") else: diff --git a/samples/magics/noxfile.py b/samples/magics/noxfile.py index 3bbef5d54..20cdfc620 100644 --- a/samples/magics/noxfile.py +++ b/samples/magics/noxfile.py @@ -187,6 +187,7 @@ def _session_tests( ) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") else: diff --git a/samples/snippets/noxfile.py b/samples/snippets/noxfile.py index 3bbef5d54..20cdfc620 100644 --- a/samples/snippets/noxfile.py +++ b/samples/snippets/noxfile.py @@ -187,6 +187,7 @@ def _session_tests( ) -> None: # check for presence of tests test_list = glob.glob("*_test.py") + glob.glob("test_*.py") + test_list.extend(glob.glob("tests")) if len(test_list) == 0: print("No tests found, skipping directory.") else: