diff --git a/noxfile.py b/noxfile.py index 7b1fd9b..4b15f4d 100644 --- a/noxfile.py +++ b/noxfile.py @@ -14,246 +14,260 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Generated by synthtool. DO NOT EDIT! - -from __future__ import absolute_import import os import pathlib +from pathlib import Path +import re import shutil -import warnings import nox + BLACK_VERSION = "black==22.3.0" -ISORT_VERSION = "isort==5.10.1" -LINT_PATHS = ["docs", "google", "tests", "noxfile.py", "setup.py"] +LINT_PATHS = ["docs", "google", "noxfile.py", "setup.py"] -# NOTE: Pin the version of grpcio-tools to 1.48.2 for compatibility with -# Protobuf 3.19.5. Please ensure that the minimum required version of -# protobuf in setup.py is compatible with the pb2 files generated -# by grpcio-tools before changing the pinned version below. -GRPCIO_TOOLS_VERSION = "grpcio-tools==1.48.2" -DEFAULT_PYTHON_VERSION = "3.8" - -UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] -UNIT_TEST_STANDARD_DEPENDENCIES = [ - "mock", - "asyncmock", - "pytest", - "pytest-cov", - "pytest-asyncio", -] -UNIT_TEST_EXTERNAL_DEPENDENCIES = [] -UNIT_TEST_LOCAL_DEPENDENCIES = [] -UNIT_TEST_DEPENDENCIES = [] -UNIT_TEST_EXTRAS = [] -UNIT_TEST_EXTRAS_BY_PYTHON = {} - -SYSTEM_TEST_PYTHON_VERSIONS = ["3.8"] -SYSTEM_TEST_STANDARD_DEPENDENCIES = [ - "mock", - "pytest", - "google-cloud-testutils", -] -SYSTEM_TEST_EXTERNAL_DEPENDENCIES = [] -SYSTEM_TEST_LOCAL_DEPENDENCIES = [] -SYSTEM_TEST_DEPENDENCIES = [] -SYSTEM_TEST_EXTRAS = [] -SYSTEM_TEST_EXTRAS_BY_PYTHON = {} +# `grpcio-tools` 1.59.0 or newer is required for protobuf 5.x compatibility. +GRPCIO_TOOLS_VERSION = "grpcio-tools==1.59.0" CURRENT_DIRECTORY = pathlib.Path(__file__).parent.absolute() - -# 'docfx' is excluded since it only needs to run in 'docs-presubmit' -nox.options.sessions = [ - "system", - "lint", - "integration_test", - "lint_setup_py", - "blacken", - "docs", - "unit", -] - -# Error if a python version is missing -nox.options.error_on_missing_interpreters = True - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def lint(session): - """Run linters. - - Returns a failure if the linters find linting errors or sufficiently - serious code quality issues. - """ - session.install("flake8", BLACK_VERSION) - session.run( - "black", - "--check", - *LINT_PATHS, - ) - session.run("flake8", "google", "tests") +UNIT_TEST_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] -@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.session(python="3.8") def blacken(session): - """Run black. Format code to uniform standard.""" - session.install(BLACK_VERSION) - session.run( - "black", - *LINT_PATHS, - ) - - -@nox.session(python=DEFAULT_PYTHON_VERSION) -def format(session): - """ - Run isort to sort imports. Then run black - to format code to uniform standard. + """Run black. + Format code to uniform standard. + This currently uses Python 3.6 due to the automated Kokoro run of synthtool. + That run uses an image that doesn't have 3.6 installed. Before updating this + check the state of the `gcp_ubuntu_config` we use for that Kokoro run. """ - session.install(BLACK_VERSION, ISORT_VERSION) - # Use the --fss option to sort imports using strict alphabetical order. - # See https://pycqa.github.io/isort/docs/configuration/options.html#force-sort-within-sections - session.run( - "isort", - "--fss", - *LINT_PATHS, - ) - session.run( - "black", - *LINT_PATHS, - ) + session.install(BLACK_VERSION) + session.run("black", "google", "setup.py") -@nox.session(python=DEFAULT_PYTHON_VERSION) +@nox.session(python="3.8") def lint_setup_py(session): - """Verify that setup.py is valid (including RST check).""" + """Verify that setup.py is valid""" session.install("docutils", "pygments") - session.run("python", "setup.py", "check", "--restructuredtext", "--strict") + session.run("python", "setup.py", "check", "--strict") -def install_unittest_dependencies(session, *constraints): - standard_deps = UNIT_TEST_STANDARD_DEPENDENCIES + UNIT_TEST_DEPENDENCIES - session.install(*standard_deps, *constraints) +def unit(session, repository, package, prerelease, protobuf_implementation): + """Run the unit test suite.""" + downstream_dir = repository + if package: + downstream_dir = f"{repository}/packages/{package}" - if UNIT_TEST_EXTERNAL_DEPENDENCIES: - warnings.warn( - "'unit_test_external_dependencies' is deprecated. Instead, please " - "use 'unit_test_dependencies' or 'unit_test_local_dependencies'.", - DeprecationWarning, - ) - session.install(*UNIT_TEST_EXTERNAL_DEPENDENCIES, *constraints) + # Install all test dependencies, then install this package in-place. + session.install("asyncmock", "pytest-asyncio") - if UNIT_TEST_LOCAL_DEPENDENCIES: - session.install(*UNIT_TEST_LOCAL_DEPENDENCIES, *constraints) + # Pin mock due to https://github.com/googleapis/python-pubsub/issues/840 + session.install("mock==5.0.0", "pytest", "pytest-cov") - if UNIT_TEST_EXTRAS_BY_PYTHON: - extras = UNIT_TEST_EXTRAS_BY_PYTHON.get(session.python, []) - elif UNIT_TEST_EXTRAS: - extras = UNIT_TEST_EXTRAS - else: - extras = [] + install_command = ["-e", f"{CURRENT_DIRECTORY}/{downstream_dir}"] - if extras: - session.install("-e", f".[{','.join(extras)}]", *constraints) + if prerelease: + install_prerelease_dependencies( + session, + f"{CURRENT_DIRECTORY}/{downstream_dir}/testing/constraints-{UNIT_TEST_PYTHON_VERSIONS[0]}.txt", + ) + # Use the `--no-deps` options to allow pre-release versions of dependencies to be installed + install_command.extend(["--no-deps"]) else: - session.install("-e", ".", *constraints) + # Install the pinned dependencies in constraints file + install_command.extend( + [ + "-c", + f"{CURRENT_DIRECTORY}/{downstream_dir}/testing/constraints-{session.python}.txt", + ] + ) + # These *must* be the last 3 install commands to get the packages from source. + session.install(*install_command) -def default(session): - # Install all test dependencies, then install this package in-place. + # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. + # The 'cpp' implementation requires Protobuf<4. + if protobuf_implementation == "cpp": + session.install("protobuf<4") - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" - ) - install_unittest_dependencies(session, "-c", constraints_path) + # Install this library from source + session.install(".", "--no-deps") - # Run py.test against the unit tests. + # Print out package versions of dependencies session.run( - "py.test", - "--quiet", - f"--junitxml=unit_{session.python}_sponge_log.xml", - "--cov=google", - "--cov=tests/unit", - "--cov-append", - "--cov-config=.coveragerc", - "--cov-report=", - "--cov-fail-under=0", - os.path.join("tests", "unit"), - *session.posargs, + "python", "-c", "import google.protobuf; print(google.protobuf.__version__)" ) + session.run("python", "-c", "import grpc; print(grpc.__version__)") + session.run("python", "-c", "import google.auth; print(google.auth.__version__)") + session.run( + "python", "-c", "import google.api_core; print(google.api_core.__version__)" + ) -def install_systemtest_dependencies(session, *constraints): - - # Use pre-release gRPC for system tests. - session.install("--pre", "grpcio") - - session.install(*SYSTEM_TEST_STANDARD_DEPENDENCIES, *constraints) - - if SYSTEM_TEST_EXTERNAL_DEPENDENCIES: - session.install(*SYSTEM_TEST_EXTERNAL_DEPENDENCIES, *constraints) - - if SYSTEM_TEST_LOCAL_DEPENDENCIES: - session.install("-e", *SYSTEM_TEST_LOCAL_DEPENDENCIES, *constraints) - - if SYSTEM_TEST_DEPENDENCIES: - session.install("-e", *SYSTEM_TEST_DEPENDENCIES, *constraints) + # Run py.test against the unit tests in the downstream repository + with session.chdir(downstream_dir): + # Run py.test against the unit tests. + session.run( + "py.test", + "--quiet", + "--cov=google/cloud", + "--cov=tests/unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "unit"), + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) - if SYSTEM_TEST_EXTRAS_BY_PYTHON: - extras = SYSTEM_TEST_EXTRAS_BY_PYTHON.get(session.python, []) - elif SYSTEM_TEST_EXTRAS: - extras = SYSTEM_TEST_EXTRAS - else: - extras = [] - if extras: - session.install("-e", f".[{','.join(extras)}]", *constraints) - else: - session.install("-e", ".", *constraints) +def install_prerelease_dependencies(session, constraints_path): + with open(constraints_path, encoding="utf-8") as constraints_file: + constraints_text = constraints_file.read() + # Ignore leading whitespace and comment lines. + constraints_deps = [ + match.group(1) + for match in re.finditer( + r"^\s*(\S+)(?===\S+)", constraints_text, flags=re.MULTILINE + ) + ] + session.install(*constraints_deps) + prerel_deps = [ + "googleapis-common-protos", + "protobuf", + "six", + "grpcio", + "grpcio-status", + "google-api-core", + "google-auth", + "proto-plus", + "google-cloud-testutils", + # dependencies of google-cloud-testutils" + "click", + ] + + for dep in prerel_deps: + session.install("--pre", "--no-deps", "--upgrade", dep) + + # Remaining dependencies + other_deps = [ + "requests", + ] + session.install(*other_deps) @nox.session(python=UNIT_TEST_PYTHON_VERSIONS) @nox.parametrize( - "library", + "library,prerelease,protobuf_implementation", [ - ("google-cloud-python", "google-cloud-asset"), + (("google-cloud-python", "google-cloud-asset"), False, "python"), + (("google-cloud-python", "google-cloud-asset"), False, "upb"), + (("google-cloud-python", "google-cloud-asset"), False, "cpp"), + (("google-cloud-python", "google-cloud-asset"), True, "python"), + (("google-cloud-python", "google-cloud-asset"), True, "upb"), + (("google-cloud-python", "google-cloud-asset"), True, "cpp"), ], - ids=["asset"], ) -def integration_test(session, library): +def unit_remote(session, library, prerelease, protobuf_implementation): """Run tests from a downstream libraries. + To verify that any changes we make here will not break downstream libraries, clone a few and run their unit and system tests. + NOTE: The unit and system test functions above are copied from the templates. They will need to be updated when the templates change. + + * Pub/Sub: GAPIC with handwritten layer. + * Speech: Full GAPIC, has long running operations. """ - package = "" - if type(library) == tuple: - library, package = library + + if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"): + session.skip("cpp implementation is not supported in python 3.11+") + + repository, package = library try: - session.run("git", "-C", library, "pull", external=True) + session.run("git", "-C", repository, "pull", external=True) except nox.command.CommandFailed: session.run( "git", "clone", "--single-branch", - f"https://github.com/googleapis/{library}", + f"https://github.com/googleapis/{repository}", external=True, ) - session.cd(library) - if package: - session.cd(f"packages/{package}") - default(session) - # system tests are run 3.7 only - if session.python == "3.7": - system(session) + unit( + session=session, + repository=repository, + package=package, + prerelease=prerelease, + protobuf_implementation=protobuf_implementation, + ) + + +@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) +@nox.parametrize("protobuf_implementation", ["python", "upb", "cpp"]) +def unit_local(session, protobuf_implementation): + """Run tests in this local repo.""" + # Install all test dependencies, then install this package in-place. + + # TODO(https://github.com/googleapis/proto-plus-python/issues/389): + # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. + # The 'cpp' implementation requires Protobuf == 3.x however version 3.x + # does not support Python 3.11 and newer. The 'cpp' implementation + # must be excluded from the test matrix for these runtimes. + if protobuf_implementation == "cpp" and session.python in ("3.11", "3.12"): + session.skip("cpp implementation is not supported in python 3.11+") + + constraints_path = str( + CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + ) + session.install( + "mock", + "asyncmock", + "pytest", + "pytest-cov", + "pytest-asyncio", + "-c", + constraints_path, + ) + + session.install("-e", ".", "-c", constraints_path) + + # Remove the 'cpp' implementation once support for Protobuf 3.x is dropped. + # The 'cpp' implementation requires Protobuf<4. + if protobuf_implementation == "cpp": + session.install("protobuf<4") + + # Run py.test against the unit tests. + session.run( + "py.test", + "--quiet", + f"--junitxml=unit_{session.python}_sponge_log.xml", + "--cov=google", + "--cov=tests/unit", + "--cov-append", + "--cov-config=.coveragerc", + "--cov-report=", + "--cov-fail-under=0", + os.path.join("tests", "unit"), + *session.posargs, + env={ + "PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION": protobuf_implementation, + }, + ) @nox.session(python="3.8") def generate_protos(session): """Generates the protos using protoc. + + This session but be last to avoid overwriting the protos used in CI runs. + Some notes on the `google` directory: 1. The `_pb2.py` files are produced by protoc. 2. The .proto files are non-functional but are left in the repository @@ -262,60 +276,12 @@ def generate_protos(session): If a new subdirectory is added, you will need to create more `__init__.py` files. """ - session.install(GRPCIO_TOOLS_VERSION) - protos = [str(p) for p in (pathlib.Path(".").glob("google/**/*.proto"))] + session.install(GRPCIO_TOOLS_VERSION) + protos = [str(p) for p in (Path(".").glob("google/**/*.proto"))] session.run( - "python", - "-m", - "grpc_tools.protoc", - "--proto_path=.", - "--python_out=.", - *protos, - ) - - -@nox.session(python=SYSTEM_TEST_PYTHON_VERSIONS) -def system(session): - """Run the system test suite.""" - constraints_path = str( - CURRENT_DIRECTORY / "testing" / f"constraints-{session.python}.txt" + "python", "-m", "grpc_tools.protoc", "--proto_path=.", "--python_out=.", *protos ) - system_test_path = os.path.join("tests", "system.py") - system_test_folder_path = os.path.join("tests", "system") - - # Check the value of `RUN_SYSTEM_TESTS` env var. It defaults to true. - if os.environ.get("RUN_SYSTEM_TESTS", "true") == "false": - session.skip("RUN_SYSTEM_TESTS is set to false, skipping") - # Install pyopenssl for mTLS testing. - if os.environ.get("GOOGLE_API_USE_CLIENT_CERTIFICATE", "false") == "true": - session.install("pyopenssl") - - system_test_exists = os.path.exists(system_test_path) - system_test_folder_exists = os.path.exists(system_test_folder_path) - # Sanity check: only run tests if found. - if not system_test_exists and not system_test_folder_exists: - session.skip("System tests were not found") - - install_systemtest_dependencies(session, "-c", constraints_path) - - # Run py.test against the system tests. - if system_test_exists: - session.run( - "py.test", - "--quiet", - f"--junitxml=system_{session.python}_sponge_log.xml", - system_test_path, - *session.posargs, - ) - if system_test_folder_exists: - session.run( - "py.test", - "--quiet", - f"--junitxml=system_{session.python}_sponge_log.xml", - system_test_folder_path, - *session.posargs, - ) @nox.session(python="3.9") @@ -399,7 +365,16 @@ def docfx(session): ) -@nox.session(python=UNIT_TEST_PYTHON_VERSIONS) -def unit(session): - """Run the unit test suite.""" - default(session) +@nox.session(python="3.8") +def lint(session): + """Run linters. + + Returns a failure if the linters find linting errors or sufficiently + serious code quality issues. + """ + session.install("flake8", BLACK_VERSION) + session.run( + "black", + "--check", + *LINT_PATHS, + ) diff --git a/setup.py b/setup.py index 9fefb46..72d57d3 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ release_status = "Development Status :: 4 - Beta" dependencies = [ "google-api-core[grpc] >= 1.34.1, <3.0.0dev,!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.*,!=2.4.*,!=2.5.*,!=2.6.*,!=2.7.*,!=2.8.*,!=2.9.*,!=2.10.*", - "protobuf>=3.19.5,<5.0.0dev,!=3.20.0,!=3.20.1,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", + "protobuf>=3.20.2,<6.0.0dev,!=4.21.1,!=4.21.2,!=4.21.3,!=4.21.4,!=4.21.5", ] # Setup boilerplate below this line. diff --git a/testing/constraints-3.7.txt b/testing/constraints-3.7.txt index 7a1c665..5dccad8 100644 --- a/testing/constraints-3.7.txt +++ b/testing/constraints-3.7.txt @@ -5,5 +5,5 @@ # # e.g., if setup.py has "foo >= 1.14.0, < 2.0.0dev", # Then this file should have foo==1.14.0 -protobuf==3.19.5 +protobuf==3.20.2 google-api-core==1.34.1