diff --git a/BUILD.bazel b/BUILD.bazel index cf7a87fbf5..4d6451aa92 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -1,21 +1,73 @@ +load("//:gapic_generator_python.bzl", "pandoc_binary", "pandoc_toolchain") load("@gapic_generator_python_pip_deps//:requirements.bzl", "requirement") +load("@bazel_tools//tools/python:toolchain.bzl", "py_runtime_pair") + +toolchain_type( + name = "pandoc_toolchain_type", + visibility = ["//visibility:public"], +) + +pandoc_toolchain( + exec_compatible_with = [ + "@bazel_tools//platforms:linux", + "@bazel_tools//platforms:x86_64", + ], + platform = "linux", +) + +pandoc_toolchain( + exec_compatible_with = [ + "@bazel_tools//platforms:osx", + "@bazel_tools//platforms:x86_64", + ], + platform = "macOS", +) + +pandoc_binary( + name = "pandoc_binary", +) + +config_setting( + name = "gapic_gen_python_3_6", + values = {"define": "gapic_gen_python=3.6"}, +) + +py_runtime( + name = "pyenv3_runtime", + interpreter = ":pyenv3wrapper.sh", + python_version="PY3", +) + +py_runtime_pair( + name = "pyenv3_runtime_pair", + py3_runtime = ":pyenv3_runtime", +) + +toolchain( + name = "pyenv3_toolchain", + toolchain = ":pyenv3_runtime_pair", + toolchain_type = "@bazel_tools//tools/python:toolchain_type", +) py_binary( name = "gapic_plugin", srcs = glob(["gapic/**/*.py"]), - data = glob(["gapic/**/*.j2"]), - main = "gapic/cli/generate.py", + data = [":pandoc_binary"] + glob(["gapic/**/*.j2"]), + main = "gapic/cli/generate_with_pandoc.py", + python_version = "PY3", visibility = ["//visibility:public"], deps = [ "@com_google_protobuf//:protobuf_python", + "@com_github_grpc_grpc//src/python/grpcio/grpc:grpcio", requirement("click"), requirement("google-api-core"), requirement("googleapis-common-protos"), - requirement("grpcio"), requirement("jinja2"), requirement("MarkupSafe"), requirement("pypandoc"), requirement("PyYAML"), - ], - python_version = "PY3", + ] + select({ + ":gapic_gen_python_3_6": [requirement("dataclasses")], + "//conditions:default": [], + }), ) diff --git a/WORKSPACE b/WORKSPACE index 57a357e7b9..b0cee19c28 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -2,9 +2,11 @@ workspace(name = "gapic_generator_python") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -# -# Import rules_python -# +http_archive( + name = "bazel_skylib", + urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/0.9.0/bazel_skylib-0.9.0.tar.gz"], +) + http_archive( name = "rules_python", strip_prefix = "rules_python-748aa53d7701e71101dfd15d800e100f6ff8e5d1", @@ -22,10 +24,15 @@ pip_repositories() # # Import gapic-generator-python specific dependencies # -load("//:repositories.bzl", "gapic_generator_python") +load("//:repositories.bzl", + "gapic_generator_python", + "gapic_generator_register_toolchains" +) gapic_generator_python() +gapic_generator_register_toolchains() + load("@gapic_generator_python_pip_deps//:requirements.bzl", "pip_install") pip_install() @@ -33,3 +40,27 @@ pip_install() load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") protobuf_deps() + +# +# Import grpc as a native bazel dependency. This avoids duplication and also +# speeds up loading phase a lot (otherwise python_rules will be building grpcio +# from sources in a single-core speed, which takes around 5 minutes on a regular +# workstation) +# +load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") + +grpc_deps() + +load("@upb//bazel:repository_defs.bzl", "bazel_version_repository") + +bazel_version_repository( + name = "bazel_version", +) + +load("@build_bazel_rules_apple//apple:repositories.bzl", "apple_rules_dependencies") + +apple_rules_dependencies() + +load("@build_bazel_apple_support//lib:repositories.bzl", "apple_support_dependencies") + +apple_support_dependencies() diff --git a/gapic/cli/generate_with_pandoc.py b/gapic/cli/generate_with_pandoc.py new file mode 100644 index 0000000000..264d1c8b0b --- /dev/null +++ b/gapic/cli/generate_with_pandoc.py @@ -0,0 +1,9 @@ +import os + +from gapic.cli import generate + +if __name__ == '__main__': + os.environ['PYPANDOC_PANDOC'] = os.path.join( + os.path.abspath(__file__).rsplit("gapic", 1)[0], "pandoc") + os.environ['LC_ALL'] = 'C.UTF-8' + generate.generate() diff --git a/gapic_generator_python.bzl b/gapic_generator_python.bzl new file mode 100644 index 0000000000..c76f51a125 --- /dev/null +++ b/gapic_generator_python.bzl @@ -0,0 +1,62 @@ +def _pandoc_binary_impl(ctx): + toolchain = ctx.toolchains["@gapic_generator_python//:pandoc_toolchain_type"] + output = ctx.actions.declare_file(ctx.attr.binary_name) + + script = """ + cp {input} {output} + chmod +x {output} + """.format( + input = toolchain.pandoc.files.to_list()[0].path, + output = output.path, + ) + ctx.actions.run_shell( + command = script, + inputs = toolchain.pandoc.files, + outputs = [output], + ) + return [DefaultInfo(files = depset(direct = [output]), executable = output)] + +pandoc_binary = rule( + attrs = { + "binary_name": attr.string(default = "pandoc") + }, + executable = True, + toolchains = ["@gapic_generator_python//:pandoc_toolchain_type"], + implementation = _pandoc_binary_impl, +) + +# +# Toolchains +# +def _pandoc_toolchain_info_impl(ctx): + return [ + platform_common.ToolchainInfo( + pandoc = ctx.attr.pandoc, + ), + ] + +_pandoc_toolchain_info = rule( + attrs = { + "pandoc": attr.label( + allow_single_file = True, + cfg = "host", + executable = True, + ), + }, + implementation = _pandoc_toolchain_info_impl, +) + +def pandoc_toolchain(platform, exec_compatible_with): + toolchain_info_name = "pandoc_toolchain_info_%s" % platform + _pandoc_toolchain_info( + name = toolchain_info_name, + pandoc = "@pandoc_%s//:pandoc" % platform, + visibility = ["//visibility:public"], + ) + + native.toolchain( + name = "pandoc_toolchain_%s" % platform, + exec_compatible_with = exec_compatible_with, + toolchain = toolchain_info_name, + toolchain_type = ":pandoc_toolchain_type", + ) diff --git a/pyenv3wrapper.sh b/pyenv3wrapper.sh new file mode 100755 index 0000000000..54176219f7 --- /dev/null +++ b/pyenv3wrapper.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +HOME_DIR=$(getent passwd "$(whoami)" | cut -d: -f6) +exec "$HOME_DIR/.pyenv/shims/python3" "$@" diff --git a/repositories.bzl b/repositories.bzl index ed39838d4f..a31c6440cf 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -1,6 +1,13 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@rules_python//python:pip.bzl", "pip_import") +_PANDOC_BUILD_FILE = """ +filegroup( + name = "pandoc", + srcs = ["bin/pandoc"], + visibility = ["//visibility:public"], +)""" + def gapic_generator_python(): _maybe( pip_import, @@ -25,6 +32,29 @@ def gapic_generator_python(): urls = ["https://github.com/bazelbuild/bazel-skylib/archive/2169ae1c374aab4a09aa90e65efe1a3aad4e279b.tar.gz"], ) + _maybe( + http_archive, + name = "com_github_grpc_grpc", + strip_prefix = "grpc-8347f4753568b5b66e49111c60ae2841278d3f33", # this is 1.25.0 with fixes + urls = ["https://github.com/grpc/grpc/archive/8347f4753568b5b66e49111c60ae2841278d3f33.zip"], + ) + + _maybe( + http_archive, + name = "pandoc_linux", + build_file_content = _PANDOC_BUILD_FILE, + strip_prefix = "pandoc-2.2.1", + url = "https://github.com/jgm/pandoc/releases/download/2.2.1/pandoc-2.2.1-linux.tar.gz", + ) + + _maybe( + http_archive, + name = "pandoc_macOS", + build_file_content = _PANDOC_BUILD_FILE, + strip_prefix = "pandoc-2.2.1", + url = "https://github.com/jgm/pandoc/releases/download/2.2.1/pandoc-2.2.1-macOS.zip", + ) + _maybe( http_archive, name = "com_google_api_codegen", @@ -32,6 +62,12 @@ def gapic_generator_python(): urls = ["https://github.com/googleapis/gapic-generator/archive/b32c73219d617f90de70bfa6ff0ea0b0dd638dfe.zip"], ) +def gapic_generator_register_toolchains(): + native.register_toolchains( + "@gapic_generator_python//:pandoc_toolchain_linux", + "@gapic_generator_python//:pandoc_toolchain_macOS", + ) + def _maybe(repo_rule, name, strip_repo_prefix = "", **kwargs): if not name.startswith(strip_repo_prefix): return diff --git a/requirements.txt b/requirements.txt index ed6e2ab8e4..58e6500478 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ click==7.1.2 google-api-core==1.17.0 googleapis-common-protos==1.51.0 -grpcio==1.28.1 jinja2==2.11.2 MarkupSafe==1.1.1 protobuf==3.11.3 pypandoc==1.5 PyYAML==5.3.1 +dataclasses==0.6 \ No newline at end of file diff --git a/rules_python_gapic/py_gapic.bzl b/rules_python_gapic/py_gapic.bzl index 50d792b82b..f132e32dce 100644 --- a/rules_python_gapic/py_gapic.bzl +++ b/rules_python_gapic/py_gapic.bzl @@ -14,7 +14,7 @@ load("@com_google_api_codegen//rules_gapic:gapic.bzl", "proto_custom_library") -def py_gapic_library(name, srcs, **kwargs): +def py_gapic_library(name, srcs, plugin_args = [], **kwargs): # srcjar_target_name = "%s_srcjar" % name srcjar_target_name = name srcjar_output_suffix = ".srcjar" @@ -23,7 +23,7 @@ def py_gapic_library(name, srcs, **kwargs): name = srcjar_target_name, deps = srcs, plugin = Label("@gapic_generator_python//:gapic_plugin"), - plugin_args = [], + plugin_args = plugin_args, plugin_file_args = {}, output_type = "python_gapic", output_suffix = srcjar_output_suffix, diff --git a/rules_python_gapic/py_gapic_pkg.bzl b/rules_python_gapic/py_gapic_pkg.bzl new file mode 100644 index 0000000000..ec80c87a84 --- /dev/null +++ b/rules_python_gapic/py_gapic_pkg.bzl @@ -0,0 +1,70 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@com_google_api_codegen//rules_gapic:gapic_pkg.bzl", "construct_package_dir_paths") + +def _py_gapic_src_pkg_impl(ctx): + srcjar_srcs = [] + for dep in ctx.attr.deps: + for f in dep.files.to_list(): + if f.extension in ("srcjar", "jar", "zip"): + srcjar_srcs.append(f) + + paths = construct_package_dir_paths(ctx.attr.package_dir, ctx.outputs.pkg, ctx.label.name) + + script = """ + mkdir -p {package_dir_path} + for srcjar_src in {srcjar_srcs}; do + unzip -q -o $srcjar_src -d {package_dir_path} + done + cd {package_dir_path}/.. + tar -zchpf {package_dir}/{package_dir}.tar.gz {package_dir} + cd - + mv {package_dir_path}/{package_dir}.tar.gz {pkg} + rm -rf {package_dir_path} + """.format( + srcjar_srcs = " ".join(["'%s'" % f.path for f in srcjar_srcs]), + package_dir_path = paths.package_dir_path, + package_dir = paths.package_dir, + pkg = ctx.outputs.pkg.path, + package_dir_expr = paths.package_dir_expr, + ) + + ctx.actions.run_shell( + inputs = srcjar_srcs, + command = script, + outputs = [ctx.outputs.pkg], + ) + +_py_gapic_src_pkg = rule( + attrs = { + "deps": attr.label_list(allow_files = True, mandatory = True), + "package_dir": attr.string(mandatory = True), + }, + outputs = {"pkg": "%{name}.tar.gz"}, + implementation = _py_gapic_src_pkg_impl, +) + +def py_gapic_assembly_pkg(name, deps, assembly_name = None, **kwargs): + package_dir = name + if assembly_name: + package_dir = "%s-%s" % (assembly_name, name) + _py_gapic_src_pkg( + name = name, + deps = deps, + package_dir = package_dir, + **kwargs + ) + +