diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 50875c8c..ba5458d4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,16 +14,22 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] + node_version: [14.x, 16.x] + protoc_version: [3.x] steps: - uses: actions/checkout@v2 with: submodules: true - - name: Setup node 12 - uses: actions/setup-node@v2 + - uses: actions/setup-node@v2 with: - node-version: 12.x + node-version: ${{matrix.node_version}} + - uses: arduino/setup-protoc@v1 + with: + version: ${{matrix.protoc_version}} + repo-token: ${{ secrets.GITHUB_TOKEN }} + - run: yarn --frozen-lockfile - run: yarn codegen - - run: yarn test + - run: yarn test --test_tag_filters=-no-${{ matrix.os }} - run: cd examples/pure && yarn --frozen-lockfile && yarn test \ No newline at end of file diff --git a/.npmignore b/.npmignore new file mode 100644 index 00000000..921e91ce --- /dev/null +++ b/.npmignore @@ -0,0 +1,15 @@ +third-party +test +docs +examples +scripts +.* +.tgz +# ignore everything that is typescript or proto file +src/**/*.ts +src/**/*.proto +# bazel stuff +BUILD.bazel +WORKSPACE +bazel-* + diff --git a/BUILD.bazel b/BUILD.bazel index a5b453b2..33c0867b 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -6,8 +6,7 @@ pkg_npm( name = "package", package_name = "protoc-gen-ts", srcs = [ - "//bin:protoc-gen-ts", - "//bin:protoc-gen-ts.cmd", + "//bin:protoc-gen-ts.js", "index.bzl", "package.json", "README.md", diff --git a/WORKSPACE b/WORKSPACE index 2c83b2d9..bf7081b6 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -10,8 +10,8 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # Setup NodeJS toolchain http_archive( name = "build_bazel_rules_nodejs", - sha256 = "5c40083120eadec50a3497084f99bc75a85400ea727e82e0b2f422720573130f", - urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/4.0.0-beta.0/rules_nodejs-4.0.0-beta.0.tar.gz"], + sha256 = "b32a4713b45095e9e1921a7fcb1adf584bc05959f3336e7351bcf77f015a2d7c", + urls = ["https://github.com/bazelbuild/rules_nodejs/releases/download/4.1.0/rules_nodejs-4.1.0.tar.gz"], ) load("@build_bazel_rules_nodejs//:index.bzl", "node_repositories", "yarn_install") @@ -29,10 +29,11 @@ yarn_install( # Setup Protocol Buffers toolchain http_archive( name = "rules_proto", - sha256 = "57001a3b33ec690a175cdf0698243431ef27233017b9bed23f96d44b9c98242f", - strip_prefix = "rules_proto-9cd4f8f1ede19d81c6d48910429fe96776e567b1", + sha256 = "66bfdf8782796239d3875d37e7de19b1d94301e8972b3cbd2446b332429b4df1", + strip_prefix = "rules_proto-4.0.0", urls = [ - "https://github.com/bazelbuild/rules_proto/archive/9cd4f8f1ede19d81c6d48910429fe96776e567b1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/rules_proto/archive/refs/tags/4.0.0.tar.gz", + "https://github.com/bazelbuild/rules_proto/archive/refs/tags/4.0.0.tar.gz", ], ) diff --git a/bin/BUILD.bazel b/bin/BUILD.bazel index a10e7c9b..82c427a3 100644 --- a/bin/BUILD.bazel +++ b/bin/BUILD.bazel @@ -4,13 +4,13 @@ package(default_visibility = ["//visibility:public"]) nodejs_binary( name = "bin", - entry_point = ":protoc-gen-ts", + entry_point = ":protoc-gen-ts.js", data = [ "//src", - ":protoc-gen-ts" + ":protoc-gen-ts.js" ], # See: https://github.com/bazelbuild/rules_nodejs/issues/2600 templated_args = ["--bazel_patch_module_resolver"], ) -exports_files(["protoc-gen-ts", "protoc-gen-ts.cmd"]) \ No newline at end of file +exports_files(["protoc-gen-ts.js"]) \ No newline at end of file diff --git a/bin/protoc-gen-ts.cmd b/bin/protoc-gen-ts.cmd deleted file mode 100644 index e9f704f2..00000000 --- a/bin/protoc-gen-ts.cmd +++ /dev/null @@ -1,14 +0,0 @@ -@echo off - -@REM check if package is installed locally. -for /f %%i in ('npm root') do set install_path=%%i - -@REM if not installed locally, set install path to global one -setlocal enabledelayedexpansion -if not exist !install_path!\protoc-gen-ts\ ( - for /f %%i in ('npm root -g') do set install_path=%%i -) -endlocal & set install_path=%install_path% - -@REM invoke the index.js -node %install_path%\protoc-gen-ts\src\index \ No newline at end of file diff --git a/bin/protoc-gen-ts b/bin/protoc-gen-ts.js similarity index 97% rename from bin/protoc-gen-ts rename to bin/protoc-gen-ts.js index ffd16038..93c36f05 100644 --- a/bin/protoc-gen-ts +++ b/bin/protoc-gen-ts.js @@ -1,3 +1,2 @@ #!/usr/bin/env node - require('../src/index') \ No newline at end of file diff --git a/examples/package.json b/examples/package.json index 7c010f39..b8ae5d37 100644 --- a/examples/package.json +++ b/examples/package.json @@ -4,13 +4,12 @@ "grpc:client": "bazel run //grpc:client", "grpc:server": "bazel run //grpc:server", "pretest": "bazel build //grpc:type //bazel:message && node ../scripts/sync_generated_protos.js grpc && node ../scripts/sync_generated_protos.js bazel", - "test": "bazel run //bazel:bin", - "preinstall": "cd ../ && bazel build :package" + "test": "bazel run //bazel:bin" }, "dependencies": { "@grpc/grpc-js": "^1.2.12", "google-protobuf": "^3.15.8", - "protoc-gen-ts": "file:../bazel-bin/package" + "protoc-gen-ts": "file:../" }, "devDependencies": { "@bazel/bazelisk": "^1.7.5", diff --git a/examples/pure/package.json b/examples/pure/package.json index a410f7ea..0775fb27 100644 --- a/examples/pure/package.json +++ b/examples/pure/package.json @@ -2,7 +2,6 @@ "name": "example", "scripts": { "preinstall": "cd ../.. && yarn bazel build :package", - "postinstall": "./scripts/download_protoc.sh", "test": "protoc -I=src --ts_out=src test.proto && tsc && node ./dist/index" }, "dependencies": { diff --git a/examples/pure/scripts/download_protoc.sh b/examples/pure/scripts/download_protoc.sh deleted file mode 100755 index 5372c3e6..00000000 --- a/examples/pure/scripts/download_protoc.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -if [ -f ./node_modules/.bin/protoc ]; then - echo "protoc binary has already present. skipping downloading" - exit 0 -fi - -ASSET_URL=$(curl -u -s https://api.github.com/repos/protocolbuffers/protobuf/releases/latest | node ./scripts/pick_asset.js) -ASSET_NAME="protoc-$(date +%s)" -echo $ASSET_URL; - -curl -sSL $ASSET_URL > "/tmp/${ASSET_NAME}.zip" - -unzip "/tmp/${ASSET_NAME}.zip" -d "/tmp/${ASSET_NAME}-extracted" - -mv "/tmp/${ASSET_NAME}-extracted/bin/protoc" ./node_modules/.bin \ No newline at end of file diff --git a/examples/pure/scripts/pick_asset.js b/examples/pure/scripts/pick_asset.js deleted file mode 100644 index a175f11e..00000000 --- a/examples/pure/scripts/pick_asset.js +++ /dev/null @@ -1,53 +0,0 @@ -const os = require("os"); -const fs = require("fs"); - -const platform = os.platform(); -const arch = os.arch(); - -let build = `${platform}-${arch}`; -switch (platform) { - case "darwin": - if (arch === "x64") { - build = 'osx-x86_64' - } - break; - case "linux": - if (arch === "x64") { - build = 'linux-x86_64' - } else if (arch === "x32") { - build = 'linux-x86_32' - } else if (arch == "arm64") { - build = 'linux-aarch_64' - } - break; - case "win32": - if (arch === "x64") { - build = 'win64' - } else if (arch === "x86") { - build = 'win32' - } - break; -} - -const response = JSON.parse(fs.readFileSync(0).toString()); - -if ( response.message ) { - throw new Error(response.message); -} - -const releaseName = `protoc-${response.tag_name.replace(/v(.*?)/, "$1")}-${build}.zip` - -let downloadUrl; - -for (const asset of response.assets) { - if (asset.name == releaseName) { - downloadUrl = asset.browser_download_url; - break; - } -} - -if ( !downloadUrl ) { - throw new Error(`Can not find any release for the platform combination ${releaseName}`); -} - -console.log(downloadUrl); \ No newline at end of file diff --git a/index.bzl b/index.bzl index 24f6b66c..1bcc29db 100644 --- a/index.bzl +++ b/index.bzl @@ -1,112 +1,211 @@ -load("@rules_proto//proto:defs.bzl", "ProtoInfo") - -def _get_bin(): - # BEGIN-INTERNAL - return "//bin" - # END-INTERNAL - return "//protoc-gen-ts/bin:protoc-gen-ts" - -def _proto_path(proto): - """ - The proto path is not really a file path - It's the path to the proto that was seen when the descriptor file was generated. - """ - path = proto.path - root = proto.root.path - ws = proto.owner.workspace_root - if path.startswith(root): - path = path[len(root):] - if path.startswith("/"): - path = path[1:] - if path.startswith(ws): - path = path[len(ws):] - if path.startswith("/"): - path = path[1:] - return path - -def _ts_proto_library(ctx): - - transitive_descriptors = [] - direct_sources = [] - - for target in ctx.attr.deps: - if ProtoInfo not in target: - fail("All targets in the deps attribute should be proto_library target.") - else: - info = target[ProtoInfo] - transitive_descriptors.extend(info.transitive_descriptor_sets.to_list()) - direct_sources.extend(info.direct_sources) - - ts_outputs = [] - - - for proto in direct_sources: - normalizedProtoName = proto.path.replace(ctx.label.package, "").lstrip("/")[:-len(proto.extension) - 1] - ts_outputs.append(ctx.actions.declare_file("%s.ts" % (normalizedProtoName))) - - protoc_args = ctx.actions.args() - - protoc_args.add("--plugin=protoc-gen-ts=%s" % ( ctx.executable.protoc_gen_ts_bin.path )) - - protoc_args.add("--ts_out=%s" % (ctx.bin_dir.path)) - - protoc_args.add("--descriptor_set_in=%s" % (":".join([desc.path for desc in transitive_descriptors]))) - - env = dict() - - protoc_args.add("--ts_opt=grpc_package=%s" % ctx.attr.grpc_package_name) - - if ctx.attr.experimental_features: - protoc_args.add("--ts_opt=unary_rpc_promise") - - protoc_args.add_all(direct_sources) - - ctx.actions.run( - inputs = direct_sources + transitive_descriptors, - tools = ctx.files.protoc_gen_ts_bin, - executable = ctx.executable._protoc, - outputs = ts_outputs, - arguments = [protoc_args], - env = env, - progress_message = "Generating Protocol Buffers for Typescript %s" % ctx.label, - ) - - return [ - DefaultInfo(files = depset(ts_outputs)) - ] - - - - - -ts_proto_library = rule( - implementation = _ts_proto_library, - attrs = { - "deps": attr.label_list( - doc = "List of proto_library targets.", - providers = [ProtoInfo], - mandatory = True - ), - "experimental_features": attr.bool( - doc = "Enable experimental features.", - default = False - ), - "grpc_package_name": attr.string( - doc = "Configures name of the grpc package to use. '@grpc/grpc-js' or 'grpc'", - default = "@grpc/grpc-js" - ), - "protoc_gen_ts_bin": attr.label( - executable = True, - cfg = "exec", - default = _get_bin(), - ), - "_protoc": attr.label( - executable = True, - cfg = "exec", - default = ( - "@com_google_protobuf//:protoc" - ), - ), - - } -) +load("@rules_proto//proto:defs.bzl", "ProtoInfo") +load("@build_bazel_rules_nodejs//third_party/github.com/bazelbuild/bazel-skylib:lib/paths.bzl", "paths") + +TSProtoOutputInfo = provider( + fields = { + "deps_outputs": "The transitive typescript files.", + "outputs": "The transitive typescript files.", + }, +) + +def _get_inputs(target): + inputs = [] + inputs += target[ProtoInfo].direct_sources + inputs += target[ProtoInfo].transitive_descriptor_sets.to_list() + return inputs + +def _get_outputs(target, ctx): + outputs = [] + for source in target[ProtoInfo].direct_sources: + # test.proto -> test + name = source.path.replace(ctx.label.package, "").lstrip("/")[:-len(source.extension) - 1] + output = ctx.actions.declare_file("%s.ts" % (name)) + outputs.append(output) + return outputs + +def _proto_path_infos(proto_info, provided_sources = []): + """Returns sequence of `ProtoFileInfo` for `proto_info`'s direct sources. + Files that are both in `proto_info`'s direct sources and in + `provided_sources` are skipped. This is useful, e.g., for well-known + protos that are already provided by the Protobuf runtime. + Args: + proto_info: An instance of `ProtoInfo`. + provided_sources: Optional. A sequence of files to ignore. + Usually, these files are already provided by the + Protocol Buffer runtime (e.g. Well-Known protos). + Returns: A sequence of `ProtoFileInfo` containing information about + `proto_info`'s direct sources. + """ + + source_root = proto_info.proto_source_root + if "." == source_root: + return [struct(file = src, import_path = src.path) for src in proto_info.direct_sources] + + offset = len(source_root) + 1 # + '/'. + + infos = [] + for src in proto_info.direct_sources: + infos.append(struct(file = src, import_path = src.path[offset:])) + + return infos + +def _as_path(path, is_windows_host): + if is_windows_host: + return path.replace("/", "\\") + else: + return path + +def _ts_proto_library_aspect(target, ctx): + + is_windows_host = ctx.configuration.host_path_separator == ";" + + args = ctx.actions.args() + + # Output and Plugin path + args.add(_as_path(ctx.executable._protoc_gen_ts_bin.path, is_windows_host), format = "--plugin=protoc-gen-ts=%s") + + args.add(paths.join(ctx.bin_dir.path, ctx.label.workspace_root), format = "--ts_out=%s") + + # Set in descriptors + descriptor_sets_paths = [desc.path for desc in target[ProtoInfo].transitive_descriptor_sets.to_list()] + + args.add_joined("--descriptor_set_in", descriptor_sets_paths, join_with = ctx.configuration.host_path_separator) + + # Options + args.add(ctx.attr.grpc_package_name, format = "--ts_opt=grpc_package=%s") + + if ctx.attr.experimental_features == "true": + args.add("--ts_opt=unary_rpc_promise") + + # Direct sources + for f in _proto_path_infos(target[ProtoInfo]): + args.add(f.import_path) + + outputs = _get_outputs(target, ctx) + + ctx.actions.run( + inputs = depset( + direct = _get_inputs(target), + transitive = [depset(ctx.files._well_known_protos)] + ), + tools = [ctx.executable._protoc_gen_ts_bin], + executable = ctx.executable._protoc, + outputs = outputs, + arguments = [args], + progress_message = "Generating Protocol Buffers for Typescript %s" % ctx.label, + ) + + return TSProtoOutputInfo( + outputs = outputs + ) + +ts_proto_library_aspect = aspect( + implementation = _ts_proto_library_aspect, + attr_aspects = ["deps"], + provides = [TSProtoOutputInfo], + required_providers = [ProtoInfo], + attrs = { + "experimental_features": attr.string( + doc = "Enable experimental features.", + default = "false", + values = ["true", "false"] + ), + "grpc_package_name": attr.string( + doc = "Configures name of the grpc package to use. '@grpc/grpc-js' or 'grpc'", + default = "@grpc/grpc-js", + values = ["@grpc/grpc-js", "grpc"] + ), + "_protoc": attr.label( + cfg = "host", + executable = True, + allow_single_file = True, + default = ( + "@com_google_protobuf//:protoc" + ), + ), + "_protoc_gen_ts_bin": attr.label( + executable = True, + cfg = "host", + default = ( + "//protoc-gen-ts/bin:protoc-gen-ts" + ), + ), + "_well_known_protos": attr.label( + allow_files = True, + default = ( + "@com_google_protobuf//:well_known_protos" + ), + ), + }, +) + + + + + +def _ts_proto_library(ctx): + outputs = [] + + for target in ctx.attr.deps: + # if ProtoInfo not in target: + # fail("All targets in the deps attribute should be proto_library target.") + # else: + # info = target[ProtoInfo] + # transitive_descriptors.extend(info.transitive_descriptor_sets.to_list()) + outputs.extend(target[TSProtoOutputInfo].outputs) + + return [ + DefaultInfo(files = depset(outputs)) + ] + + +ts_proto_library_ = rule( + implementation = _ts_proto_library, + attrs = { + "deps": attr.label_list( + doc = "List of proto_library targets.", + providers = [ProtoInfo], + aspects = [ts_proto_library_aspect], + mandatory = True + ), + "experimental_features": attr.string( + doc = "Enable experimental features.", + default = "false" + ), + "grpc_package_name": attr.string( + doc = "Configures name of the grpc package to use. '@grpc/grpc-js' or 'grpc'", + default = "@grpc/grpc-js" + ), + "_protoc_gen_ts_bin": attr.label( + executable = True, + cfg = "host", + default = ( + "//protoc-gen-ts/bin:protoc-gen-ts" + ), + ), + "_protoc": attr.label( + cfg = "host", + executable = True, + allow_single_file = True, + default = ( + "@com_google_protobuf//:protoc" + ), + ), + "_well_known_protos": attr.label( + allow_files = True, + default = ( + "@com_google_protobuf//:well_known_protos" + ), + ), + } +) + + +def ts_proto_library(name, **kwargs): + experimental_features = kwargs.pop("experimental_features", False) + + ts_proto_library_( + name = name, + experimental_features = "true" if experimental_features else "false", + **kwargs + ) diff --git a/package.json b/package.json index 2a225d01..0cfc3f46 100644 --- a/package.json +++ b/package.json @@ -13,12 +13,12 @@ "url": "https://www.buymeacoffee.com/thesayyn" }, "scripts": { - "test": "bazel build //test:codegen //test:codegen_experimental && node ./scripts/sync_generated_protos.js test && bazel test //test/... --test_output=errors", + "test": "bazel build //test:codegen //test:codegen_experimental //test/conformance:codegen && node ./scripts/sync_generated_protos.js test && bazel test //test/... --test_output=errors", "release": "bazel run //:package.publish -- --access public --tag latest --registry https://registry.npmjs.org", "codegen": "node ./scripts/sync_compiler_protos.js && bazel build //src:codegen && node ./scripts/sync_generated_protos.js src/compiler && bazel build //src:compiler && node ./scripts/sync_generated_protos.js src/compiler js" }, "bin": { - "protoc-gen-ts": "./bin/protoc-gen-ts" + "protoc-gen-ts": "./bin/protoc-gen-ts.js" }, "peerDependencies": { "google-protobuf": "^3.13.0", diff --git a/protoc-gen-ts/bin/BUILD.bazel b/protoc-gen-ts/bin/BUILD.bazel new file mode 100644 index 00000000..b9742b73 --- /dev/null +++ b/protoc-gen-ts/bin/BUILD.bazel @@ -0,0 +1,5 @@ +alias( + name = "protoc-gen-ts", + actual = "//bin", + visibility = ["//visibility:public"] +) diff --git a/scripts/sync_generated_protos.js b/scripts/sync_generated_protos.js index df092955..8a6c787a 100644 --- a/scripts/sync_generated_protos.js +++ b/scripts/sync_generated_protos.js @@ -1,10 +1,10 @@ const fs = require("fs"); +const path = require("path"); const [, , source, dtsAndJs, dest] = process.argv; -const sourceDir = `./bazel-bin/${source}`; -const destDir = `./${dest || source}`; - +const sourceDir = path.join(".", "bazel-bin", source); +const destDir = path.join(".", dest || source); let check = (object) => object.endsWith(".ts") && !object.endsWith(".spec.ts") && !object.endsWith(".d.ts"); @@ -15,8 +15,8 @@ if ( dtsAndJs ) { function sync(sourceDir, destDir) { const objects = fs.readdirSync(sourceDir); for (const object of objects) { - const sourcePath = `${sourceDir}/${object}`; - const targetPath = `${destDir}/${object}`; + const sourcePath = path.join(sourceDir, object); + const targetPath = path.join(destDir, object); if (fs.statSync(sourcePath).isDirectory() && !/.runfiles/.test(sourcePath)) { sync(sourcePath, targetPath) } else if (check(object)) { diff --git a/test/conformance/BUILD.bazel b/test/conformance/BUILD.bazel new file mode 100644 index 00000000..ae03715b --- /dev/null +++ b/test/conformance/BUILD.bazel @@ -0,0 +1,43 @@ + +load("@npm//@bazel/jasmine:index.bzl", "jasmine_node_test") +load("@npm//@bazel/typescript:index.bzl", "ts_project") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("//:index.bzl", "ts_proto_library") + +proto_library( + name = "protos", + srcs = glob(["**/*.proto"]) +) + +ts_proto_library( + name = "codegen", + deps = [ + ":protos", + ], +) + +ts_project( + name = "test_lib", + srcs = glob(["**/*.ts"]), + tsconfig = { + "compilerOptions": { + "target": "ES2020", + "module": "CommonJS" + }, + }, + deps = [ + "@npm//@types/jasmine", + "@npm//@types/node", + "@npm//google-protobuf", + "@npm//@grpc/grpc-js", + ], +) + +jasmine_node_test( + name = "test", + data = glob(["**/*.bin"]), + tags = ["no-windows-latest"], + deps = [ + ":test_lib", + ], +) diff --git a/test/conformance/map/map.spec.ts b/test/conformance/map/map.spec.ts index d4ec4290..4f6a441f 100644 --- a/test/conformance/map/map.spec.ts +++ b/test/conformance/map/map.spec.ts @@ -1,8 +1,9 @@ import * as fs from "fs"; +import * as path from "path"; import { maps } from "./maps/map"; describe("maps", () => { - const bin = fs.readFileSync(__dirname + "/map.bin"); + const bin = fs.readFileSync(path.join(__dirname, "map.bin")); it("should be able to deserialize from go", () => { const tags = maps.Tags.deserialize(bin); expect(tags.toObject()).toEqual({ diff --git a/test/conformance/packedproto2/packedproto2.spec.ts b/test/conformance/packedproto2/packedproto2.spec.ts index bb4bb799..607122bc 100644 --- a/test/conformance/packedproto2/packedproto2.spec.ts +++ b/test/conformance/packedproto2/packedproto2.spec.ts @@ -1,8 +1,9 @@ import * as fs from "fs"; -import {WebMessageInfo} from "./packedproto2"; +import * as path from "path"; +import { WebMessageInfo } from "./packedproto2"; -describe("packed proto 2", ()=> { - const bin = fs.readFileSync(__dirname + "/packedproto2.bin"); +describe("packed proto 2", () => { + const bin = fs.readFileSync(path.join(__dirname, "packedproto2.bin")); it("should not pack scanLengths", () => { const info = WebMessageInfo.deserialize(bin); expect(info.message.imageMessage.scanLengths).toEqual([5453]);