Skip to content
This repository has been archived by the owner on Mar 3, 2023. It is now read-only.

Switch to using external pex skylark rules #1114

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
workspace(name = "com_twitter_heron")

# versions shared across artifacts that should be upgraded together
aws_version = "1.11.14"
curator_version = "2.9.0"
Expand Down Expand Up @@ -286,3 +288,11 @@ maven_jar(
name = "org_mockito_mockito_all",
artifact = "org.mockito:mockito-all:1.10.19",
)

git_repository(
name = "io_bazel_rules_pex",
remote = "https://github.com/benley/bazel_rules_pex.git",
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can these rules get a better home either with the bazel or pex projects? This is fine for testing but we don't want heron to depend on an individual's git project.

Also this will fail in Twitters internal CI build since it's a closed environment without access to github, only a maven repo mirror and a pipy mirror. Could this code be distributed by either of those means somehow?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Better home: yes, I hope so. I haven't directly requested a github project from bazelbuild, but moving it out of my personal account does seem appropriate before merging this change.

Internal CI: it would work equally well to get these rules from a .zip or .tgz archive using a http_archive rule, or even from a path on disk using local_repository if that turns out to be easier. In theory we could publish a pypi project containing the bazel rules, though it would be a bit awkward. I'm not very familiar with Maven - can it be used to deliver arbitrary files? If so, I imagine that could be made to work as well.

If none of that is practical, we could just vendor things back into this repo rather than fetching it externally as part of the build.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed pypi would be awkward, but maven does have support for tar.gz artifacts. See http://stackoverflow.com/questions/32088885/install-tar-gz-file-in-maven-repository

tag = "0.2",
)
load("@io_bazel_rules_pex//pex:pex_rules.bzl", "pex_repositories")
pex_repositories()
2 changes: 0 additions & 2 deletions heron/cli/src/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_library", "pex_binary")

pex_library(
name = "heron-lib",
srcs = glob(
Expand Down
3 changes: 1 addition & 2 deletions heron/cli/tests/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
load("/tools/rules/pex_rules", "pex_test")

pex_test(
pex_pytest(
name = "opts_unittest",
srcs = ["opts_unittest.py"],
deps = [
Expand Down
2 changes: 0 additions & 2 deletions heron/common/src/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_library")

pex_library(
name = 'common',
srcs = glob(['**/*.py']),
Expand Down
2 changes: 0 additions & 2 deletions heron/executor/src/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_library", "pex_binary")

pex_binary(
name = "heron-executor",
srcs = ["heron-executor.py"],
Expand Down
3 changes: 1 addition & 2 deletions heron/executor/tests/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
load("/tools/rules/pex_rules", "pex_test")

pex_test(
pex_pytest(
name = "aurora_unittest",
srcs = ["aurora_unittest.py"],
reqs = [
Expand Down
2 changes: 0 additions & 2 deletions heron/explorer/src/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_library", "pex_binary")

pex_library(
name = "heron-explorer-lib",
srcs = glob(["**/*.py"],),
Expand Down
3 changes: 1 addition & 2 deletions heron/explorer/tests/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
load("/tools/rules/pex_rules", "pex_test")

pex_test(
pex_pytest(
name = "explorer_unittest",
srcs = ["explorer_unittest.py"],
deps = [
Expand Down
1 change: 0 additions & 1 deletion heron/proto/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/genproto", "proto_library")
load("/tools/rules/pex_rules", "pex_library")

proto_library(
name = "proto_common",
Expand Down
4 changes: 1 addition & 3 deletions heron/shell/src/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_library", "pex_binary")

pex_library(
name = "heron-shell-lib",
srcs = glob(
Expand All @@ -20,7 +18,7 @@ pex_binary(
deps = [
":heron-shell-lib",
],
resources = [
data = [
"//heron/shell/assets:assets",
],
)
2 changes: 0 additions & 2 deletions heron/statemgrs/src/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_library")

pex_library(
name = 'zk-statemgr-py',
srcs = glob(['**/*.py']),
Expand Down
2 changes: 0 additions & 2 deletions heron/tracker/src/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_library", "pex_binary")

pex_library(
name = "heron-tracker-lib",
srcs = glob(
Expand Down
14 changes: 7 additions & 7 deletions heron/tracker/tests/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
load("/tools/rules/pex_rules", "pex_test")
load("/tools/rules/pex_rules", "pex_library")

pex_library(
name = 'mock_proto',
Expand All @@ -9,10 +7,11 @@ pex_library(
],
)

pex_test(
pex_pytest(
name = "topology_helpers_unittest",
srcs = ["topology_helpers_unittest.py"],
deps = [
":mock_proto",
"//heron/tracker/src/python:heron-tracker-lib",
],
reqs = [
Expand All @@ -24,10 +23,11 @@ pex_test(
size = "small",
)

pex_test(
pex_pytest(
name = "topology_unittest",
srcs = ["topology_unittest.py"],
deps = [
":mock_proto",
"//heron/tracker/src/python:heron-tracker-lib",
],
reqs = [
Expand All @@ -39,7 +39,7 @@ pex_test(
size = "small",
)

pex_test(
pex_pytest(
name = "query_operator_unittest",
srcs = ["query_operator_unittest.py"],
deps = [
Expand All @@ -54,7 +54,7 @@ pex_test(
size = "small",
)

pex_test(
pex_pytest(
name = "query_unittest",
srcs = ["query_unittest.py"],
deps = [
Expand All @@ -69,7 +69,7 @@ pex_test(
size = "small",
)

pex_test(
pex_pytest(
name = "tracker_unittest",
srcs = ["tracker_unittest.py"],
deps = [
Expand Down
2 changes: 1 addition & 1 deletion heron/tracker/tests/python/topology_helpers_unittest.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from heron.common.src.python import constants
from heron.tracker.src.python import topology_helpers
from mock_proto import MockProto
from heron.tracker.tests.python.mock_proto import MockProto

class TopologyHelpersTest(unittest.TestCase):
def setUp(self):
Expand Down
2 changes: 1 addition & 1 deletion heron/tracker/tests/python/topology_unittest.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
# pylint: disable=missing-docstring
import unittest2 as unittest

from mock_proto import MockProto
from heron.tracker.tests.python.mock_proto import MockProto
from heron.tracker.src.python.topology import Topology

class TopologyTest(unittest.TestCase):
Expand Down
4 changes: 1 addition & 3 deletions heron/ui/src/python/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_library", "pex_binary")

pex_library(
name = "heron-ui-lib",
srcs = glob(
Expand All @@ -25,7 +23,7 @@ pex_binary(
deps = [
":heron-ui-lib",
],
resources = [
data = [
"//heron/ui/resources:templates",
"//heron/ui/resources:static",
],
Expand Down
3 changes: 0 additions & 3 deletions integration-test/src/python/http_server/BUILD
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_binary")

pex_binary(
name = "http-server",
srcs = [
"main.py"
],
main = "main.py",
reqs = [
"tornado==4.0.2",
],
Expand Down
5 changes: 1 addition & 4 deletions integration-test/src/python/local_test_runner/BUILD
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_binary")

pex_binary(
name = "local-test-runner",
srcs = [
"main.py"
],
main = "main.py",
resources = [
data = [
"resources/test.conf",
],
reqs = ["argparse==1.4.0"],
Expand Down
5 changes: 1 addition & 4 deletions integration-test/src/python/test_runner/BUILD
Original file line number Diff line number Diff line change
@@ -1,14 +1,11 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_binary")

pex_binary(
name = "test-runner",
srcs = [
"main.py"
],
main = "main.py",
resources = [
data = [
"resources/test.conf",
],
reqs = ["argparse==1.4.0"],
Expand Down
2 changes: 0 additions & 2 deletions release/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package(default_visibility = ["//visibility:public"])

load("/tools/build_defs/pkg/pkg", "pkg_tar")

# load("/tools/rules/heron_client", "heron_client_files")
load("/tools/rules/heron_client", "heron_client_bin_files")
load("/tools/rules/heron_client", "heron_client_conf_files")
Expand Down
2 changes: 0 additions & 2 deletions scripts/centos5/BUILD
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
package(default_visibility = ["//visibility:public"])

load("/tools/rules/pex_rules", "pex_binary")

release_version = "unversioned"

filegroup(
Expand Down
19 changes: 0 additions & 19 deletions third_party/pex/BUILD

This file was deleted.

Loading