Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check tags in track and team repos #713

Merged
merged 1 commit into from
Jun 17, 2019
Merged

Check tags in track and team repos #713

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 18 additions & 9 deletions esrally/utils/git.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,15 +57,15 @@ def clone(src, remote):
def fetch(src, remote="origin"):
# Don't swallow output but silence git at least a bit... (--quiet)
if process.run_subprocess(
"git -C {0} fetch --prune --quiet {1}".format(src, remote)):
"git -C {0} fetch --prune --prune-tags --tags --quiet {1}".format(src, remote)):
raise exceptions.SupplyError("Could not fetch source tree from [%s]" % remote)


@probed
def checkout(src_dir, branch="master"):
if process.run_subprocess(
"git -C {0} checkout --quiet {1}".format(src_dir, branch)):
raise exceptions.SupplyError("Could not checkout branch [%s]. Do you have uncommitted changes?" % branch)
raise exceptions.SupplyError("Could not checkout [%s]. Do you have uncommitted changes?" % branch)


@probed
Expand All @@ -83,17 +83,17 @@ def pull(src_dir, remote="origin", branch="master"):

@probed
def pull_ts(src_dir, ts):
if process.run_subprocess(
"git -C {0} fetch --prune --quiet origin && git -C {0} checkout --quiet `git -C {0} rev-list -n 1 --before=\"{1}\" "
"--date=iso8601 origin/master`".format(src_dir, ts)):
raise exceptions.SupplyError("Could not fetch source tree for timestamped revision [%s]" % ts)
fetch(src_dir)
if process.run_subprocess("git -C {0} checkout --quiet `git -C {0} rev-list -n 1 --before=\"{1}\" "
"--date=iso8601 origin/master`".format(src_dir, ts)):
raise exceptions.SupplyError("Could not checkout source tree for timestamped revision [%s]" % ts)


@probed
def pull_revision(src_dir, revision):
if process.run_subprocess(
"git -C {0} fetch --prune --quiet origin && git -C {0} checkout --quiet {1}".format(src_dir, revision)):
raise exceptions.SupplyError("Could not fetch source tree for revision [%s]" % revision)
fetch(src_dir)
if process.run_subprocess("git -C {0} checkout --quiet {1}".format(src_dir, revision)):
raise exceptions.SupplyError("Could not checkout source tree for revision [%s]" % revision)


@probed
Expand All @@ -118,9 +118,18 @@ def branches(src_dir, remote=True):
"git -C {src} for-each-ref refs/heads/ --format='%(refname:short)'".format(src=src_dir)))


@probed
def tags(src_dir):
return _cleanup_tag_names(process.run_subprocess_with_output("git -C {0} tag".format(src_dir)))


def _cleanup_remote_branch_names(branch_names):
return [(b[b.index("/") + 1:]).strip() for b in branch_names if not b.endswith("/HEAD")]


def _cleanup_local_branch_names(branch_names):
return [b.strip() for b in branch_names if not b.endswith("HEAD")]


def _cleanup_tag_names(tag_names):
return [t.strip() for t in tag_names]
23 changes: 21 additions & 2 deletions esrally/utils/repo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,14 +79,33 @@ def update(self, distribution_version):
branch = versions.best_match(git.branches(self.repo_dir, remote=False), distribution_version)
if branch:
if git.current_branch(self.repo_dir) != branch:
self.logger.info("Checking out [%s] in [%s] for distribution version [%s].", branch, self.repo_dir, distribution_version)
self.logger.info("Checking out [%s] in [%s] for distribution version [%s].",
branch, self.repo_dir, distribution_version)
git.checkout(self.repo_dir, branch=branch)
else:
raise exceptions.SystemSetupError("Cannot find %s for distribution version %s" % (self.resource_name, distribution_version))
self.logger.info("No local branch found for distribution version [%s] in [%s]. Checking tags.",
distribution_version, self.repo_dir)
tag = self._find_matching_tag(distribution_version)
if tag:
self.logger.info("Checking out tag [%s] in [%s] for distribution version [%s].",
tag, self.repo_dir, distribution_version)
git.checkout(self.repo_dir, branch=tag)
else:
raise exceptions.SystemSetupError("Cannot find %s for distribution version %s"
% (self.resource_name, distribution_version))
except exceptions.SupplyError as e:
tb = sys.exc_info()[2]
raise exceptions.DataError("Cannot update %s in [%s] (%s)." % (self.resource_name, self.repo_dir, e.message)).with_traceback(tb)

def _find_matching_tag(self, distribution_version):
tags = git.tags(self.repo_dir)
for version in versions.versions(distribution_version):
# tags have a "v" prefix by convention.
tag_candidate = "v{}".format(version)
if tag_candidate in tags:
return tag_candidate
return None

def checkout(self, revision):
self.logger.info("Checking out revision [%s] in [%s].", revision, self.repo_dir)
git.checkout(self.repo_dir, revision)
42 changes: 32 additions & 10 deletions tests/utils/git_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,7 @@ def test_fetch_successful(self, run_subprocess_with_logging, run_subprocess):
run_subprocess_with_logging.return_value = 0
run_subprocess.return_value = False
git.fetch("/src", remote="my-origin")
run_subprocess.assert_called_with("git -C /src fetch --prune --quiet my-origin")
run_subprocess.assert_called_with("git -C /src fetch --prune --prune-tags --tags --quiet my-origin")

@mock.patch("esrally.utils.process.run_subprocess")
@mock.patch("esrally.utils.process.run_subprocess_with_logging")
Expand All @@ -85,7 +85,7 @@ def test_fetch_with_error(self, run_subprocess_with_logging, run_subprocess):
with self.assertRaises(exceptions.SupplyError) as ctx:
git.fetch("/src", remote="my-origin")
self.assertEqual("Could not fetch source tree from [my-origin]", ctx.exception.args[0])
run_subprocess.assert_called_with("git -C /src fetch --prune --quiet my-origin")
run_subprocess.assert_called_with("git -C /src fetch --prune --prune-tags --tags --quiet my-origin")

@mock.patch("esrally.utils.process.run_subprocess")
@mock.patch("esrally.utils.process.run_subprocess_with_logging")
Expand All @@ -102,7 +102,7 @@ def test_checkout_with_error(self, run_subprocess_with_logging, run_subprocess):
run_subprocess.return_value = True
with self.assertRaises(exceptions.SupplyError) as ctx:
git.checkout("/src", "feature-branch")
self.assertEqual("Could not checkout branch [feature-branch]. Do you have uncommitted changes?", ctx.exception.args[0])
self.assertEqual("Could not checkout [feature-branch]. Do you have uncommitted changes?", ctx.exception.args[0])
run_subprocess.assert_called_with("git -C /src checkout --quiet feature-branch")

@mock.patch("esrally.utils.process.run_subprocess")
Expand All @@ -124,7 +124,7 @@ def test_pull(self, run_subprocess_with_logging, run_subprocess):
run_subprocess.return_value = False
git.pull("/src", remote="my-origin", branch="feature-branch")
calls = [
mock.call("git -C /src fetch --prune --quiet my-origin"),
mock.call("git -C /src fetch --prune --prune-tags --tags --quiet my-origin"),
mock.call("git -C /src checkout --quiet feature-branch"),
mock.call("git -C /src rebase --quiet my-origin/feature-branch")
]
Expand All @@ -134,19 +134,24 @@ def test_pull(self, run_subprocess_with_logging, run_subprocess):
@mock.patch("esrally.utils.process.run_subprocess_with_logging")
def test_pull_ts(self, run_subprocess_with_logging, run_subprocess):
run_subprocess_with_logging.return_value = 0
run_subprocess.return_value = False
run_subprocess.side_effect = [False, False]
git.pull_ts("/src", "20160101T110000Z")
run_subprocess.assert_called_with(
"git -C /src fetch --prune --quiet origin && git -C /src checkout "
"--quiet `git -C /src rev-list -n 1 --before=\"20160101T110000Z\" --date=iso8601 origin/master`")
run_subprocess.has_calls([
mock.call("git -C /src fetch --prune --prune-tags --tags --quiet origin"),
mock.call("git -C /src checkout --quiet `git -C /src rev-list -n 1 --before=\"20160101T110000Z\" "
"--date=iso8601 origin/master`"),
])

@mock.patch("esrally.utils.process.run_subprocess")
@mock.patch("esrally.utils.process.run_subprocess_with_logging")
def test_pull_revision(self, run_subprocess_with_logging, run_subprocess):
run_subprocess_with_logging.return_value = 0
run_subprocess.return_value = False
run_subprocess.side_effect = [False, False]
git.pull_revision("/src", "3694a07")
run_subprocess.assert_called_with("git -C /src fetch --prune --quiet origin && git -C /src checkout --quiet 3694a07")
run_subprocess.has_calls([
mock.call("git -C /src fetch --prune --prune-tags --tags --quiet origin"),
mock.call("git -C /src checkout --quiet 3694a07"),
])

@mock.patch("esrally.utils.process.run_subprocess_with_output")
@mock.patch("esrally.utils.process.run_subprocess_with_logging")
Expand Down Expand Up @@ -177,3 +182,20 @@ def test_list_local_branches(self, run_subprocess_with_logging, run_subprocess):
" 5"]
self.assertEqual(["master", "5.0.0-alpha1", "5"], git.branches("/src", remote=False))
run_subprocess.assert_called_with("git -C /src for-each-ref refs/heads/ --format='%(refname:short)'")

@mock.patch("esrally.utils.process.run_subprocess_with_output")
@mock.patch("esrally.utils.process.run_subprocess_with_logging")
def test_list_tags_with_tags_present(self, run_subprocess_with_logging, run_subprocess):
run_subprocess_with_logging.return_value = 0
run_subprocess.return_value = [" v1",
" v2"]
self.assertEqual(["v1", "v2"], git.tags("/src"))
run_subprocess.assert_called_with("git -C /src tag")

@mock.patch("esrally.utils.process.run_subprocess_with_output")
@mock.patch("esrally.utils.process.run_subprocess_with_logging")
def test_list_tags_no_tags_available(self, run_subprocess_with_logging, run_subprocess):
run_subprocess_with_logging.return_value = 0
run_subprocess.return_value = ""
self.assertEqual([], git.tags("/src"))
run_subprocess.assert_called_with("git -C /src tag")
42 changes: 39 additions & 3 deletions tests/utils/repo_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,11 +183,40 @@ def test_updates_locally(self, curr_branch, rebase, checkout, branches, fetch, i

@mock.patch("esrally.utils.git.is_working_copy", autospec=True)
@mock.patch("esrally.utils.git.fetch", autospec=True)
@mock.patch("esrally.utils.git.tags", autospec=True)
@mock.patch("esrally.utils.git.branches", autospec=True)
@mock.patch("esrally.utils.git.checkout", autospec=True)
@mock.patch("esrally.utils.git.rebase")
@mock.patch("esrally.utils.git.current_branch")
def test_fallback_to_tags(self, curr_branch, rebase, checkout, branches, tags, fetch, is_working_copy):
curr_branch.return_value = "master"
branches.return_value = ["5", "master"]
tags.return_value = ["v1", "v1.7", "v2"]
is_working_copy.return_value = True

r = repo.RallyRepository(
remote_url=None,
root_dir="/rally-resources",
repo_name="unit-test",
resource_name="unittest-resources",
offline=False)

r.update(distribution_version="1.7.4")

branches.assert_called_with("/rally-resources/unit-test", remote=False)
self.assertEqual(0, rebase.call_count)
tags.assert_called_with("/rally-resources/unit-test")
checkout.assert_called_with("/rally-resources/unit-test", branch="v1.7")

@mock.patch("esrally.utils.git.is_working_copy", autospec=True)
@mock.patch("esrally.utils.git.fetch", autospec=True)
@mock.patch("esrally.utils.git.tags", autospec=True)
@mock.patch("esrally.utils.git.branches", autospec=True)
@mock.patch("esrally.utils.git.checkout")
@mock.patch("esrally.utils.git.rebase")
def test_does_not_update_unknown_branch_remotely(self, rebase, checkout, branches, fetch, is_working_copy):
def test_does_not_update_unknown_branch_remotely(self, rebase, checkout, branches, tags, fetch, is_working_copy):
branches.return_value = ["1", "2", "5", "master"]
tags.return_value = []
is_working_copy.return_value = True

r = repo.RallyRepository(
Expand All @@ -212,19 +241,23 @@ def test_does_not_update_unknown_branch_remotely(self, rebase, checkout, branche
]

branches.assert_has_calls(calls)
tags.assert_called_with("/rally-resources/unit-test")
self.assertEqual(0, checkout.call_count)
self.assertEqual(0, rebase.call_count)

@mock.patch("esrally.utils.git.is_working_copy", autospec=True)
@mock.patch("esrally.utils.git.fetch", autospec=True)
@mock.patch("esrally.utils.git.tags", autospec=True)
@mock.patch("esrally.utils.git.branches", autospec=True)
@mock.patch("esrally.utils.git.checkout", autospec=True)
@mock.patch("esrally.utils.git.rebase")
@mock.patch("esrally.utils.git.current_branch")
def test_does_not_update_unknown_branch_remotely_local_fallback(self, curr_branch, rebase, checkout, branches, fetch, is_working_copy):
def test_does_not_update_unknown_branch_remotely_local_fallback(self, curr_branch, rebase, checkout, branches, tags,
fetch, is_working_copy):
curr_branch.return_value = "master"
# we have only "master" remotely but a few more branches locally
branches.side_effect = ["5", ["1", "2", "5", "master"]]
tags.return_value = []
is_working_copy.return_value = True

r = repo.RallyRepository(
Expand All @@ -244,16 +277,19 @@ def test_does_not_update_unknown_branch_remotely_local_fallback(self, curr_branc
]

branches.assert_has_calls(calls)
self.assertEqual(0, tags.call_count)
checkout.assert_called_with("/rally-resources/unit-test", branch="1")
self.assertEqual(0, rebase.call_count)

@mock.patch("esrally.utils.git.is_working_copy", autospec=True)
@mock.patch("esrally.utils.git.fetch", autospec=True)
@mock.patch("esrally.utils.git.tags", autospec=True)
@mock.patch("esrally.utils.git.branches", autospec=True)
@mock.patch("esrally.utils.git.checkout")
@mock.patch("esrally.utils.git.rebase")
def test_does_not_update_unknown_branch_locally(self, rebase, checkout, branches, fetch, is_working_copy):
def test_does_not_update_unknown_branch_locally(self, rebase, checkout, branches, tags, fetch, is_working_copy):
branches.return_value = ["1", "2", "5", "master"]
tags.return_value = []
is_working_copy.return_value = True

r = repo.RallyRepository(
Expand Down