From c26b123b9fa79bc3b0389acfafb76e5d14256485 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Sat, 4 May 2019 13:18:06 -0400 Subject: [PATCH 1/3] Remove git-url-parse requirement Fixes #72 - replace giturlparse.parse with local implementation based on urllib.parse.urlparse - Strip tilde from owner for Bitbucket Server user repos - put src_dir and lock dir under directory named after the repo hostname - remove _get_lock_dir() --- gilt/config.py | 58 ++++++++++++++++++++++++++++++++++++++---------- requirements.txt | 1 - 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/gilt/config.py b/gilt/config.py index a170df8..102c5c0 100644 --- a/gilt/config.py +++ b/gilt/config.py @@ -24,11 +24,16 @@ import errno import os -import giturlparse import yaml from gilt import interpolation +try: + from urllib.parse import urlparse +except ImportError: + # Python 2 + from urlparse import urlparse + class ParseError(Exception): """ Error raised when a config can't be loaded properly. """ @@ -59,6 +64,37 @@ def config(filename): return [Config(**d) for d in _get_config_generator(filename)] +def _parse_repo_uri(uri, scm="git"): + """ + Construct and return a `ParsedRepo` object. + + :param uri: A SCM repository URI. + :return: ParsedRepo + """ + ParsedRepo = collections.namedtuple('ParsedRepo', [ + 'hostname', + 'owner', + 'name' + ]) + + scm_ext = "." + scm + o = urlparse(uri) + if not o.hostname: # scp-style "URI", so fake it + uri = "ssh://" + uri.replace(":/", "/").replace(":", "/") + o = urlparse(uri) + path = o.path + if path.endswith(scm_ext): + path = path[0:-len(scm_ext)] + try: + name, owner = path.rsplit("/", 2)[-1:-3:-1] + owner = owner.lstrip("~") + except ValueError: + # if repo is the root of the path, then there is no owner + owner = '' + + return ParsedRepo(o.hostname, owner, name) + + def _get_files_config(src_dir, files_list): """ Construct `FileConfig` object and return a list. @@ -85,9 +121,12 @@ def _get_config_generator(filename): """ for d in _get_config(filename): repo = d['git'] - parsedrepo = giturlparse.parse(repo) - name = '{}.{}'.format(parsedrepo.owner, parsedrepo.name) - src_dir = os.path.join(_get_clone_dir(), name) + parsedrepo = _parse_repo_uri(repo) + if parsedrepo.owner: + name = '{}.{}'.format(parsedrepo.owner, parsedrepo.name) + else: + name = parsedrepo.name + src_dir = os.path.join(_get_clone_dir(), parsedrepo.hostname, name) files = d.get('files') post_commands = d.get('post_commands', []) dst_dir = None @@ -95,7 +134,9 @@ def _get_config_generator(filename): dst_dir = _get_dst_dir(d['dst']) yield { 'git': repo, - 'lock_file': _get_lock_file(name), + 'lock_file': os.path.join( + _get_lock_dir(), parsedrepo.hostname, name + ), 'version': d['version'], 'name': name, 'src': src_dir, @@ -156,13 +197,6 @@ def _get_dst_dir(dst_dir): return os.path.join(wd, dst_dir) -def _get_lock_file(name): - """ Return the lock file for the given name. """ - return os.path.join( - _get_lock_dir(), - name, ) - - def _get_base_dir(): """ Return gilt's base working directory. """ return os.path.expanduser(BASE_WORKING_DIR) diff --git a/requirements.txt b/requirements.txt index 9364580..03709cb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,6 @@ click colorama fasteners -git-url-parse pbr PyYAML sh From 1a0a74921264fad7b243bbdd98740e9e1073dfc8 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Mon, 6 May 2019 12:49:04 -0400 Subject: [PATCH 2/3] Update tests for clone/lock files in hostname subdirectory --- test/test_config.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/test_config.py b/test/test_config.py index 11c9b34..eb2f3f6 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -38,9 +38,10 @@ def test_config(gilt_config_file): assert 'https://github.com/retr0h/ansible-etcd.git' == r.git assert 'master' == r.version assert 'retr0h.ansible-etcd' == r.name - assert (gilt_root, 'clone', 'retr0h.ansible-etcd') == os_split(r.src)[-3:] - assert (gilt_root, 'lock', - 'retr0h.ansible-etcd') == os_split(r.lock_file)[-3:] + assert (gilt_root, 'clone', 'github.com', + 'retr0h.ansible-etcd') == os_split(r.src)[-4:] + assert (gilt_root, 'lock', 'github.com', + 'retr0h.ansible-etcd') == os_split(r.lock_file)[-4:] assert ('roles', 'retr0h.ansible-etcd', '') == os_split(r.dst)[-3:] assert [] == r.files @@ -52,8 +53,9 @@ def test_config(gilt_config_file): assert r.dst is None f = r.files[0] - x = (gilt_root, 'clone', 'lorin.openstack-ansible-modules', '*_manage') - assert x == os_split(f.src)[-4:] + x = (gilt_root, 'clone', 'github.com', + 'lorin.openstack-ansible-modules', '*_manage') + assert x == os_split(f.src)[-5:] assert ('library', '') == os_split(f.dst)[-2:] From 744bfaea110acd610731295da5ee68e3dadf3f03 Mon Sep 17 00:00:00 2001 From: Philip Douglass Date: Mon, 6 May 2019 12:50:49 -0400 Subject: [PATCH 3/3] Add tests for repo uri testing --- test/test_config.py | 164 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) diff --git a/test/test_config.py b/test/test_config.py index eb2f3f6..c84cf6f 100644 --- a/test/test_config.py +++ b/test/test_config.py @@ -59,6 +59,170 @@ def test_config(gilt_config_file): assert ('library', '') == os_split(f.dst)[-2:] +gilt_repos = [ + ( + "example.com:/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "example.com:owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "example.com:repo.git", + {"name": "repo", "owner": "", "hostname": "example.com"}, + ), + ( + "git+https://example.com/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "git+ssh://example.com/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "git+ssh://git@git.example.com/~philip.sd6/test.repo.git", + { + "name": "test.repo", + "owner": "philip.sd6", + "hostname": "git.example.com", + }, + ), + ( + "git://example.com/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "http://example.com/owner/repo", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "http://example.com/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "http://example.com/repo", + {"name": "repo", "owner": "", "hostname": "example.com"}, + ), + ( + "http://example.com:29418/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "http://user@example.com/user/repo", + {"name": "repo", "owner": "user", "hostname": "example.com"}, + ), + ( + "http://user@example.com:29418/user/repo", + {"name": "repo", "owner": "user", "hostname": "example.com"}, + ), + ( + "https://example.com/git/scm/project/my-sample.repo.git", + { + "name": "my-sample.repo", + "owner": "project", + "hostname": "example.com", + }, + ), + ( + "https://example.com/git/scm/~philip.sd6/my-sample.repo.git", + { + "name": "my-sample.repo", + "owner": "philip.sd6", + "hostname": "example.com", + }, + ), + ( + "https://example.com/owner/repo", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "https://example.com/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "https://example.com/repo", + {"name": "repo", "owner": "", "hostname": "example.com"}, + ), + ( + "https://example.com:29418/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "https://github.com/sphinx-doc/sphinx.git", + {"name": "sphinx", "owner": "sphinx-doc", "hostname": "github.com"}, + ), + ( + "https://github.com/tterranigma/Stouts.openvpn", + { + "name": "Stouts.openvpn", + "owner": "tterranigma", + "hostname": "github.com", + }, + ), + ( + "https://github.com/tterranigma/Stouts.openvpn.git", + { + "name": "Stouts.openvpn", + "owner": "tterranigma", + "hostname": "github.com", + }, + ), + ( + "https://user@example.com/user/repo", + {"name": "repo", "owner": "user", "hostname": "example.com"}, + ), + ( + "https://user@example.com:29418/user/repo", + {"name": "repo", "owner": "user", "hostname": "example.com"}, + ), + ( + "rsync://example.com/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "ssh://example.com/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "ssh://example.com:29418/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "ssh://user@example.com/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "ssh://user@example.com:29418/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "user@example.com:/owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "user@example.com:owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "example.com"}, + ), + ( + "user@example.com:repo.git", + {"name": "repo", "owner": "", "hostname": "example.com"}, + ), + ( + "user@foo-example.com:owner/repo.git", + {"name": "repo", "owner": "owner", "hostname": "foo-example.com"}, + ), +] + + +@pytest.mark.parametrize('uri,expected', gilt_repos) +def test_config_repo(uri, expected): + parsedrepo = config._parse_repo_uri(uri) + assert parsedrepo.hostname == expected['hostname'] + assert parsedrepo.owner == expected['owner'] + assert parsedrepo.name == expected['name'] + + @pytest.fixture() def missing_git_key_data(): return [{'foo': 'https://github.com/retr0h/ansible-etcd.git'}]