diff --git a/README.md b/README.md index f532a92..8ad8709 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,15 @@ deploystream Track the progress of feature development across repositories gathering information from a multitude of sources. +Getting started +--------------- + +You'll need to create a ``github_auth.py`` file with your TOKEN information from +github. + +Do this by running ``get_github_token.py`` and follow the steps. That will +enable access to github. Alter the configuration of GITHUB_CONFIG in +local_settings.py to point to the repo of choice. Running the server locally -------------------------- @@ -36,3 +45,4 @@ Defining which providers to use Configuring a provider ~~~~~~~~~~~~~~~~~~~~~~ + diff --git a/TODO.txt b/TODO.txt index 9c80044..c47862e 100644 --- a/TODO.txt +++ b/TODO.txt @@ -1,7 +1,6 @@ Alex ==== - - Finish Git plugin - to make it look across a group of repos - Code that handles the creation of hierarchical branch trees - git plugin code to use hierarchical trees to determine merge statuses @@ -11,8 +10,6 @@ Unclaimed - Code to handle multiple plugins - we'll need to merge data between them I suspect. - Code that passes per-plugin settings to the plugins. - There's inconsistency in the naming between providers and plugins. Which do you prefer? - - Creation of jenkins plugin. - - Creation of CI github plugin - Addition of normalised Result class for buildinfo - Normalised status for a feature - Normalised feature type diff --git a/deploystream/apps/feature/lib.py b/deploystream/apps/feature/lib.py index 2669c11..1a03221 100644 --- a/deploystream/apps/feature/lib.py +++ b/deploystream/apps/feature/lib.py @@ -1,5 +1,3 @@ -import itertools - from deploystream import app from deploystream.providers import ( PLANNING_PLUGINS, SOURCE_CODE_PLUGINS, BUILD_INFO_PLUGINS @@ -14,8 +12,6 @@ def get_all_features(): for plugin in PLANNING_PLUGINS: features += plugin.get_features() - # features = itertools.chain([plugin.get_features() for plugin in PLANNING_PLUGINS]) - return features diff --git a/deploystream/providers/git_plugin/__init__.py b/deploystream/providers/git_plugin/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/deploystream/providers/git_plugin/plugin.py b/deploystream/providers/git_plugin/plugin.py deleted file mode 100644 index f547904..0000000 --- a/deploystream/providers/git_plugin/plugin.py +++ /dev/null @@ -1,66 +0,0 @@ -import git - - -class GitPlugin(object): - - def __init__(self, **configuration): - pass - - def get_repo_branches_involved(self, feature_id): - """ - Get all the repo branches involved. - - For each repository in each repo location defined in configuration, - call ``get_branches_involved`` and return a list of tuples. - - :returns: - A list of iterables containing at position: - - 0: repo name - 1: branch name - 2: latest commit - """ - return [] # no info available for now - - def get_branches_involved(self, repo_location, feature_id): - """ - Get the set of brances involved in the given repo and feature. - - :param repo_location: - The location of the repository to search for branches. - - :param feature_id: - The id of the feature to look for in branches. - - :returns: - A list of iterables containing at position: - - 0: branch name - 1: latest commit - """ - repo = git.Repo("{repo_location}/.git" - .format(repo_location=repo_location)) - remote = git.remote.Remote(repo, 'origin') - affected = [] - for remote_ref in remote.refs: - if feature_id in remote_ref.remote_head: - affected.append((remote_ref.remote_head, - str(remote_ref.commit))) - - return affected - - def set_merged_status(self, repo_name, hierarchy_tree, **kwargs): - """ - Set the merged status of the given tree in the repo. - - :param repo_name: - The name of the repository to search for branches. - - :param hierarchy_tree: - A tree-like object. - - TODO: - more definition here... (traversing, etc) - - """ - pass diff --git a/deploystream/providers/git_provider/__init__.py b/deploystream/providers/git_provider/__init__.py new file mode 100644 index 0000000..72aae72 --- /dev/null +++ b/deploystream/providers/git_provider/__init__.py @@ -0,0 +1,115 @@ +import os +from os.path import join, exists +import re + +import git + + +class GitProvider(object): + + def __init__(self, code_dir='.', + feature_breakup_regex='', + branch_finder_template=''): + """ + Create a GitProvider. + + :param code_dir: + The filesystem path to the directory within which all repositories + live that are to be queried. + + :param feature_breakup_regex: + A regular expression to be used to breakup feature ids into + understandable parts. The regex should use named groups to be + of use to the ``branch_finder_template``. + + eg. "(?P[a-zA-Z]+)-?(?P[0-9]+)" + + :param branch_finder_template: + A template regular expression with named gaps to be filled by the + outcome of breaking up the feature. + + eg. ".*{id}.*" + + """ + self.code_dir = code_dir + self.feature_breakup_regex = feature_breakup_regex + self.branch_finder_template = branch_finder_template + + def get_repo_branches_involved(self, feature_id): + """ + Get all the repo branches involved. + + For each repository in each repo location defined in configuration, + call ``get_branches_involved`` and return a list of tuples. + + :returns: + A list of iterables containing at position: + + 0: repo name + 1: branch name + 2: latest commit + """ + # Every folder inside self.code_dir that is a git repo will be looked + # at + repo_branches = [] + for repo_name in os.listdir(self.code_dir): + repo_location = join(self.code_dir, repo_name) + if exists(join(repo_location, ".git")): + branches = self.get_branches_involved(repo_location, + feature_id) + repo_branches.extend([ + (repo_name, ) + branch for branch in branches]) + return repo_branches + + def _get_feature_breakdown(self, feature_id): + """ + Break up the feature_id using the regex in configuration. + """ + match = re.search(self.feature_breakup_regex, feature_id) + if match: + return match.groupdict() + + def get_branches_involved(self, repo_location, feature_id): + """ + Get the set of brances involved in the given repo and feature. + + :param repo_location: + The location of the repository to search for branches. + + :param feature_id: + The id of the feature to look for in branches. + + :returns: + A list of iterables containing at position: + + 0: branch name + 1: latest commit + """ + repo = git.Repo("{repo_location}/.git" + .format(repo_location=repo_location)) + remote = git.remote.Remote(repo, 'origin') + affected = [] + feature_breakup = self._get_feature_breakdown(feature_id) + regex = self.branch_finder_template.format(**feature_breakup) + for remote_ref in remote.refs: + if re.search(regex, remote_ref.remote_head): + affected.append((remote_ref.remote_head, + str(remote_ref.commit))) + + return affected + + def set_merged_status(self, repo_name, hierarchy_tree, **kwargs): + """ + Set the merged status of the given tree in the repo. + + :param repo_name: + The name of the repository to search for branches. + + :param hierarchy_tree: + A tree-like object. + + TODO: + more definition here... (traversing, etc) + + """ + pass diff --git a/deploystream/settings.py b/deploystream/settings.py index f5b4df2..b8d2a07 100644 --- a/deploystream/settings.py +++ b/deploystream/settings.py @@ -1,8 +1,8 @@ -from local_settings import GITHUB_CONFIG +from local_settings import GITHUB_CONFIG, GIT_CONFIG SOURCE_CODE_PLUGINS = [ - ('deploystream.providers.git_plugin.plugin.GitPlugin', {}), + ('deploystream.providers.git_provider.GitProvider', GIT_CONFIG), ] PLANNING_PLUGINS = [ diff --git a/local_settings.py b/local_settings.py index 92bb3e2..12e505a 100644 --- a/local_settings.py +++ b/local_settings.py @@ -5,3 +5,10 @@ 'repositories': [('pretenders', 'deploystream')], 'token': TOKEN, } + + +GIT_CONFIG = { + 'code_dir': None, + 'feature_breakup_regex': "(?P[a-zA-Z]+)-?(?P[0-9]+)", + 'branch_finder_template': ".*(?i){project}.*" +} diff --git a/tests/test_providers/test_git_provider.py b/tests/test_providers/test_git_provider.py new file mode 100644 index 0000000..e00d665 --- /dev/null +++ b/tests/test_providers/test_git_provider.py @@ -0,0 +1,56 @@ +import os +from os.path import join, exists, dirname + +from nose.tools import assert_equal, with_setup + +from deploystream.providers.git_provider import GitProvider + +DUMMY_CODE_DIR = join(dirname(__file__), 'data') + + +def ensure_dummy_clone_available(): + """ + Check that we have access to pretenders' dummyrepo + """ + if not os.path.exists(DUMMY_CODE_DIR): + os.mkdir(DUMMY_CODE_DIR) + folder_name = join(DUMMY_CODE_DIR, 'dummyrepo') + if not exists(folder_name): + os.system('git clone git+ssh://git@github.com/pretenders/dummyrepo {0}' + .format(folder_name)) + else: + os.system('git --git-dir={0} fetch'.format(folder_name)) + + +@with_setup(ensure_dummy_clone_available) +def test_git_plugin_finds_branches_across_repos(): + """ + Test that the GitPlugin finds branches in repos in the dir given. + + Clone the dummyrepo into the data folder if not already there. + + The data in this test is found by looking at the dummyrepo and getting + the branch names and latest commit of any branches that match "FeAtUrE". + """ + provider = GitProvider(code_dir=DUMMY_CODE_DIR, + feature_breakup_regex="(?P[a-zA-Z]+)-?(?P[0-9]+)", + branch_finder_template=".*(?i){project}.*") + branches = provider.get_repo_branches_involved('FeAtUrE-99') + + assert_equal([ + ('dummyrepo', 'my/feature_branch', + 'cf9130d3c07b061a88569153f10a7c7779338cfa'), + ], branches) + + +def test_git_plugin_feature_breakup_regex(): + """ + Test that GitPlugin breaks up feature ids into appropriate parts. + """ + provider = GitProvider( + feature_breakup_regex="(?P[a-zA-Z]+)-?(?P[0-9]+)") + for feature, expected in [ + ('DD-334', {'id': '334', 'project':'DD'}), + ('DD334', {'id': '334', 'project':'DD'}), + ]: + assert_equal(provider._get_feature_breakdown('DD-334'), expected) diff --git a/tests/test_providers/test_interfaces.py b/tests/test_providers/test_interfaces.py index 0732f8d..128684a 100644 --- a/tests/test_providers/test_interfaces.py +++ b/tests/test_providers/test_interfaces.py @@ -12,6 +12,7 @@ def test_implements_source_control_plugin(self): class MyPlugin(object): def get_repo_branches_involved(self, feature_id): pass + def set_merged_status(self, repo_name, hierarchy_tree): pass @@ -21,6 +22,7 @@ def test_does_not_implement_source_control_plugin(self): class MyPlugin(object): def get_repo_branches_involved(self, feature_id): pass + def set_merged_status(self, repo_name): pass @@ -49,6 +51,7 @@ def test_implements_planning_plugin(self): class MyPlugin(object): def get_features(self, **filters): pass + def get_feature_info(self, feature_id): pass