Add gitshelf as a format and fetch certain repos

This adds support for gitshelf as a format for a list
of repositories. This also adds support for fetching
only repos that are listed in a list of repositories,
as specified by --import-repos and --format.

Closes ksdme#6 and ksdme#29

org_status/formatters/__init__.py

@@ -0,0 +1,15 @@
+class RepoListFormatter:
+    NAME = None
+
+    def convert_repo_list_to_format(self, repos):
+        raise NotImplementedError()
+
+
+def get_all_supported_formatters():
+    from org_status.formatters.gitman import GitManFormatter
+    from org_status.formatters.gitshelf import GitShelfFormatter
+
+    return (
+        GitManFormatter,
+        GitShelfFormatter,
+    )

org_status/formatters/gitman.py

@@ -0,0 +1,32 @@
+import yaml
+from giturlparse import parse
+
+from org_status.formatters import RepoListFormatter
+
+
+class GitManFormatter(RepoListFormatter):
+    NAME = 'gitman'
+
+    def convert_repo_list_to_format(self, repos):
+        yml_data = {'sources': []}
+
+        for repo in repos:
+            name = parse(repo.web_url).repo
+            yml_data['sources'].append({'name': name,
+                                        'repo': repo.web_url,
+                                        'rev': 'master'})
+
+        return yaml.dump(yml_data, default_flow_style=False)
+
+    def convert_format_to_repo_list(self, file_name):
+        repos = []
+
+        try:
+            with open(file_name, 'r') as file:
+                yml_data = yaml.load(file)
+
+                for repo in yml_data['sources']:
+                    repos.append([repo['name'], repo['repo']])
+                return repos
+        except FileNotFoundError:
+            print(f'unable to find file {file_name}')

org_status/formatters/gitshelf.py

@@ -0,0 +1,32 @@
+import yaml
+from giturlparse import parse
+
+from org_status.formatters import RepoListFormatter
+
+
+class GitShelfFormatter(RepoListFormatter):
+    NAME = 'gitshelf'
+
+    def convert_repo_list_to_format(self, repos):
+        yml_data = {'books': []}
+
+        for repo in repos:
+            name = parse(repo.web_url).repo
+            yml_data['books'].append({'book': name,
+                                      'git': repo.web_url,
+                                      'branch': 'master'})
+
+        return yaml.dump(yml_data, default_flow_style=False)
+
+    def convert_format_to_repo_list(self, file_name):
+        repos = []
+
+        try:
+            with open(file_name, 'r') as file:
+                yml_data = yaml.load(file)
+
+                for repo in yml_data['books']:
+                    repos.append([repo['book'], repo['git']])
+                return repos
+        except FileNotFoundError:
+            print(f'unable to find file {file_name}')

org_status/org_hosts/github.py

@@ -14,28 +14,29 @@ class GitHubOrg(OrgHost):
     HostName = 'github'
     StatusProvider = [TravisBuildStatus, AppVeyorStatus]
 
-    def __init__(self, token, group, **kargs):
+    def __init__(self, token, group, fetch, **kargs):
         super().__init__(**kargs)
 
         self._group = group
-        self._token = GitHubToken(token)
-        self._org = GitHubOrganization(self._token, self._group)
-
         self._status_provider = []
         for i in enumerate(self.StatusProvider):
             self._status_provider.append(self.StatusProvider[i[0]](self._group))
 
+        if fetch:
+            self._token = GitHubToken(token)
+            self._org = GitHubOrganization(self._token, self._group)
+
     @classmethod
     def get_host_status(cls):
         status = requests.get('https://status.github.com/api/status.json')
         status = json.loads(status.text)
         return status['status'] == 'good'
 
-    def process_repository(self, repo, branch='master'):
-        self.print_status(repo.web_url)
+    def process_repository(self, web_url, branch='master'):
+        self.print_status(web_url)
 
         # reliable enough?
-        repo_name = repo.web_url.split('/')[-1]
+        repo_name = web_url.split('/')[-1]
         repo_status = []
         for i in enumerate(self._status_provider):
             repo_status.append(self._status_provider[i[0]]
@@ -56,7 +57,7 @@ def process_repository(self, repo, branch='master'):
         elif Status.ERROR in repo_status:
             repo_status = Status.ERROR
 
-        return RepoStatus(repo.web_url, repo_status)
+        return RepoStatus(web_url, repo_status)
 
     @property
     def repositories(self):

org_status/org_hosts/gitlab.py

@@ -15,31 +15,32 @@ class GitLabOrg(OrgHost):
     HOST_STATUS_URL = ('https://api.status.io/1.0/status'
                        '/5b36dc6502d06804c08349f7')
 
-    def __init__(self, token, group, **kargs):
+    def __init__(self, token, group, fetch, **kargs):
         super().__init__(**kargs)
 
         self._group = group
-        self._token = GitLabPrivateToken(token)
-        self._org = GitLabOrganization(self._token, self._group)
-
         self._status_provider = self.StatusProvider(self._group)
 
+        if fetch:
+            self._token = GitLabPrivateToken(token)
+            self._org = GitLabOrganization(self._token, self._group)
+
     @classmethod
     def get_host_status(cls):
         status = requests.get(cls.HOST_STATUS_URL)
         status = json.loads(status.text)
         return status['result']['status_overall']['status'] == 'Operational'
 
-    def process_repository(self, repo, branch='master'):
-        self.print_status(repo.web_url)
+    def process_repository(self, web_url, branch='master'):
+        self.print_status(web_url)
 
         # reliable enough?
-        repo_name = '/'.join(repo.web_url.split('/')[4:])
+        repo_name = '/'.join(web_url.split('/')[4:])
         repo_status = self._status_provider.get_status(repo_name,
                                                        self.HostName,
                                                        branch=branch)
 
-        return RepoStatus(repo.web_url, repo_status)
+        return RepoStatus(web_url, repo_status)
 
     @property
     def repositories(self):

org_status/org_status.py

@@ -6,7 +6,7 @@
 
 from org_status.status_providers import Status, get_supported_status_providers
 from org_status.org_hosts import get_all_supported_hosts
-from org_status.encoders import get_all_supported_encoders
+from org_status.formatters import get_all_supported_formatters
 
 
 def get_host_token(host_name):
@@ -44,7 +44,15 @@ def generate_fetch_jobs(org_strings):
 
 def aggregate_org_status(org_host, threads=2):
     with Pool(processes=threads) as pool:
-        return pool.map(org_host.process_repository, org_host.repositories)
+        urls = []
+        for repo in org_host.repositories:
+            urls.append(repo.web_url)
+        return pool.map(org_host.process_repository, urls)
+
+
+def aggregate_org_status_from_repo_list(org_host, repos, threads=2):
+    with Pool(processes=threads) as pool:
+        return pool.map(org_host.process_repository, repos)
 
 
 def present_status(statuses, no_color):
@@ -84,6 +92,7 @@ def get_argument_parser():
     parser.add_argument('--verbose', '-v', action='store_true')
     parser.add_argument('--hosts-only', '-o', action='store_true')
     parser.add_argument('--skip-host-checks', action='store_true')
+    parser.add_argument('--use-repo-list', type=str)
     parser.add_argument('--export-repos', type=str)
     parser.add_argument('--format', type=str, default='gitman')
     parser.add_argument('--check-providers-only', action='store_true')
@@ -100,14 +109,31 @@ def encode_repo_list(repo_data, encoder_name, styled):
             try:
                 encoded_repo_list = encoder().convert_repo_list_to_format(
                     repo_data)
+                return encoded_repo_list
             except NotImplementedError:
                 print(styled(
                         f'{encoder_name} does not support exporting results',
                         'red'))
-        else:
-            print(styled(f'unknown export format {encoder_name}', 'red'))
 
-    return encoded_repo_list
+    print(styled(f'unknown export format {encoder_name}', 'red'))
+
+
+def decode_repo_list(file_name, decoder_name, styled):
+    decoders = get_all_supported_formatters()
+    decoded_repo_list = None
+
+    for decoder in decoders:
+        if decoder.NAME == decoder_name:
+            try:
+                decoded_repo_list = decoder().convert_format_to_repo_list(
+                    file_name)
+                return decoded_repo_list
+            except NotImplementedError:
+                print(styled(
+                        f'{decoder_name} does not support exporting results',
+                        'red'))
+
+    print(styled(f'unknown export format {decoder_name}', 'red'))
 
 
 def write_data_to_file(encoded_data, filename, styled, verbose):
@@ -148,55 +174,78 @@ def main():
 
     all_repositories = []
 
-    for Host, org in generate_fetch_jobs(args.orgs):
-        token = None
+    if args.use_repo_list:
+        repos = decode_repo_list(args.use_repo_list, args.format, styled)
 
-        if not args.skip_host_checks:
-            try:
-                if not Host.get_host_status():
-                    print(styled(f'{Host.HostName} is currently down', 'red'))
-                    continue
-                else:
-                    hosts_only_print(f'{Host.HostName} is up')
-            except NotImplementedError:
-                verbose(
-                    f'{Host.HostName} does not support checking host status')
+        for Host, org in generate_fetch_jobs(args.orgs):
+            repo_names = []
+            repo_urls = []
+            for repo in repos:
+                repo_names.append(repo[0])
+                if repo[1].split('.com')[0].split('//')[1] == Host.HostName:
+                    repo_urls.append(repo[1])
 
-        if args.hosts_only and args.skip_host_checks:
-            verbose('nothing to do')
-            return
-        elif args.hosts_only:
-            continue
+            if args.export_repos:
+                all_repositories += repo_names
+                continue
 
-        if (args.verbose):
-            print(f'processing org {Host.HostName}:{org}')
+            org_host = Host(None, org, False, verbose=args.verbose)
+            org_status = aggregate_org_status_from_repo_list(
+                org_host, repo_urls, threads=args.threads)
+            present_status(org_status, args.no_color)
+    else:
+        for Host, org in generate_fetch_jobs(args.orgs):
+            token = None
+
+            if not args.skip_host_checks:
+                try:
+                    if not Host.get_host_status():
+                        print(styled(
+                            f'{Host.HostName} is currently down', 'red'))
+                        continue
+                    else:
+                        hosts_only_print(f'{Host.HostName} is up')
+                except NotImplementedError:
+                    verbose(
+                        f'{Host.HostName} '
+                        f'does not support checking host status')
+
+            if args.hosts_only and args.skip_host_checks:
+                verbose('nothing to do')
+                return
+            elif args.hosts_only:
+                continue
 
-        try:
-            token = get_host_token(Host.HostName)
-        except KeyError as exp:
-            clean_exp = str(exp).replace("'", '')
+            if (args.verbose):
+                print(f'processing org {Host.HostName}:{org}')
 
-            if clean_exp.endswith('TOKEN'):
-                token_type = clean_exp.replace('_TOKEN', '').title()
+            try:
+                token = get_host_token(Host.HostName)
+            except KeyError as exp:
+                clean_exp = str(exp).replace("'", '')
 
-                text = (f'Lookup requires an access token from {token_type} '
-                        f'with permissions to this organization. Please set '
-                        f'an environment variable named {clean_exp}.')
+                if clean_exp.endswith('TOKEN'):
+                    token_type = clean_exp.replace('_TOKEN', '').title()
 
-                print(styled(text, 'red'))
+                    text = (f'Lookup requires an access token from '
+                            f'{token_type} with permissions to this '
+                            f'organization. Please set an environment '
+                            f'variable named {clean_exp}.')
 
-                continue
+                    print(styled(text, 'red'))
+
+                    continue
 
-            raise exp
+                raise exp
 
-        org_host = Host(token, org, verbose=args.verbose)
+            org_host = Host(token, org, True, verbose=args.verbose)
 
-        if args.export_repos:
-            all_repositories += org_host.repositories
-            continue
+            if args.export_repos:
+                all_repositories += org_host.repositories
+                continue
 
-        org_status = aggregate_org_status(org_host, threads=args.threads)
-        present_status(org_status, args.no_color)
+            org_status = aggregate_org_status(org_host, threads=args.threads)
+            present_status(org_status, args.no_color)
 
     if args.export_repos:
         export_data = encode_repo_list(all_repositories, args.format, styled)