From 076c1c17765df7aae76d7e420fece95ce2eb7796 Mon Sep 17 00:00:00 2001
From: Charlie Liu <CLiu13@users.noreply.github.com>
Date: Tue, 13 Nov 2018 00:06:30 -0500
Subject: [PATCH] Add gitshelf as a format and fetch certain repos

This adds support for gitshelf as a format for a list
of repositories. This also adds support for fetching
only repos that are listed in a list of repositories,
as specified by --import-repos and --format.

Closes https://github.com/ksdme/org-status/issues/29
---
 org_status/encoders/__init__.py   |  13 ---
 org_status/encoders/gitman.py     |  19 ----
 org_status/formatters/__init__.py |  18 ++++
 org_status/formatters/gitman.py   |  21 ++++
 org_status/formatters/gitshelf.py |  21 ++++
 org_status/org_hosts/github.py    |  24 +++--
 org_status/org_hosts/gitlab.py    |  19 ++--
 org_status/org_status.py          | 158 ++++++++++++++++++++----------
 8 files changed, 192 insertions(+), 101 deletions(-)
 delete mode 100644 org_status/encoders/__init__.py
 delete mode 100644 org_status/encoders/gitman.py
 create mode 100644 org_status/formatters/__init__.py
 create mode 100644 org_status/formatters/gitman.py
 create mode 100644 org_status/formatters/gitshelf.py

diff --git a/org_status/encoders/__init__.py b/org_status/encoders/__init__.py
deleted file mode 100644
index 1879851..0000000
--- a/org_status/encoders/__init__.py
+++ /dev/null
@@ -1,13 +0,0 @@
-class RepoListEncoder:
-    NAME = None
-
-    def convert_repo_list_to_format(self, repos):
-        raise NotImplementedError()
-
-
-def get_all_supported_encoders():
-    from org_status.encoders.gitman import GitManEncoder
-
-    return (
-        GitManEncoder,
-    )
diff --git a/org_status/encoders/gitman.py b/org_status/encoders/gitman.py
deleted file mode 100644
index 12eea2f..0000000
--- a/org_status/encoders/gitman.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import yaml
-from giturlparse import parse
-
-from org_status.encoders import RepoListEncoder
-
-
-class GitManEncoder(RepoListEncoder):
-    NAME = 'gitman'
-
-    def convert_repo_list_to_format(self, repos):
-        yml_data = {'sources': []}
-
-        for repo in repos:
-            name = parse(repo.web_url).repo
-            yml_data['sources'].append({'name': name,
-                                        'repo': repo.web_url,
-                                        'rev': 'master'})
-
-        return yaml.dump(yml_data, default_flow_style=False)
diff --git a/org_status/formatters/__init__.py b/org_status/formatters/__init__.py
new file mode 100644
index 0000000..0331b71
--- /dev/null
+++ b/org_status/formatters/__init__.py
@@ -0,0 +1,18 @@
+class RepoListFormatter:
+    NAME = None
+
+    def encode_repo_list(self, repos):
+        raise NotImplementedError()
+    
+    def decode_repo_list(self, file_name):
+        raise NotImplementedError()
+
+
+def get_all_supported_formatters():
+    from org_status.formatters.gitman import GitManFormatter
+    from org_status.formatters.gitshelf import GitShelfFormatter
+
+    return (
+        GitManFormatter,
+        GitShelfFormatter,
+    )
diff --git a/org_status/formatters/gitman.py b/org_status/formatters/gitman.py
new file mode 100644
index 0000000..7437691
--- /dev/null
+++ b/org_status/formatters/gitman.py
@@ -0,0 +1,21 @@
+import yaml
+from giturlparse import parse
+
+from org_status.formatters import RepoListFormatter
+
+
+class GitManFormatter(RepoListFormatter):
+    NAME = 'gitman'
+
+    def encode_repo_list(self, urls):
+        for url in urls:
+            name = parse(url).repo
+            yield {'name': name,
+                   'repo': url,
+                   'rev': 'master'}
+
+    def decode_repo_list(self, file_name):
+        with open(file_name, 'r') as file:
+            yml_data = yaml.load(file)
+            for repo in yml_data['sources']:
+                yield [repo['name'], repo['repo']]
diff --git a/org_status/formatters/gitshelf.py b/org_status/formatters/gitshelf.py
new file mode 100644
index 0000000..0d8327f
--- /dev/null
+++ b/org_status/formatters/gitshelf.py
@@ -0,0 +1,21 @@
+import yaml
+from giturlparse import parse
+
+from org_status.formatters import RepoListFormatter
+
+
+class GitShelfFormatter(RepoListFormatter):
+    NAME = 'gitshelf'
+
+    def encode_repo_list(self, urls):
+        for url in urls:
+            name = parse(url).repo
+            yield {'book': name,
+                   'git': url,
+                   'branch': 'master'}
+
+    def decode_repo_list(self, file_name):
+        with open(file_name, 'r') as file:
+            yml_data = yaml.load(file)
+            for repo in yml_data['books']:
+                yield [repo['book'], repo['git']]
diff --git a/org_status/org_hosts/github.py b/org_status/org_hosts/github.py
index 65a5f1d..5733db0 100644
--- a/org_status/org_hosts/github.py
+++ b/org_status/org_hosts/github.py
@@ -14,28 +14,30 @@ class GitHubOrg(OrgHost):
     HostName = 'github'
     StatusProvider = [TravisBuildStatus, AppVeyorStatus]
 
-    def __init__(self, token, group, **kargs):
+    def __init__(self, token, group, sync=True, **kargs):
         super().__init__(**kargs)
 
         self._group = group
-        self._token = GitHubToken(token)
-        self._org = GitHubOrganization(self._token, self._group)
-
         self._status_provider = []
         for i in enumerate(self.StatusProvider):
             self._status_provider.append(self.StatusProvider[i[0]](self._group))
 
+        self._sync = sync
+        if sync:
+            self._token = GitHubToken(token)
+            self._org = GitHubOrganization(self._token, self._group)
+
     @classmethod
     def get_host_status(cls):
         status = requests.get('https://status.github.com/api/status.json')
         status = json.loads(status.text)
         return status['status'] == 'good'
 
-    def process_repository(self, repo, branch='master'):
-        self.print_status(repo.web_url)
+    def process_repository(self, web_url, branch='master'):
+        self.print_status(web_url)
 
         # reliable enough?
-        repo_name = repo.web_url.split('/')[-1]
+        repo_name = web_url.split('/')[-1]
         repo_status = []
         for i in enumerate(self._status_provider):
             repo_status.append(self._status_provider[i[0]]
@@ -56,8 +58,12 @@ def process_repository(self, repo, branch='master'):
         elif Status.ERROR in repo_status:
             repo_status = Status.ERROR
 
-        return RepoStatus(repo.web_url, repo_status)
+        return RepoStatus(web_url, repo_status)
 
     @property
     def repositories(self):
-        return self._org.repositories
+        return [repo.web_url for repo in self._org.repositories]
+    
+    @property
+    def sync(self):
+        return self._sync
diff --git a/org_status/org_hosts/gitlab.py b/org_status/org_hosts/gitlab.py
index 65ac0e9..82a75ca 100644
--- a/org_status/org_hosts/gitlab.py
+++ b/org_status/org_hosts/gitlab.py
@@ -15,32 +15,33 @@ class GitLabOrg(OrgHost):
     HOST_STATUS_URL = ('https://api.status.io/1.0/status'
                        '/5b36dc6502d06804c08349f7')
 
-    def __init__(self, token, group, **kargs):
+    def __init__(self, token, group, sync=True, **kargs):
         super().__init__(**kargs)
 
         self._group = group
-        self._token = GitLabPrivateToken(token)
-        self._org = GitLabOrganization(self._token, self._group)
-
         self._status_provider = self.StatusProvider(self._group)
 
+        if sync:
+            self._token = GitLabPrivateToken(token)
+            self._org = GitLabOrganization(self._token, self._group)
+
     @classmethod
     def get_host_status(cls):
         status = requests.get(cls.HOST_STATUS_URL)
         status = json.loads(status.text)
         return status['result']['status_overall']['status'] == 'Operational'
 
-    def process_repository(self, repo, branch='master'):
-        self.print_status(repo.web_url)
+    def process_repository(self, web_url, branch='master'):
+        self.print_status(web_url)
 
         # reliable enough?
-        repo_name = '/'.join(repo.web_url.split('/')[4:])
+        repo_name = '/'.join(web_url.split('/')[4:])
         repo_status = self._status_provider.get_status(repo_name,
                                                        self.HostName,
                                                        branch=branch)
 
-        return RepoStatus(repo.web_url, repo_status)
+        return RepoStatus(web_url, repo_status)
 
     @property
     def repositories(self):
-        return self._org.repositories
+        return [repo.web_url for repo in self._org.repositories]
diff --git a/org_status/org_status.py b/org_status/org_status.py
index a9e6f52..fa91cb2 100644
--- a/org_status/org_status.py
+++ b/org_status/org_status.py
@@ -1,12 +1,14 @@
 from os import environ, path
 from multiprocessing.dummy import Pool
 from argparse import ArgumentParser
-
+from giturlparse import parse
 from termcolor import colored
 
+import yaml
+
 from org_status.status_providers import Status, get_supported_status_providers
 from org_status.org_hosts import get_all_supported_hosts
-from org_status.encoders import get_all_supported_encoders
+from org_status.formatters import get_all_supported_formatters
 
 
 def get_host_token(host_name):
@@ -21,8 +23,8 @@ def get_status_provider_statuses():
             yield (provider, None)
 
 
-def generate_fetch_jobs(org_strings):
-    for org_string in org_strings:
+def generate_fetch_jobs(args):
+    for org_string in args.orgs:
         host, sym, org = org_string.strip().partition(':')
         host = host.lower()
 
@@ -35,16 +37,37 @@ def generate_fetch_jobs(org_strings):
         if host != '':
             for supported_host in get_all_supported_hosts():
                 if host == supported_host.HostName:
-                    yield (supported_host, org)
+                    repo_urls = get_repo_urls(args, supported_host)
+                    yield (supported_host, org, repo_urls)
                     raise StopIteration
         else:
             for available_host in get_all_supported_hosts():
-                yield (available_host, org)
+                repo_urls = get_repo_urls(args, available_host)
+                yield (available_host, org, repo_urls)
+
+
+def get_repo_urls(args, host):
+    repo_urls = []
+    if args.use_repo_list:
+        styled = (lambda l, *_: l) if args.no_color else colored
+        repos = decode_repo_list(args.use_repo_list, args.format, styled)
+        for repo in repos:
+            if parse(repo[1]).platform == host.HostName:
+                repo_urls.append(repo[1])
+    return repo_urls
 
 
 def aggregate_org_status(org_host, threads=2):
     with Pool(processes=threads) as pool:
-        return pool.map(org_host.process_repository, org_host.repositories)
+        urls = []
+        for repo in org_host.repositories:
+            urls.append(repo.web_url)
+        return pool.map(org_host.process_repository, urls)
+
+
+def aggregate_org_status_from_repo_list(org_host, repos, threads=2):
+    with Pool(processes=threads) as pool:
+        return pool.map(org_host.process_repository, repos)
 
 
 def present_status(statuses, no_color):
@@ -84,6 +107,7 @@ def get_argument_parser():
     parser.add_argument('--verbose', '-v', action='store_true')
     parser.add_argument('--hosts-only', '-o', action='store_true')
     parser.add_argument('--skip-host-checks', action='store_true')
+    parser.add_argument('--use-repo-list', type=str)
     parser.add_argument('--export-repos', type=str)
     parser.add_argument('--format', type=str, default='gitman')
     parser.add_argument('--check-providers-only', action='store_true')
@@ -92,22 +116,40 @@ def get_argument_parser():
 
 
 def encode_repo_list(repo_data, encoder_name, styled):
-    encoders = get_all_supported_encoders()
+    encoders = get_all_supported_formatters()
     encoded_repo_list = None
 
     for encoder in encoders:
         if encoder.NAME == encoder_name:
             try:
-                encoded_repo_list = encoder().convert_repo_list_to_format(
-                    repo_data)
+                encoded_repo_list = yaml.dump(
+                    list(encoder().encode_repo_list(
+                    repo_data)), default_flow_style=False)
+                return encoded_repo_list
             except NotImplementedError:
                 print(styled(
                         f'{encoder_name} does not support exporting results',
                         'red'))
-        else:
-            print(styled(f'unknown export format {encoder_name}', 'red'))
 
-    return encoded_repo_list
+    print(styled(f'unknown export format {encoder_name}', 'red'))
+
+
+def decode_repo_list(file_name, decoder_name, styled):
+    decoders = get_all_supported_formatters()
+    decoded_repo_list = None
+
+    for decoder in decoders:
+        if decoder.NAME == decoder_name:
+            try:
+                decoded_repo_list = list(decoder().decode_repo_list(
+                    file_name))
+                return decoded_repo_list
+            except NotImplementedError:
+                print(styled(
+                        f'{decoder_name} does not support exporting results',
+                        'red'))
+
+    print(styled(f'unknown export format {decoder_name}', 'red'))
 
 
 def write_data_to_file(encoded_data, filename, styled, verbose):
@@ -148,55 +190,69 @@ def main():
 
     all_repositories = []
 
-    for Host, org in generate_fetch_jobs(args.orgs):
-        token = None
-
-        if not args.skip_host_checks:
-            try:
-                if not Host.get_host_status():
-                    print(styled(f'{Host.HostName} is currently down', 'red'))
-                    continue
-                else:
-                    hosts_only_print(f'{Host.HostName} is up')
-            except NotImplementedError:
-                verbose(
-                    f'{Host.HostName} does not support checking host status')
+    if args.use_repo_list:
+        for Host, org, repo_urls in generate_fetch_jobs(args):
+            if args.export_repos:
+                all_repositories += repo_urls
+                continue
 
-        if args.hosts_only and args.skip_host_checks:
-            verbose('nothing to do')
-            return
-        elif args.hosts_only:
-            continue
+            org_host = Host(None, org, False, verbose=args.verbose)
+            org_status = aggregate_org_status_from_repo_list(
+                org_host, repo_urls, threads=args.threads)
+            present_status(org_status, args.no_color)
+    else:
+        for Host, org, _ in generate_fetch_jobs(args):
+            token = None
+
+            if not args.skip_host_checks:
+                try:
+                    if not Host.get_host_status():
+                        print(styled(
+                            f'{Host.HostName} is currently down', 'red'))
+                        continue
+                    else:
+                        hosts_only_print(f'{Host.HostName} is up')
+                except NotImplementedError:
+                    verbose(
+                        f'{Host.HostName} '
+                        f'does not support checking host status')
+
+            if args.hosts_only and args.skip_host_checks:
+                verbose('nothing to do')
+                return
+            elif args.hosts_only:
+                continue
 
-        if (args.verbose):
-            print(f'processing org {Host.HostName}:{org}')
+            if (args.verbose):
+                print(f'processing org {Host.HostName}:{org}')
 
-        try:
-            token = get_host_token(Host.HostName)
-        except KeyError as exp:
-            clean_exp = str(exp).replace("'", '')
+            try:
+                token = get_host_token(Host.HostName)
+            except KeyError as exp:
+                clean_exp = str(exp).replace("'", '')
 
-            if clean_exp.endswith('TOKEN'):
-                token_type = clean_exp.replace('_TOKEN', '').title()
+                if clean_exp.endswith('TOKEN'):
+                    token_type = clean_exp.replace('_TOKEN', '').title()
 
-                text = (f'Lookup requires an access token from {token_type} '
-                        f'with permissions to this organization. Please set '
-                        f'an environment variable named {clean_exp}.')
+                    text = (f'Lookup requires an access token from '
+                            f'{token_type} with permissions to this '
+                            f'organization. Please set an environment '
+                            f'variable named {clean_exp}.')
 
-                print(styled(text, 'red'))
+                    print(styled(text, 'red'))
 
-                continue
+                    continue
 
-            raise exp
+                raise exp
 
-        org_host = Host(token, org, verbose=args.verbose)
+            org_host = Host(token, org, True, verbose=args.verbose)
 
-        if args.export_repos:
-            all_repositories += org_host.repositories
-            continue
+            if args.export_repos:
+                all_repositories += org_host.repositories
+                continue
 
-        org_status = aggregate_org_status(org_host, threads=args.threads)
-        present_status(org_status, args.no_color)
+            org_status = aggregate_org_status(org_host, threads=args.threads)
+            present_status(org_status, args.no_color)
 
     if args.export_repos:
         export_data = encode_repo_list(all_repositories, args.format, styled)