From 076c1c17765df7aae76d7e420fece95ce2eb7796 Mon Sep 17 00:00:00 2001 From: Charlie Liu <CLiu13@users.noreply.github.com> Date: Tue, 13 Nov 2018 00:06:30 -0500 Subject: [PATCH] Add gitshelf as a format and fetch certain repos This adds support for gitshelf as a format for a list of repositories. This also adds support for fetching only repos that are listed in a list of repositories, as specified by --import-repos and --format. Closes https://github.com/ksdme/org-status/issues/29 --- org_status/encoders/__init__.py | 13 --- org_status/encoders/gitman.py | 19 ---- org_status/formatters/__init__.py | 18 ++++ org_status/formatters/gitman.py | 21 ++++ org_status/formatters/gitshelf.py | 21 ++++ org_status/org_hosts/github.py | 24 +++-- org_status/org_hosts/gitlab.py | 19 ++-- org_status/org_status.py | 158 ++++++++++++++++++++---------- 8 files changed, 192 insertions(+), 101 deletions(-) delete mode 100644 org_status/encoders/__init__.py delete mode 100644 org_status/encoders/gitman.py create mode 100644 org_status/formatters/__init__.py create mode 100644 org_status/formatters/gitman.py create mode 100644 org_status/formatters/gitshelf.py diff --git a/org_status/encoders/__init__.py b/org_status/encoders/__init__.py deleted file mode 100644 index 1879851..0000000 --- a/org_status/encoders/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -class RepoListEncoder: - NAME = None - - def convert_repo_list_to_format(self, repos): - raise NotImplementedError() - - -def get_all_supported_encoders(): - from org_status.encoders.gitman import GitManEncoder - - return ( - GitManEncoder, - ) diff --git a/org_status/encoders/gitman.py b/org_status/encoders/gitman.py deleted file mode 100644 index 12eea2f..0000000 --- a/org_status/encoders/gitman.py +++ /dev/null @@ -1,19 +0,0 @@ -import yaml -from giturlparse import parse - -from org_status.encoders import RepoListEncoder - - -class GitManEncoder(RepoListEncoder): - NAME = 'gitman' - - def convert_repo_list_to_format(self, repos): - yml_data = {'sources': []} - - for repo in repos: - name = parse(repo.web_url).repo - yml_data['sources'].append({'name': name, - 'repo': repo.web_url, - 'rev': 'master'}) - - return yaml.dump(yml_data, default_flow_style=False) diff --git a/org_status/formatters/__init__.py b/org_status/formatters/__init__.py new file mode 100644 index 0000000..0331b71 --- /dev/null +++ b/org_status/formatters/__init__.py @@ -0,0 +1,18 @@ +class RepoListFormatter: + NAME = None + + def encode_repo_list(self, repos): + raise NotImplementedError() + + def decode_repo_list(self, file_name): + raise NotImplementedError() + + +def get_all_supported_formatters(): + from org_status.formatters.gitman import GitManFormatter + from org_status.formatters.gitshelf import GitShelfFormatter + + return ( + GitManFormatter, + GitShelfFormatter, + ) diff --git a/org_status/formatters/gitman.py b/org_status/formatters/gitman.py new file mode 100644 index 0000000..7437691 --- /dev/null +++ b/org_status/formatters/gitman.py @@ -0,0 +1,21 @@ +import yaml +from giturlparse import parse + +from org_status.formatters import RepoListFormatter + + +class GitManFormatter(RepoListFormatter): + NAME = 'gitman' + + def encode_repo_list(self, urls): + for url in urls: + name = parse(url).repo + yield {'name': name, + 'repo': url, + 'rev': 'master'} + + def decode_repo_list(self, file_name): + with open(file_name, 'r') as file: + yml_data = yaml.load(file) + for repo in yml_data['sources']: + yield [repo['name'], repo['repo']] diff --git a/org_status/formatters/gitshelf.py b/org_status/formatters/gitshelf.py new file mode 100644 index 0000000..0d8327f --- /dev/null +++ b/org_status/formatters/gitshelf.py @@ -0,0 +1,21 @@ +import yaml +from giturlparse import parse + +from org_status.formatters import RepoListFormatter + + +class GitShelfFormatter(RepoListFormatter): + NAME = 'gitshelf' + + def encode_repo_list(self, urls): + for url in urls: + name = parse(url).repo + yield {'book': name, + 'git': url, + 'branch': 'master'} + + def decode_repo_list(self, file_name): + with open(file_name, 'r') as file: + yml_data = yaml.load(file) + for repo in yml_data['books']: + yield [repo['book'], repo['git']] diff --git a/org_status/org_hosts/github.py b/org_status/org_hosts/github.py index 65a5f1d..5733db0 100644 --- a/org_status/org_hosts/github.py +++ b/org_status/org_hosts/github.py @@ -14,28 +14,30 @@ class GitHubOrg(OrgHost): HostName = 'github' StatusProvider = [TravisBuildStatus, AppVeyorStatus] - def __init__(self, token, group, **kargs): + def __init__(self, token, group, sync=True, **kargs): super().__init__(**kargs) self._group = group - self._token = GitHubToken(token) - self._org = GitHubOrganization(self._token, self._group) - self._status_provider = [] for i in enumerate(self.StatusProvider): self._status_provider.append(self.StatusProvider[i[0]](self._group)) + self._sync = sync + if sync: + self._token = GitHubToken(token) + self._org = GitHubOrganization(self._token, self._group) + @classmethod def get_host_status(cls): status = requests.get('https://status.github.com/api/status.json') status = json.loads(status.text) return status['status'] == 'good' - def process_repository(self, repo, branch='master'): - self.print_status(repo.web_url) + def process_repository(self, web_url, branch='master'): + self.print_status(web_url) # reliable enough? - repo_name = repo.web_url.split('/')[-1] + repo_name = web_url.split('/')[-1] repo_status = [] for i in enumerate(self._status_provider): repo_status.append(self._status_provider[i[0]] @@ -56,8 +58,12 @@ def process_repository(self, repo, branch='master'): elif Status.ERROR in repo_status: repo_status = Status.ERROR - return RepoStatus(repo.web_url, repo_status) + return RepoStatus(web_url, repo_status) @property def repositories(self): - return self._org.repositories + return [repo.web_url for repo in self._org.repositories] + + @property + def sync(self): + return self._sync diff --git a/org_status/org_hosts/gitlab.py b/org_status/org_hosts/gitlab.py index 65ac0e9..82a75ca 100644 --- a/org_status/org_hosts/gitlab.py +++ b/org_status/org_hosts/gitlab.py @@ -15,32 +15,33 @@ class GitLabOrg(OrgHost): HOST_STATUS_URL = ('https://api.status.io/1.0/status' '/5b36dc6502d06804c08349f7') - def __init__(self, token, group, **kargs): + def __init__(self, token, group, sync=True, **kargs): super().__init__(**kargs) self._group = group - self._token = GitLabPrivateToken(token) - self._org = GitLabOrganization(self._token, self._group) - self._status_provider = self.StatusProvider(self._group) + if sync: + self._token = GitLabPrivateToken(token) + self._org = GitLabOrganization(self._token, self._group) + @classmethod def get_host_status(cls): status = requests.get(cls.HOST_STATUS_URL) status = json.loads(status.text) return status['result']['status_overall']['status'] == 'Operational' - def process_repository(self, repo, branch='master'): - self.print_status(repo.web_url) + def process_repository(self, web_url, branch='master'): + self.print_status(web_url) # reliable enough? - repo_name = '/'.join(repo.web_url.split('/')[4:]) + repo_name = '/'.join(web_url.split('/')[4:]) repo_status = self._status_provider.get_status(repo_name, self.HostName, branch=branch) - return RepoStatus(repo.web_url, repo_status) + return RepoStatus(web_url, repo_status) @property def repositories(self): - return self._org.repositories + return [repo.web_url for repo in self._org.repositories] diff --git a/org_status/org_status.py b/org_status/org_status.py index a9e6f52..fa91cb2 100644 --- a/org_status/org_status.py +++ b/org_status/org_status.py @@ -1,12 +1,14 @@ from os import environ, path from multiprocessing.dummy import Pool from argparse import ArgumentParser - +from giturlparse import parse from termcolor import colored +import yaml + from org_status.status_providers import Status, get_supported_status_providers from org_status.org_hosts import get_all_supported_hosts -from org_status.encoders import get_all_supported_encoders +from org_status.formatters import get_all_supported_formatters def get_host_token(host_name): @@ -21,8 +23,8 @@ def get_status_provider_statuses(): yield (provider, None) -def generate_fetch_jobs(org_strings): - for org_string in org_strings: +def generate_fetch_jobs(args): + for org_string in args.orgs: host, sym, org = org_string.strip().partition(':') host = host.lower() @@ -35,16 +37,37 @@ def generate_fetch_jobs(org_strings): if host != '': for supported_host in get_all_supported_hosts(): if host == supported_host.HostName: - yield (supported_host, org) + repo_urls = get_repo_urls(args, supported_host) + yield (supported_host, org, repo_urls) raise StopIteration else: for available_host in get_all_supported_hosts(): - yield (available_host, org) + repo_urls = get_repo_urls(args, available_host) + yield (available_host, org, repo_urls) + + +def get_repo_urls(args, host): + repo_urls = [] + if args.use_repo_list: + styled = (lambda l, *_: l) if args.no_color else colored + repos = decode_repo_list(args.use_repo_list, args.format, styled) + for repo in repos: + if parse(repo[1]).platform == host.HostName: + repo_urls.append(repo[1]) + return repo_urls def aggregate_org_status(org_host, threads=2): with Pool(processes=threads) as pool: - return pool.map(org_host.process_repository, org_host.repositories) + urls = [] + for repo in org_host.repositories: + urls.append(repo.web_url) + return pool.map(org_host.process_repository, urls) + + +def aggregate_org_status_from_repo_list(org_host, repos, threads=2): + with Pool(processes=threads) as pool: + return pool.map(org_host.process_repository, repos) def present_status(statuses, no_color): @@ -84,6 +107,7 @@ def get_argument_parser(): parser.add_argument('--verbose', '-v', action='store_true') parser.add_argument('--hosts-only', '-o', action='store_true') parser.add_argument('--skip-host-checks', action='store_true') + parser.add_argument('--use-repo-list', type=str) parser.add_argument('--export-repos', type=str) parser.add_argument('--format', type=str, default='gitman') parser.add_argument('--check-providers-only', action='store_true') @@ -92,22 +116,40 @@ def get_argument_parser(): def encode_repo_list(repo_data, encoder_name, styled): - encoders = get_all_supported_encoders() + encoders = get_all_supported_formatters() encoded_repo_list = None for encoder in encoders: if encoder.NAME == encoder_name: try: - encoded_repo_list = encoder().convert_repo_list_to_format( - repo_data) + encoded_repo_list = yaml.dump( + list(encoder().encode_repo_list( + repo_data)), default_flow_style=False) + return encoded_repo_list except NotImplementedError: print(styled( f'{encoder_name} does not support exporting results', 'red')) - else: - print(styled(f'unknown export format {encoder_name}', 'red')) - return encoded_repo_list + print(styled(f'unknown export format {encoder_name}', 'red')) + + +def decode_repo_list(file_name, decoder_name, styled): + decoders = get_all_supported_formatters() + decoded_repo_list = None + + for decoder in decoders: + if decoder.NAME == decoder_name: + try: + decoded_repo_list = list(decoder().decode_repo_list( + file_name)) + return decoded_repo_list + except NotImplementedError: + print(styled( + f'{decoder_name} does not support exporting results', + 'red')) + + print(styled(f'unknown export format {decoder_name}', 'red')) def write_data_to_file(encoded_data, filename, styled, verbose): @@ -148,55 +190,69 @@ def main(): all_repositories = [] - for Host, org in generate_fetch_jobs(args.orgs): - token = None - - if not args.skip_host_checks: - try: - if not Host.get_host_status(): - print(styled(f'{Host.HostName} is currently down', 'red')) - continue - else: - hosts_only_print(f'{Host.HostName} is up') - except NotImplementedError: - verbose( - f'{Host.HostName} does not support checking host status') + if args.use_repo_list: + for Host, org, repo_urls in generate_fetch_jobs(args): + if args.export_repos: + all_repositories += repo_urls + continue - if args.hosts_only and args.skip_host_checks: - verbose('nothing to do') - return - elif args.hosts_only: - continue + org_host = Host(None, org, False, verbose=args.verbose) + org_status = aggregate_org_status_from_repo_list( + org_host, repo_urls, threads=args.threads) + present_status(org_status, args.no_color) + else: + for Host, org, _ in generate_fetch_jobs(args): + token = None + + if not args.skip_host_checks: + try: + if not Host.get_host_status(): + print(styled( + f'{Host.HostName} is currently down', 'red')) + continue + else: + hosts_only_print(f'{Host.HostName} is up') + except NotImplementedError: + verbose( + f'{Host.HostName} ' + f'does not support checking host status') + + if args.hosts_only and args.skip_host_checks: + verbose('nothing to do') + return + elif args.hosts_only: + continue - if (args.verbose): - print(f'processing org {Host.HostName}:{org}') + if (args.verbose): + print(f'processing org {Host.HostName}:{org}') - try: - token = get_host_token(Host.HostName) - except KeyError as exp: - clean_exp = str(exp).replace("'", '') + try: + token = get_host_token(Host.HostName) + except KeyError as exp: + clean_exp = str(exp).replace("'", '') - if clean_exp.endswith('TOKEN'): - token_type = clean_exp.replace('_TOKEN', '').title() + if clean_exp.endswith('TOKEN'): + token_type = clean_exp.replace('_TOKEN', '').title() - text = (f'Lookup requires an access token from {token_type} ' - f'with permissions to this organization. Please set ' - f'an environment variable named {clean_exp}.') + text = (f'Lookup requires an access token from ' + f'{token_type} with permissions to this ' + f'organization. Please set an environment ' + f'variable named {clean_exp}.') - print(styled(text, 'red')) + print(styled(text, 'red')) - continue + continue - raise exp + raise exp - org_host = Host(token, org, verbose=args.verbose) + org_host = Host(token, org, True, verbose=args.verbose) - if args.export_repos: - all_repositories += org_host.repositories - continue + if args.export_repos: + all_repositories += org_host.repositories + continue - org_status = aggregate_org_status(org_host, threads=args.threads) - present_status(org_status, args.no_color) + org_status = aggregate_org_status(org_host, threads=args.threads) + present_status(org_status, args.no_color) if args.export_repos: export_data = encode_repo_list(all_repositories, args.format, styled)