From 8083d7c3622e238de29f881a6eb52bc9bc881d25 Mon Sep 17 00:00:00 2001 From: Charlie Liu Date: Tue, 13 Nov 2018 00:06:30 -0500 Subject: [PATCH] Add gitshelf as a format and fetch certain repos This adds support for gitshelf as a format for a list of repositories. This also adds support for fetching only repos that are listed in a list of repositories, as specified by --import-repos and --format. Closes https://github.com/ksdme/org-status/issues/6 and https://github.com/ksdme/org-status/issues/29 --- org_status/encoders/__init__.py | 13 --- org_status/encoders/gitman.py | 19 ----- org_status/formatters/__init__.py | 15 ++++ org_status/formatters/gitman.py | 32 +++++++ org_status/formatters/gitshelf.py | 32 +++++++ org_status/org_hosts/github.py | 17 ++-- org_status/org_hosts/gitlab.py | 17 ++-- org_status/org_status.py | 133 ++++++++++++++++++++---------- 8 files changed, 188 insertions(+), 90 deletions(-) delete mode 100644 org_status/encoders/__init__.py delete mode 100644 org_status/encoders/gitman.py create mode 100644 org_status/formatters/__init__.py create mode 100644 org_status/formatters/gitman.py create mode 100644 org_status/formatters/gitshelf.py diff --git a/org_status/encoders/__init__.py b/org_status/encoders/__init__.py deleted file mode 100644 index 1879851..0000000 --- a/org_status/encoders/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -class RepoListEncoder: - NAME = None - - def convert_repo_list_to_format(self, repos): - raise NotImplementedError() - - -def get_all_supported_encoders(): - from org_status.encoders.gitman import GitManEncoder - - return ( - GitManEncoder, - ) diff --git a/org_status/encoders/gitman.py b/org_status/encoders/gitman.py deleted file mode 100644 index 12eea2f..0000000 --- a/org_status/encoders/gitman.py +++ /dev/null @@ -1,19 +0,0 @@ -import yaml -from giturlparse import parse - -from org_status.encoders import RepoListEncoder - - -class GitManEncoder(RepoListEncoder): - NAME = 'gitman' - - def convert_repo_list_to_format(self, repos): - yml_data = {'sources': []} - - for repo in repos: - name = parse(repo.web_url).repo - yml_data['sources'].append({'name': name, - 'repo': repo.web_url, - 'rev': 'master'}) - - return yaml.dump(yml_data, default_flow_style=False) diff --git a/org_status/formatters/__init__.py b/org_status/formatters/__init__.py new file mode 100644 index 0000000..3c46161 --- /dev/null +++ b/org_status/formatters/__init__.py @@ -0,0 +1,15 @@ +class RepoListFormatter: + NAME = None + + def convert_repo_list_to_format(self, repos): + raise NotImplementedError() + + +def get_all_supported_formatters(): + from org_status.formatters.gitman import GitManFormatter + from org_status.formatters.gitshelf import GitShelfFormatter + + return ( + GitManFormatter, + GitShelfFormatter, + ) diff --git a/org_status/formatters/gitman.py b/org_status/formatters/gitman.py new file mode 100644 index 0000000..2d47cf9 --- /dev/null +++ b/org_status/formatters/gitman.py @@ -0,0 +1,32 @@ +import yaml +from giturlparse import parse + +from org_status.formatters import RepoListFormatter + + +class GitManFormatter(RepoListFormatter): + NAME = 'gitman' + + def convert_repo_list_to_format(self, repos): + yml_data = {'sources': []} + + for repo in repos: + name = parse(repo.web_url).repo + yml_data['sources'].append({'name': name, + 'repo': repo.web_url, + 'rev': 'master'}) + + return yaml.dump(yml_data, default_flow_style=False) + + def convert_format_to_repo_list(self, file_name): + repos = [] + + try: + with open(file_name, 'r') as file: + yml_data = yaml.load(file) + + for repo in yml_data['sources']: + repos.append([repo['name'], repo['repo']]) + return repos + except FileNotFoundError: + print(f'unable to find file {file_name}') diff --git a/org_status/formatters/gitshelf.py b/org_status/formatters/gitshelf.py new file mode 100644 index 0000000..37e54da --- /dev/null +++ b/org_status/formatters/gitshelf.py @@ -0,0 +1,32 @@ +import yaml +from giturlparse import parse + +from org_status.formatters import RepoListFormatter + + +class GitShelfFormatter(RepoListFormatter): + NAME = 'gitshelf' + + def convert_repo_list_to_format(self, repos): + yml_data = {'books': []} + + for repo in repos: + name = parse(repo.web_url).repo + yml_data['books'].append({'book': name, + 'git': repo.web_url, + 'branch': 'master'}) + + return yaml.dump(yml_data, default_flow_style=False) + + def convert_format_to_repo_list(self, file_name): + repos = [] + + try: + with open(file_name, 'r') as file: + yml_data = yaml.load(file) + + for repo in yml_data['books']: + repos.append([repo['book'], repo['git']]) + return repos + except FileNotFoundError: + print(f'unable to find file {file_name}') diff --git a/org_status/org_hosts/github.py b/org_status/org_hosts/github.py index 65a5f1d..cf58549 100644 --- a/org_status/org_hosts/github.py +++ b/org_status/org_hosts/github.py @@ -14,28 +14,29 @@ class GitHubOrg(OrgHost): HostName = 'github' StatusProvider = [TravisBuildStatus, AppVeyorStatus] - def __init__(self, token, group, **kargs): + def __init__(self, token, group, fetch, **kargs): super().__init__(**kargs) self._group = group - self._token = GitHubToken(token) - self._org = GitHubOrganization(self._token, self._group) - self._status_provider = [] for i in enumerate(self.StatusProvider): self._status_provider.append(self.StatusProvider[i[0]](self._group)) + if fetch: + self._token = GitHubToken(token) + self._org = GitHubOrganization(self._token, self._group) + @classmethod def get_host_status(cls): status = requests.get('https://status.github.com/api/status.json') status = json.loads(status.text) return status['status'] == 'good' - def process_repository(self, repo, branch='master'): - self.print_status(repo.web_url) + def process_repository(self, web_url, branch='master'): + self.print_status(web_url) # reliable enough? - repo_name = repo.web_url.split('/')[-1] + repo_name = web_url.split('/')[-1] repo_status = [] for i in enumerate(self._status_provider): repo_status.append(self._status_provider[i[0]] @@ -56,7 +57,7 @@ def process_repository(self, repo, branch='master'): elif Status.ERROR in repo_status: repo_status = Status.ERROR - return RepoStatus(repo.web_url, repo_status) + return RepoStatus(web_url, repo_status) @property def repositories(self): diff --git a/org_status/org_hosts/gitlab.py b/org_status/org_hosts/gitlab.py index 65ac0e9..5d0ce24 100644 --- a/org_status/org_hosts/gitlab.py +++ b/org_status/org_hosts/gitlab.py @@ -15,31 +15,32 @@ class GitLabOrg(OrgHost): HOST_STATUS_URL = ('https://api.status.io/1.0/status' '/5b36dc6502d06804c08349f7') - def __init__(self, token, group, **kargs): + def __init__(self, token, group, fetch, **kargs): super().__init__(**kargs) self._group = group - self._token = GitLabPrivateToken(token) - self._org = GitLabOrganization(self._token, self._group) - self._status_provider = self.StatusProvider(self._group) + if fetch: + self._token = GitLabPrivateToken(token) + self._org = GitLabOrganization(self._token, self._group) + @classmethod def get_host_status(cls): status = requests.get(cls.HOST_STATUS_URL) status = json.loads(status.text) return status['result']['status_overall']['status'] == 'Operational' - def process_repository(self, repo, branch='master'): - self.print_status(repo.web_url) + def process_repository(self, web_url, branch='master'): + self.print_status(web_url) # reliable enough? - repo_name = '/'.join(repo.web_url.split('/')[4:]) + repo_name = '/'.join(web_url.split('/')[4:]) repo_status = self._status_provider.get_status(repo_name, self.HostName, branch=branch) - return RepoStatus(repo.web_url, repo_status) + return RepoStatus(web_url, repo_status) @property def repositories(self): diff --git a/org_status/org_status.py b/org_status/org_status.py index a9e6f52..a0b1631 100644 --- a/org_status/org_status.py +++ b/org_status/org_status.py @@ -6,7 +6,7 @@ from org_status.status_providers import Status, get_supported_status_providers from org_status.org_hosts import get_all_supported_hosts -from org_status.encoders import get_all_supported_encoders +from org_status.formatters import get_all_supported_formatters def get_host_token(host_name): @@ -44,7 +44,15 @@ def generate_fetch_jobs(org_strings): def aggregate_org_status(org_host, threads=2): with Pool(processes=threads) as pool: - return pool.map(org_host.process_repository, org_host.repositories) + urls = [] + for repo in org_host.repositories: + urls.append(repo.web_url) + return pool.map(org_host.process_repository, urls) + + +def aggregate_org_status_from_repo_list(org_host, repos, threads=2): + with Pool(processes=threads) as pool: + return pool.map(org_host.process_repository, repos) def present_status(statuses, no_color): @@ -84,6 +92,7 @@ def get_argument_parser(): parser.add_argument('--verbose', '-v', action='store_true') parser.add_argument('--hosts-only', '-o', action='store_true') parser.add_argument('--skip-host-checks', action='store_true') + parser.add_argument('--use-repo-list', type=str) parser.add_argument('--export-repos', type=str) parser.add_argument('--format', type=str, default='gitman') parser.add_argument('--check-providers-only', action='store_true') @@ -100,14 +109,31 @@ def encode_repo_list(repo_data, encoder_name, styled): try: encoded_repo_list = encoder().convert_repo_list_to_format( repo_data) + return encoded_repo_list except NotImplementedError: print(styled( f'{encoder_name} does not support exporting results', 'red')) - else: - print(styled(f'unknown export format {encoder_name}', 'red')) - return encoded_repo_list + print(styled(f'unknown export format {encoder_name}', 'red')) + + +def decode_repo_list(file_name, decoder_name, styled): + decoders = get_all_supported_formatters() + decoded_repo_list = None + + for decoder in decoders: + if decoder.NAME == decoder_name: + try: + decoded_repo_list = decoder().convert_format_to_repo_list( + file_name) + return decoded_repo_list + except NotImplementedError: + print(styled( + f'{decoder_name} does not support exporting results', + 'red')) + + print(styled(f'unknown export format {decoder_name}', 'red')) def write_data_to_file(encoded_data, filename, styled, verbose): @@ -148,55 +174,78 @@ def main(): all_repositories = [] - for Host, org in generate_fetch_jobs(args.orgs): - token = None + if args.use_repo_list: + repos = decode_repo_list(args.use_repo_list, args.format, styled) - if not args.skip_host_checks: - try: - if not Host.get_host_status(): - print(styled(f'{Host.HostName} is currently down', 'red')) - continue - else: - hosts_only_print(f'{Host.HostName} is up') - except NotImplementedError: - verbose( - f'{Host.HostName} does not support checking host status') + for Host, org in generate_fetch_jobs(args.orgs): + repo_names = [] + repo_urls = [] + for repo in repos: + repo_names.append(repo[0]) + if repo[1].split('.com')[0].split('//')[1] == Host.HostName: + repo_urls.append(repo[1]) - if args.hosts_only and args.skip_host_checks: - verbose('nothing to do') - return - elif args.hosts_only: - continue + if args.export_repos: + all_repositories += repo_names + continue - if (args.verbose): - print(f'processing org {Host.HostName}:{org}') + org_host = Host(None, org, False, verbose=args.verbose) + org_status = aggregate_org_status_from_repo_list( + org_host, repo_urls, threads=args.threads) + present_status(org_status, args.no_color) + else: + for Host, org in generate_fetch_jobs(args.orgs): + token = None + + if not args.skip_host_checks: + try: + if not Host.get_host_status(): + print(styled( + f'{Host.HostName} is currently down', 'red')) + continue + else: + hosts_only_print(f'{Host.HostName} is up') + except NotImplementedError: + verbose( + f'{Host.HostName} ' + f'does not support checking host status') + + if args.hosts_only and args.skip_host_checks: + verbose('nothing to do') + return + elif args.hosts_only: + continue - try: - token = get_host_token(Host.HostName) - except KeyError as exp: - clean_exp = str(exp).replace("'", '') + if (args.verbose): + print(f'processing org {Host.HostName}:{org}') - if clean_exp.endswith('TOKEN'): - token_type = clean_exp.replace('_TOKEN', '').title() + try: + token = get_host_token(Host.HostName) + except KeyError as exp: + clean_exp = str(exp).replace("'", '') - text = (f'Lookup requires an access token from {token_type} ' - f'with permissions to this organization. Please set ' - f'an environment variable named {clean_exp}.') + if clean_exp.endswith('TOKEN'): + token_type = clean_exp.replace('_TOKEN', '').title() - print(styled(text, 'red')) + text = (f'Lookup requires an access token from ' + f'{token_type} with permissions to this ' + f'organization. Please set an environment ' + f'variable named {clean_exp}.') - continue + print(styled(text, 'red')) + + continue - raise exp + raise exp - org_host = Host(token, org, verbose=args.verbose) + org_host = Host(token, org, True, verbose=args.verbose) - if args.export_repos: - all_repositories += org_host.repositories - continue + if args.export_repos: + all_repositories += org_host.repositories + continue - org_status = aggregate_org_status(org_host, threads=args.threads) - present_status(org_status, args.no_color) + org_status = aggregate_org_status(org_host, threads=args.threads) + present_status(org_status, args.no_color) if args.export_repos: export_data = encode_repo_list(all_repositories, args.format, styled)