From b1ca1b7e45178702e604bd18bb879be9ba84c9e8 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 28 Jan 2022 11:16:59 +0800 Subject: [PATCH 01/78] fix auto-ask-check bug --- scripts/release_issue_status/main.py | 22 +++++++------------ .../release_issue_status.yml | 6 +++++ .../release_issue_status/reply_generator.py | 3 ++- scripts/release_issue_status/requirement.txt | 3 +-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/release_issue_status/main.py b/scripts/release_issue_status/main.py index eff130e04ba5..f892a08eae46 100644 --- a/scripts/release_issue_status/main.py +++ b/scripts/release_issue_status/main.py @@ -7,9 +7,8 @@ import logging from github import Github -from azure.storage.blob import BlobClient +from reply_generator import AUTO_ASK_FOR_CHECK, begin_reply_generate -import reply_generator as rg from utils import update_issue_body, get_readme_and_output_folder, \ get_python_pipelines, get_pipeline_url, auto_close_issue @@ -144,7 +143,7 @@ def _latest_comment_time(comments, delay_from_create_date): return delay_from_create_date if not q else int((time.time() - q[-1][0]) / 3600 / 24) -def auto_reply(item, request_repo, rest_repo, sdk_repo, duplicated_issue, python_piplines, assigner_repoes): +def auto_reply(item, request_repo, rest_repo, duplicated_issue, python_piplines, assigner_repoes): logging.info("new issue number: {}".format(item.issue_object.number)) assigner_repo = assigner_repoes[item.assignee] if 'auto-link' not in item.labels: @@ -171,8 +170,7 @@ def auto_reply(item, request_repo, rest_repo, sdk_repo, duplicated_issue, python raise try: pipeline_url = get_pipeline_url(python_piplines, output_folder) - rg.begin_reply_generate(item=item, rest_repo=rest_repo, readme_link=readme_link, - pipeline_url=pipeline_url) + begin_reply_generate(item=item, rest_repo=rest_repo, readme_link=readme_link, pipeline_url=pipeline_url) if 'Configured' in item.labels: item.issue_object.remove_from_labels('Configured') except Exception as e: @@ -253,10 +251,11 @@ def main(): item.issue_object.add_to_assignees(_PYTHON_SDK_ASSIGNEES[assign_count]) item.assignee = item.issue_object.assignee.login item.issue_object.add_to_labels('assigned') - try: - auto_reply(item, request_repo, rest_repo, sdk_repo, duplicated_issue, python_piplines, assigner_repoes) - except Exception as e: - continue + if AUTO_ASK_FOR_CHECK not in item.labels: + try: + auto_reply(item, request_repo, rest_repo, duplicated_issue, python_piplines, assigner_repoes) + except Exception as e: + continue elif not item.author_latest_comment in _PYTHON_SDK_ADMINISTRATORS: item.bot_advice = 'new comment.
' if item.comment_num > 1 and item.language == 'Python': @@ -295,11 +294,6 @@ def main(): print_check('git commit -m \"update excel\"') print_check('git push -f origin HEAD') -# upload to storage account(it is created in advance) -# blob = BlobClient.from_connection_string(conn_str=os.getenv('CONN_STR'), container_name=os.getenv('FILE'), -# blob_name=_FILE_OUT) -# with open(_FILE_OUT, 'rb') as data: -# blob.upload_blob(data, overwrite=True) if __name__ == '__main__': main() diff --git a/scripts/release_issue_status/release_issue_status.yml b/scripts/release_issue_status/release_issue_status.yml index d79e7dd19b2d..2e384ec47b36 100644 --- a/scripts/release_issue_status/release_issue_status.yml +++ b/scripts/release_issue_status/release_issue_status.yml @@ -7,6 +7,12 @@ trigger: exclude: - '*' +# avoid being triggered as part of CI check +pr: + branches: + exclude: + - '*' + schedules: - cron: "0,30 1-9 * * *" displayName: Daily release diff --git a/scripts/release_issue_status/reply_generator.py b/scripts/release_issue_status/reply_generator.py index 3659f19bd33d..5a3ebc6b206c 100644 --- a/scripts/release_issue_status/reply_generator.py +++ b/scripts/release_issue_status/reply_generator.py @@ -6,6 +6,7 @@ logging.basicConfig(level=logging.INFO, format='[auto-reply log] - %(funcName)s[line:%(lineno)d] - %(levelname)s: %(message)s') +AUTO_ASK_FOR_CHECK = 'auto-ask-check' def readme_comparison(rest_repo, link_dict, labels): # to see whether need change readme @@ -68,6 +69,6 @@ def begin_reply_generate(item, rest_repo, readme_link, pipeline_url): logging.info(f'{issue_object_rg.number} run pipeline successfully') else: logging.info(f'{issue_object_rg.number} run pipeline fail') - issue_object_rg.add_to_labels('auto-ask-check') + issue_object_rg.add_to_labels(AUTO_ASK_FOR_CHECK) else: logging.info('issue {} need config readme'.format(issue_object_rg.number)) diff --git a/scripts/release_issue_status/requirement.txt b/scripts/release_issue_status/requirement.txt index 512d9702666a..95f800db46db 100644 --- a/scripts/release_issue_status/requirement.txt +++ b/scripts/release_issue_status/requirement.txt @@ -2,6 +2,5 @@ PyGithub datetime requests bs4 -azure.storage.blob==12.8.1 -azure-devops +azure-devops msrest From 5ae58ab80ac68b7f25e74ad99abe0b4e5894fc2a Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Wed, 16 Feb 2022 17:07:08 +0800 Subject: [PATCH 02/78] update comment --- scripts/auto_release/main.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 34e56022cd2e..eb2f1a746e27 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -410,6 +410,11 @@ def check_file(self): def sdk_code_path(self) -> str: return str(Path(f'sdk/{self.sdk_folder}/azure-mgmt-{self.package_name}')) + @property + def whether_single_path(self) -> bool: + path = sum([os.path.isdir(listx) for listx in os.listdir(str(Path(f'sdk/{self.sdk_folder}'))]) + return True if path==1 else False + @return_origin_path def install_package_locally(self): os.chdir(self.sdk_code_path()) @@ -454,6 +459,8 @@ def create_pr_proc(self): pr_head = "{}:{}".format(os.getenv('USR_NAME'), self.new_branch) pr_base = 'main' pr_body = "{} \n{} \n{}".format(self.issue_link, self.test_result, self.pipeline_link) + if not self.whether_single_path: + pr_body += f'\nBuildTargetingString: {self.package_name}\nSkip.CreateApiReview: true' res_create = api.pulls.create(pr_title, pr_head, pr_base, pr_body) # Add issue link on PR From 79a3594aad784b3da87b13f474d4679c18e53bda Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Wed, 16 Feb 2022 17:26:21 +0800 Subject: [PATCH 03/78] update comment --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index eb2f1a746e27..b8e4ef288b1f 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -412,7 +412,7 @@ def sdk_code_path(self) -> str: @property def whether_single_path(self) -> bool: - path = sum([os.path.isdir(listx) for listx in os.listdir(str(Path(f'sdk/{self.sdk_folder}'))]) + path = sum([os.path.isdir(listx) for listx in os.listdir(str(Path(f'sdk/{self.sdk_folder}')))]) return True if path==1 else False @return_origin_path From 7071377db0a8fcbdc037acc47beaed7ab13c0572 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Thu, 17 Feb 2022 10:03:42 +0800 Subject: [PATCH 04/78] fix bug --- scripts/auto_release/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index b8e4ef288b1f..4e4aa35ca259 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -412,8 +412,9 @@ def sdk_code_path(self) -> str: @property def whether_single_path(self) -> bool: - path = sum([os.path.isdir(listx) for listx in os.listdir(str(Path(f'sdk/{self.sdk_folder}')))]) - return True if path==1 else False + path = str(Path(f'sdk/{self.sdk_folder}')) + num = sum([os.path.isdir(f'{path}/{listx}') for listx in os.listdir(path)]) + return True if num == 1 else False @return_origin_path def install_package_locally(self): @@ -460,7 +461,7 @@ def create_pr_proc(self): pr_base = 'main' pr_body = "{} \n{} \n{}".format(self.issue_link, self.test_result, self.pipeline_link) if not self.whether_single_path: - pr_body += f'\nBuildTargetingString: {self.package_name}\nSkip.CreateApiReview: true' + pr_body += f'\nBuildTargetingString\n azure-mgmt-{self.package_name}\nSkip.CreateApiReview\ntrue' res_create = api.pulls.create(pr_title, pr_head, pr_base, pr_body) # Add issue link on PR From 8f6fe79664db8c2957009fcc3058c85d68c3ccfd Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Thu, 17 Feb 2022 10:26:05 +0800 Subject: [PATCH 05/78] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 4e4aa35ca259..97620c4f9b64 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -414,7 +414,7 @@ def sdk_code_path(self) -> str: def whether_single_path(self) -> bool: path = str(Path(f'sdk/{self.sdk_folder}')) num = sum([os.path.isdir(f'{path}/{listx}') for listx in os.listdir(path)]) - return True if num == 1 else False + return num == 1 @return_origin_path def install_package_locally(self): From 8d23d87d7dea6d635316ac75fddc1f37aaeef644 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 17 Feb 2022 10:44:21 +0800 Subject: [PATCH 06/78] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 97620c4f9b64..ef846cc0534a 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -413,7 +413,7 @@ def sdk_code_path(self) -> str: @property def whether_single_path(self) -> bool: path = str(Path(f'sdk/{self.sdk_folder}')) - num = sum([os.path.isdir(f'{path}/{listx}') for listx in os.listdir(path)]) + num = sum([os.path.isdir(str(Path(f'{path}/{listx}'))) for listx in os.listdir(path)]) return num == 1 @return_origin_path From 44faa80efb2b2b16c65e93f55df1432feb4cb8f1 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 17 Feb 2022 10:46:51 +0800 Subject: [PATCH 07/78] Update main.py --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ef846cc0534a..b6a791bda638 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -411,7 +411,7 @@ def sdk_code_path(self) -> str: return str(Path(f'sdk/{self.sdk_folder}/azure-mgmt-{self.package_name}')) @property - def whether_single_path(self) -> bool: + def is_single_path(self) -> bool: path = str(Path(f'sdk/{self.sdk_folder}')) num = sum([os.path.isdir(str(Path(f'{path}/{listx}'))) for listx in os.listdir(path)]) return num == 1 @@ -460,7 +460,7 @@ def create_pr_proc(self): pr_head = "{}:{}".format(os.getenv('USR_NAME'), self.new_branch) pr_base = 'main' pr_body = "{} \n{} \n{}".format(self.issue_link, self.test_result, self.pipeline_link) - if not self.whether_single_path: + if not self.is_single_path: pr_body += f'\nBuildTargetingString\n azure-mgmt-{self.package_name}\nSkip.CreateApiReview\ntrue' res_create = api.pulls.create(pr_title, pr_head, pr_base, pr_body) From 84768ebf73ec5cb679abd0a91a97651cb6ee2daf Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 18 Feb 2022 11:01:16 +0800 Subject: [PATCH 08/78] fix changelog format for new service --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index b6a791bda638..1b129be68cce 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -367,7 +367,7 @@ def edit_changelog_for_new_service(self): def edit_changelog_for_new_service_proc(content: List[str]): for i in range(0, len(content)): if '##' in content[i]: - content[i] = f'## {self.next_version}({current_time()})' + content[i] = f'## {self.next_version}({current_time()})\n\n' break modify_file(str(Path(self.sdk_code_path()) / 'CHANGELOG.md'), edit_changelog_for_new_service_proc) From 29f9a86b805ca6f507a0fde8fd6f70ea36d4c19d Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 18 Feb 2022 15:39:12 +0800 Subject: [PATCH 09/78] fix check_pprint_name bug --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 1b129be68cce..ea59218a1b1c 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -266,7 +266,7 @@ def edit_file_for_pprint_name(content: List[str]): for file in os.listdir(self.sdk_code_path()): if os.path.isfile(file): - modify_file(file, edit_file_for_pprint_name) + modify_file(str(Path(self.sdk_code_path()) / file), edit_file_for_pprint_name) log(f' replace \"MyService\" with \"{pprint_name}\" successfully ') def get_all_files_under_package_folder(self) -> List[str]: From 5b11c1267cd1e2f0480f0b8b67b3e021c9e4b042 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 18 Feb 2022 17:14:49 +0800 Subject: [PATCH 10/78] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ea59218a1b1c..f3b229318e8c 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -367,7 +367,7 @@ def edit_changelog_for_new_service(self): def edit_changelog_for_new_service_proc(content: List[str]): for i in range(0, len(content)): if '##' in content[i]: - content[i] = f'## {self.next_version}({current_time()})\n\n' + content[i] = f'## {self.next_version}({current_time()})\n' break modify_file(str(Path(self.sdk_code_path()) / 'CHANGELOG.md'), edit_changelog_for_new_service_proc) From 6b13133eb4920748e9d34d317117b5bec2d4aad9 Mon Sep 17 00:00:00 2001 From: BigCat20196 <1095260342@qq.com> Date: Fri, 18 Feb 2022 17:35:29 +0800 Subject: [PATCH 11/78] fix check_pprint_name bug --- scripts/auto_release/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ea59218a1b1c..5f56b69f5677 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -265,8 +265,9 @@ def edit_file_for_pprint_name(content: List[str]): content[i] = content[i].replace('MyService', pprint_name) for file in os.listdir(self.sdk_code_path()): - if os.path.isfile(file): - modify_file(str(Path(self.sdk_code_path()) / file), edit_file_for_pprint_name) + file_path = str(Path(self.sdk_code_path()) / file) + if os.path.isfile(file_path): + modify_file(file_path, edit_file_for_pprint_name) log(f' replace \"MyService\" with \"{pprint_name}\" successfully ') def get_all_files_under_package_folder(self) -> List[str]: From c53585bd9a43bb28d99907fe0c9a52692c7d4a9e Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Mon, 21 Feb 2022 10:02:18 +0800 Subject: [PATCH 12/78] Update main.py --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 89f44635cbf6..be3277b51dba 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -368,7 +368,7 @@ def edit_changelog_for_new_service(self): def edit_changelog_for_new_service_proc(content: List[str]): for i in range(0, len(content)): if '##' in content[i]: - content[i] = f'## {self.next_version}({current_time()})\n' + content[i] = f'## {self.next_version} ({current_time()})\n' break modify_file(str(Path(self.sdk_code_path()) / 'CHANGELOG.md'), edit_changelog_for_new_service_proc) From 90ab52db6e0b0bb00dc8b5ee0b3ae5e62b8b496e Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Fri, 4 Mar 2022 10:34:22 +0800 Subject: [PATCH 13/78] Update main.py --- scripts/auto_release/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index be3277b51dba..adeaa514a600 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -549,7 +549,8 @@ def create_pr(self): # commit all code print_exec('git add sdk/') print_exec('git commit -m \"code and test\"') - print_check('git push origin HEAD -f') + print_exec('git remote add azure https://github.com/Azure/azure-sdk-for-python.git') + print_check('git push azure HEAD -f') # create PR self.create_pr_proc() From c12493377629bfa23b137ce47bc4cbd3929c0742 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 15:15:54 +0800 Subject: [PATCH 14/78] test --- scripts/auto_release/main.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index adeaa514a600..ae1ab4c4685a 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -44,6 +44,10 @@ def print_exec_output(cmd: str) -> List[str]: def print_check(cmd): log(cmd) subprocess.check_call(cmd, shell=True) + +def print_check_with_path(cmd, path): + log(cmd) + subprocess.check_call(cmd, shell=True, cwd=path) def preview_version_plus(preview_label: str, last_version: str) -> str: @@ -257,6 +261,11 @@ def edit_sdk_setup(content: List[str]): modify_file(str(Path(self.sdk_code_path()) / 'setup.py'), edit_sdk_setup) + # Use the template to update readme and setup by packaging_tools + def check_sdk_readme_and_setup(self): + path = self.sdk_folder + print_check_with_path(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}',path) + def check_pprint_name(self): pprint_name = self.package_name.capitalize() @@ -403,6 +412,8 @@ def check_ci_file(self): def check_file(self): self.check_pprint_name() + # self.check_sdk_readme_and_setup() + print(f'**{self.sdk_folder=}') self.check_sdk_setup() self.check_version() self.check_changelog_file() From e6f89a28194227fc445630a5219d68e92b6b865c Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 15:33:30 +0800 Subject: [PATCH 15/78] test --- scripts/auto_release/main.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ae1ab4c4685a..f3bbf7354eba 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -263,8 +263,9 @@ def edit_sdk_setup(content: List[str]): # Use the template to update readme and setup by packaging_tools def check_sdk_readme_and_setup(self): - path = self.sdk_folder + path = str(Path(f'sdk/{self.sdk_folder}')) print_check_with_path(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}',path) + log(f' packaging_tools --build-conf successfully ') def check_pprint_name(self): pprint_name = self.package_name.capitalize() @@ -412,9 +413,8 @@ def check_ci_file(self): def check_file(self): self.check_pprint_name() - # self.check_sdk_readme_and_setup() - print(f'**{self.sdk_folder=}') - self.check_sdk_setup() + self.check_sdk_readme_and_setup() + # self.check_sdk_setup() self.check_version() self.check_changelog_file() self.check_ci_file() From 76faf366ed883ff42f43201e443e42d529b97329 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 15:38:38 +0800 Subject: [PATCH 16/78] change to azclibot --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index f3bbf7354eba..6c19b2ad524b 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -560,8 +560,8 @@ def create_pr(self): # commit all code print_exec('git add sdk/') print_exec('git commit -m \"code and test\"') - print_exec('git remote add azure https://github.com/Azure/azure-sdk-for-python.git') - print_check('git push azure HEAD -f') + print_exec('git remote add azclibot https://github.com/azclibot/azure-sdk-for-python.git') + print_check('git push azclibot HEAD -f') # create PR self.create_pr_proc() From ffb8635ac54beb77a939a8d6425d40449fa264a5 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 16:03:04 +0800 Subject: [PATCH 17/78] change to azure --- scripts/auto_release/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 6c19b2ad524b..e7b3c1f1fd25 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -560,8 +560,9 @@ def create_pr(self): # commit all code print_exec('git add sdk/') print_exec('git commit -m \"code and test\"') - print_exec('git remote add azclibot https://github.com/azclibot/azure-sdk-for-python.git') - print_check('git push azclibot HEAD -f') + print_exec('git remote add azure https://github.com/azure/azure-sdk-for-python.git') + res1 = print_exec_output('git push azure HEAD -f') + print(f'{res1=}') # create PR self.create_pr_proc() From 40a0f7469c08c85052d1b7970a8b776b5b39a20f Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 16:28:05 +0800 Subject: [PATCH 18/78] fix push --- scripts/auto_release/main.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index e7b3c1f1fd25..907319d6dd27 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -560,9 +560,7 @@ def create_pr(self): # commit all code print_exec('git add sdk/') print_exec('git commit -m \"code and test\"') - print_exec('git remote add azure https://github.com/azure/azure-sdk-for-python.git') - res1 = print_exec_output('git push azure HEAD -f') - print(f'{res1=}') + print_exec('git push origin HEAD -f') # create PR self.create_pr_proc() From 2f6b17443de2adc53d98e0663c596920e57c9cde Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 16:55:04 +0800 Subject: [PATCH 19/78] test --- scripts/auto_release/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 907319d6dd27..f57c7fc22d08 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -85,8 +85,10 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - usr = 'Azure' - branch = 'main' + # usr = 'Azure' + usr = 'azclibot' + # branch = 'main' + branch = 't2-advisor-2022-03-09-54263' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From d5ce82678a6873af4ecb7add4e8d357aa80cddf2 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 17:16:59 +0800 Subject: [PATCH 20/78] update --- scripts/auto_release/main.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index f57c7fc22d08..840d003da8f6 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -85,10 +85,8 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - # usr = 'Azure' - usr = 'azclibot' - # branch = 'main' - branch = 't2-advisor-2022-03-09-54263' + usr = 'Azure' + branch = 'main' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') @@ -416,7 +414,6 @@ def check_ci_file(self): def check_file(self): self.check_pprint_name() self.check_sdk_readme_and_setup() - # self.check_sdk_setup() self.check_version() self.check_changelog_file() self.check_ci_file() @@ -562,7 +559,7 @@ def create_pr(self): # commit all code print_exec('git add sdk/') print_exec('git commit -m \"code and test\"') - print_exec('git push origin HEAD -f') + print_check('git push origin HEAD -f') # create PR self.create_pr_proc() From 79a553e181ba5df0ff1df165259a6804c5dbb7f3 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 17:35:48 +0800 Subject: [PATCH 21/78] update --- scripts/auto_release/main.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 840d003da8f6..3d56acdb7b67 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -44,10 +44,6 @@ def print_exec_output(cmd: str) -> List[str]: def print_check(cmd): log(cmd) subprocess.check_call(cmd, shell=True) - -def print_check_with_path(cmd, path): - log(cmd) - subprocess.check_call(cmd, shell=True, cwd=path) def preview_version_plus(preview_label: str, last_version: str) -> str: @@ -262,9 +258,10 @@ def edit_sdk_setup(content: List[str]): modify_file(str(Path(self.sdk_code_path()) / 'setup.py'), edit_sdk_setup) # Use the template to update readme and setup by packaging_tools + @return_origin_path def check_sdk_readme_and_setup(self): - path = str(Path(f'sdk/{self.sdk_folder}')) - print_check_with_path(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}',path) + os.chdir(Path(f'sdk/{self.sdk_folder}')) + print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}') log(f' packaging_tools --build-conf successfully ') def check_pprint_name(self): @@ -414,6 +411,8 @@ def check_ci_file(self): def check_file(self): self.check_pprint_name() self.check_sdk_readme_and_setup() + self.check_sdk_readme() + self.check_sdk_setup() self.check_version() self.check_changelog_file() self.check_ci_file() From ec747f10069fc9a93a933dc1873cc3d21b626a3e Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 17:36:48 +0800 Subject: [PATCH 22/78] test --- scripts/auto_release/main.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 3d56acdb7b67..301eb40de681 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -81,8 +81,10 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - usr = 'Azure' - branch = 'main' + # usr = 'Azure' + # branch = 'main' + usr = 'azclibot' + branch = 't2-advisor-2022-03-09-54263' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 6f51fc8d963e90275e951bbc9deac79d04aa25d7 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 17:48:34 +0800 Subject: [PATCH 23/78] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 301eb40de681..b58cec549b3e 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -241,7 +241,7 @@ def prepare_branch(self): # self.prepare_branch_with_base_branch() def check_sdk_readme(self): - sdk_readme = str(Path(f'sdk/{self.sdk_folder}/{self.package_name}/README.md')) + sdk_readme = str(Path(f'sdk/{self.sdk_folder}/azure-mgmt-{self.package_name}/README.md')) def edit_sdk_readme(content: List[str]): for i in range(0, len(content)): From 7107723a4d1ada144363da3cb3e0e03cecbb426a Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 9 Mar 2022 17:57:28 +0800 Subject: [PATCH 24/78] reduction --- scripts/auto_release/main.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index b58cec549b3e..347e9c4bc671 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -81,10 +81,8 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): - # usr = 'Azure' - # branch = 'main' - usr = 'azclibot' - branch = 't2-advisor-2022-03-09-54263' + usr = 'Azure' + branch = 'main' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 6885266bf40f30462904c8720dcdfb60e4dfb06d Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 11 Mar 2022 10:35:37 +0800 Subject: [PATCH 25/78] update main --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 347e9c4bc671..86667e0648aa 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -259,7 +259,7 @@ def edit_sdk_setup(content: List[str]): # Use the template to update readme and setup by packaging_tools @return_origin_path - def check_sdk_readme_and_setup(self): + def check_file_with_packaging_tool(self): os.chdir(Path(f'sdk/{self.sdk_folder}')) print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}') log(f' packaging_tools --build-conf successfully ') @@ -409,8 +409,8 @@ def check_ci_file(self): self.check_ci_file_proc('azure-mgmt-core>=1.3.0,<2.0.0') def check_file(self): + self.check_file_with_packaging_tool() self.check_pprint_name() - self.check_sdk_readme_and_setup() self.check_sdk_readme() self.check_sdk_setup() self.check_version() From 6c9bbbd6d39b9511ff5cece8fc7b7a78eb09f6bf Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 11 Mar 2022 13:26:47 +0800 Subject: [PATCH 26/78] delete f --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 86667e0648aa..1eba85942dde 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -262,7 +262,7 @@ def edit_sdk_setup(content: List[str]): def check_file_with_packaging_tool(self): os.chdir(Path(f'sdk/{self.sdk_folder}')) print_check(f'python -m packaging_tools --build-conf azure-mgmt-{self.package_name}') - log(f' packaging_tools --build-conf successfully ') + log(' packaging_tools --build-conf successfully ') def check_pprint_name(self): pprint_name = self.package_name.capitalize() From ac2f9e016d6f5d602f82ae887c32d024c7389fbc Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Thu, 17 Mar 2022 14:25:04 +0800 Subject: [PATCH 27/78] test --- scripts/auto_release/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 1eba85942dde..99a4edc84800 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,8 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'main' + # branch = 'main' + branch = 'test-resources-2022-03-17' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 6ed1348d0d0eb752a48dfd7846df98a528bee942 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Thu, 17 Mar 2022 14:37:06 +0800 Subject: [PATCH 28/78] revoke --- scripts/auto_release/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 99a4edc84800..1eba85942dde 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,8 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - # branch = 'main' - branch = 'test-resources-2022-03-17' + branch = 'main' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 6e6d69ea8d81b0f1953615ae46ecf65c551071b5 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Thu, 17 Mar 2022 14:39:28 +0800 Subject: [PATCH 29/78] test(change rest branch) --- scripts/auto_release/PythonSdkLiveTest.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index c5b16b57b579..9a23ee3eba80 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -46,6 +46,10 @@ jobs: if [ "$(SPEC_README)" ]; then mkdir azure-rest-api-specs git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs + cd azure-rest-api-specs + git fetch origin test-resources-2022-03-17 + git checkout test-resources-2022-03-17 + cd .. # install autorest sudo npm install -g n From 9419f1102be15bf584bf936e5954f9c38d045b99 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Thu, 17 Mar 2022 15:28:41 +0800 Subject: [PATCH 30/78] test --- scripts/auto_release/PythonSdkLiveTest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 9a23ee3eba80..a9b389822f09 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -47,8 +47,8 @@ jobs: mkdir azure-rest-api-specs git clone https://github.com/Azure/azure-rest-api-specs.git $(pwd)/azure-rest-api-specs cd azure-rest-api-specs - git fetch origin test-resources-2022-03-17 - git checkout test-resources-2022-03-17 + git fetch origin test-hdinsight-03-17 + git checkout test-hdinsight-03-17 cd .. # install autorest From 96bb9a35d6351e1b134d6a952e35c8e0441362fe Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Fri, 18 Mar 2022 15:22:48 +0800 Subject: [PATCH 31/78] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 1eba85942dde..66b71e3ed13f 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -82,7 +82,7 @@ def all_files(path: str, files: List[str]): def checkout_azure_default_branch(): usr = 'Azure' - branch = 'main' + branch = 'test_autorest_03_18' print_exec(f'git remote add {usr} https://github.com/{usr}/azure-sdk-for-python.git') print_check(f'git fetch {usr} {branch}') print_check(f'git checkout {usr}/{branch}') From 247224da60bce2e1b37ec089d2dd685563037c66 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 22 Mar 2022 15:50:19 +0800 Subject: [PATCH 32/78] test --- scripts/auto_release/main.py | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 1eba85942dde..4e1d66e6c28e 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -124,6 +124,37 @@ def set_test_env_var(): def start_test_proxy(): print_check('pwsh eng/common/testproxy/docker-start-proxy.ps1 \"start\"') +def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): + try: + process = subprocess.Popen( + cmd_line, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + universal_newlines=True, + cwd=cwd, + shell=shell, + env=env, + encoding="utf-8", + ) + output_buffer = [] + for line in process.stdout: + output_buffer.append(line.rstrip()) + _LOG.info(f"==[autorest22]" + output_buffer[-1]) + process.wait() + output = "\n".join(output_buffer) + if process.returncode: + # print necessary error info + for i in range(len(output_buffer)): + _LOG.error(f"[Autorest11] {output_buffer[i]}") + # print(f"[Autorest22] {output_buffer[i]}") + raise subprocess.CalledProcessError(process.returncode, cmd_line, output) + return output + except Exception as err: + _LOG.error(err) + raise + else: + _LOGGER.info("Return code: %s", process.returncode) + class CodegenTestPR: """ @@ -185,6 +216,14 @@ def generate_code(self): # generate code print_exec('python scripts/dev_setup.py -p azure-core') + cmd_line = ['/usr/local/bin/autorest', '/home/vsts/work/1/azure-rest-api-specs/specification/hdinsight/resource-manager/readme.md', + '--multiapi', '--python', '--python-mode=update', + '--python-sdks-folder=/home/vsts/work/1/azure-sdk-for-python/sdk', '--python3-only', '--track2', + '--use=@autorest/python@5.12.0', '--use=@autorest/modelerfour@4.19.3', '--version=3.7.2'] + + execute_simple_command(cmd_line) + print("*****************************************") + print_check(f'python -m packaging_tools.auto_codegen {self.autorest_result} {self.autorest_result}') print_check(f'python -m packaging_tools.auto_package {self.autorest_result} {self.autorest_result}') From baf973f870a593e3df81e58ce0781374c327225c Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 22 Mar 2022 16:40:56 +0800 Subject: [PATCH 33/78] test --- scripts/auto_release/main.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 0a98c9d29725..a840dcfb71a0 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -201,6 +201,14 @@ def get_sdk_folder_with_autorest_result(self): def generate_code(self): checkout_azure_default_branch() + print_exec('python scripts/dev_setup.py -p azure-core') + cmd_line = ['/usr/local/bin/autorest', '/home/vsts/work/1/azure-rest-api-specs/specification/hdinsight/resource-manager/readme.md', + '--multiapi', '--python', '--python-mode=update', + '--python-sdks-folder=/home/vsts/work/1/azure-sdk-for-python/sdk', '--python3-only', '--track2', + '--use=@autorest/python@5.12.0', '--use=@autorest/modelerfour@4.19.3', '--version=3.7.2'] + + execute_simple_command(cmd_line) + print("*****************************************") # prepare input data input_data = { @@ -215,17 +223,8 @@ def generate_code(self): json.dump(input_data, file) # generate code - print_exec('python scripts/dev_setup.py -p azure-core') - cmd_line = ['/usr/local/bin/autorest', '/home/vsts/work/1/azure-rest-api-specs/specification/hdinsight/resource-manager/readme.md', - '--multiapi', '--python', '--python-mode=update', - '--python-sdks-folder=/home/vsts/work/1/azure-sdk-for-python/sdk', '--python3-only', '--track2', - '--use=@autorest/python@5.12.0', '--use=@autorest/modelerfour@4.19.3', '--version=3.7.2'] - - execute_simple_command(cmd_line) - print("*****************************************") - - print_check(f'python -m packaging_tools.auto_codegen {self.autorest_result} {self.autorest_result}') - print_check(f'python -m packaging_tools.auto_package {self.autorest_result} {self.autorest_result}') + # print_check(f'python -m packaging_tools.auto_codegen {self.autorest_result} {self.autorest_result}') + # print_check(f'python -m packaging_tools.auto_package {self.autorest_result} {self.autorest_result}') def get_package_name_with_autorest_result(self): generate_result = self.get_autorest_result() From c1d7c576113bb479e0d77f29ab99b62d39df5f32 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 22 Mar 2022 16:50:55 +0800 Subject: [PATCH 34/78] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index a840dcfb71a0..6ce9b94b20c8 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -614,7 +614,7 @@ def run(self): if __name__ == '__main__': main_logger = logging.getLogger() logging.basicConfig() - main_logger.setLevel(logging.INFO) + main_logger.setLevel(logging.DEBUG) instance = CodegenTestPR() instance.run() From 8480e50b99a72ca77196260868e3cae6d1e0cba6 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 22 Mar 2022 16:55:56 +0800 Subject: [PATCH 35/78] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 6ce9b94b20c8..f0eff3fba7bc 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -205,7 +205,7 @@ def generate_code(self): cmd_line = ['/usr/local/bin/autorest', '/home/vsts/work/1/azure-rest-api-specs/specification/hdinsight/resource-manager/readme.md', '--multiapi', '--python', '--python-mode=update', '--python-sdks-folder=/home/vsts/work/1/azure-sdk-for-python/sdk', '--python3-only', '--track2', - '--use=@autorest/python@5.12.0', '--use=@autorest/modelerfour@4.19.3', '--version=3.7.2'] + '--use=@autorest/python@5.8.1', '--use=@autorest/modelerfour@4.19.2', '--version=3.4.2'] execute_simple_command(cmd_line) print("*****************************************") From 34d802af62d79af23926a90fa797c1cd978cb0cc Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Tue, 22 Mar 2022 17:50:57 +0800 Subject: [PATCH 36/78] test --- scripts/auto_release/main.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index f0eff3fba7bc..e232ff24d88f 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -137,10 +137,11 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): encoding="utf-8", ) output_buffer = [] + process.wait() for line in process.stdout: output_buffer.append(line.rstrip()) _LOG.info(f"==[autorest22]" + output_buffer[-1]) - process.wait() + # process.wait() output = "\n".join(output_buffer) if process.returncode: # print necessary error info @@ -205,7 +206,7 @@ def generate_code(self): cmd_line = ['/usr/local/bin/autorest', '/home/vsts/work/1/azure-rest-api-specs/specification/hdinsight/resource-manager/readme.md', '--multiapi', '--python', '--python-mode=update', '--python-sdks-folder=/home/vsts/work/1/azure-sdk-for-python/sdk', '--python3-only', '--track2', - '--use=@autorest/python@5.8.1', '--use=@autorest/modelerfour@4.19.2', '--version=3.4.2'] + '--use=@autorest/python@5.12.0', '--use=@autorest/modelerfour@4.19.3', '--version=3.7.2'] execute_simple_command(cmd_line) print("*****************************************") From f4c0f63f6bde77ff5fc748d8c61622574f3d6ba3 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 09:24:36 +0800 Subject: [PATCH 37/78] test --- scripts/auto_release/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index e232ff24d88f..5cea57671bf8 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -126,7 +126,7 @@ def start_test_proxy(): def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: - process = subprocess.Popen( + process = subprocess.check_call( cmd_line, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, @@ -140,13 +140,13 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): process.wait() for line in process.stdout: output_buffer.append(line.rstrip()) - _LOG.info(f"==[autorest22]" + output_buffer[-1]) + _LOG.info(f"==[autorest33]" + output_buffer[-1]) # process.wait() output = "\n".join(output_buffer) if process.returncode: # print necessary error info for i in range(len(output_buffer)): - _LOG.error(f"[Autorest11] {output_buffer[i]}") + _LOG.error(f"[Autorest44] {output_buffer[i]}") # print(f"[Autorest22] {output_buffer[i]}") raise subprocess.CalledProcessError(process.returncode, cmd_line, output) return output From 3044e8fa1d2bfccdfe2487b9bae1f4dcd4bbbf86 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 11:43:38 +0800 Subject: [PATCH 38/78] test --- scripts/auto_release/main.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 5cea57671bf8..50d16b410626 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -126,15 +126,8 @@ def start_test_proxy(): def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: - process = subprocess.check_call( - cmd_line, - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, - universal_newlines=True, - cwd=cwd, - shell=shell, - env=env, - encoding="utf-8", + process = subprocess.getoutput( + cmd_line ) output_buffer = [] process.wait() From 388e8ca7cc59d7ed06e34f8a71055597b7bed8df Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 11:48:47 +0800 Subject: [PATCH 39/78] test --- scripts/auto_release/main.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 50d16b410626..5fd9e73de0ab 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -130,7 +130,6 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): cmd_line ) output_buffer = [] - process.wait() for line in process.stdout: output_buffer.append(line.rstrip()) _LOG.info(f"==[autorest33]" + output_buffer[-1]) From 175d237b01425a8a442f01a4620d5592426c8c07 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 13:49:40 +0800 Subject: [PATCH 40/78] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 5fd9e73de0ab..dd4087ca40cc 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -130,7 +130,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): cmd_line ) output_buffer = [] - for line in process.stdout: + for line in process: output_buffer.append(line.rstrip()) _LOG.info(f"==[autorest33]" + output_buffer[-1]) # process.wait() From 5e3b856fcbb2d8efbd57b36e2b116f551dd4117d Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 13:54:39 +0800 Subject: [PATCH 41/78] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index dd4087ca40cc..1c853fd994d4 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -132,7 +132,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): output_buffer = [] for line in process: output_buffer.append(line.rstrip()) - _LOG.info(f"==[autorest33]" + output_buffer[-1]) + _LOG.info(f"==[autorest33]" + output_buffer) # process.wait() output = "\n".join(output_buffer) if process.returncode: From 0a901ef8b3669229b9d7192860f32ccfdf1ffcba Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 14:00:53 +0800 Subject: [PATCH 42/78] test --- scripts/auto_release/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 1c853fd994d4..c0f82845b841 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -130,9 +130,10 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): cmd_line ) output_buffer = [] - for line in process: - output_buffer.append(line.rstrip()) - _LOG.info(f"==[autorest33]" + output_buffer) + # for line in process: + # output_buffer.append(line.rstrip()) + # _LOG.info(f"==[autorest33]" + output_buffer) + _LOG.info(f"==[autorest33]" + process) # process.wait() output = "\n".join(output_buffer) if process.returncode: From b49882b7bebf91d573f2badf160d6cb953471899 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 14:21:12 +0800 Subject: [PATCH 43/78] test --- scripts/auto_release/main.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index c0f82845b841..7f92007c73b0 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -126,14 +126,20 @@ def start_test_proxy(): def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: - process = subprocess.getoutput( - cmd_line + process = subprocess.call( + cmd_line, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + universal_newlines=True, + cwd=cwd, + shell=shell, + env=env, ) output_buffer = [] - # for line in process: - # output_buffer.append(line.rstrip()) - # _LOG.info(f"==[autorest33]" + output_buffer) - _LOG.info(f"==[autorest33]" + process) + process.wait() + for line in process.stdout: + output_buffer.append(line.rstrip()) + _LOG.info(f"==[autorest33]" + output_buffer[-1]) # process.wait() output = "\n".join(output_buffer) if process.returncode: From 4b263a6892d901450402e0b39d01d856d08598a4 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 14:27:05 +0800 Subject: [PATCH 44/78] test --- scripts/auto_release/main.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 7f92007c73b0..aa1caf219358 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -132,14 +132,15 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): stdout=subprocess.PIPE, universal_newlines=True, cwd=cwd, - shell=shell, + shell=True, env=env, ) output_buffer = [] - process.wait() - for line in process.stdout: + # process.wait() + for line in process: output_buffer.append(line.rstrip()) - _LOG.info(f"==[autorest33]" + output_buffer[-1]) + _LOG.info(f"==[autorest33]" + line.rstrip()) + # _LOG.info(f"==[autorest33]" + output_buffer[-1]) # process.wait() output = "\n".join(output_buffer) if process.returncode: From efbdae96369b24c801afcb3a5b71777a10a6ce38 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 14:42:08 +0800 Subject: [PATCH 45/78] test --- scripts/auto_release/main.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index aa1caf219358..861144d1f604 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -126,22 +126,21 @@ def start_test_proxy(): def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: - process = subprocess.call( + process = subprocess.Popen( cmd_line, - stderr=subprocess.STDOUT, + stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True, cwd=cwd, - shell=True, + shell=shell, env=env, + encoding="utf-8", ) output_buffer = [] - # process.wait() - for line in process: + for line in process.stdout: output_buffer.append(line.rstrip()) - _LOG.info(f"==[autorest33]" + line.rstrip()) - # _LOG.info(f"==[autorest33]" + output_buffer[-1]) - # process.wait() + _LOG.info(f"==[autorest333]" + output_buffer[-1]) + process.wait() output = "\n".join(output_buffer) if process.returncode: # print necessary error info From 8ab6c9ac3bc3774ba7dfd0c68ff2dce29718dec3 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 14:49:17 +0800 Subject: [PATCH 46/78] test --- scripts/auto_release/main.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 861144d1f604..887e12006178 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -137,10 +137,13 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): encoding="utf-8", ) output_buffer = [] + process.wait() for line in process.stdout: output_buffer.append(line.rstrip()) - _LOG.info(f"==[autorest333]" + output_buffer[-1]) - process.wait() + _LOG.info(f"==[autorest33]" + output_buffer[-1]) + for line in process.stderr: + print(f"==[autorest333]" + line) + output = "\n".join(output_buffer) if process.returncode: # print necessary error info From 78f609e1600fa0f855b409144a0957450013911c Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 14:56:40 +0800 Subject: [PATCH 47/78] test --- scripts/auto_release/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 887e12006178..d5135570170f 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -137,7 +137,10 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): encoding="utf-8", ) output_buffer = [] - process.wait() + stdout, stderr = process.communicate() + print(f'== {stdout=}') + print(f'== {stderr=}') + # process.wait() for line in process.stdout: output_buffer.append(line.rstrip()) _LOG.info(f"==[autorest33]" + output_buffer[-1]) From bc67be4ec60bca80f554f0fae97d35e93e0a542a Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 17:08:35 +0800 Subject: [PATCH 48/78] test --- scripts/auto_release/main.py | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index d5135570170f..c502bad1f949 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -128,35 +128,32 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: process = subprocess.Popen( cmd_line, - stderr=subprocess.PIPE, + stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True, cwd=cwd, shell=shell, env=env, - encoding="utf-8", + encoding='utf-8' ) output_buffer = [] - stdout, stderr = process.communicate() - print(f'== {stdout=}') - print(f'== {stderr=}') - # process.wait() + time.sleep(20) for line in process.stdout: output_buffer.append(line.rstrip()) - _LOG.info(f"==[autorest33]" + output_buffer[-1]) - for line in process.stderr: - print(f"==[autorest333]" + line) - + _LOGGER.info(f"==[autorest222]" + output_buffer[-1]) + process.wait() output = "\n".join(output_buffer) + if process.returncode: + print(f'++++ {process.returncode}') # print necessary error info for i in range(len(output_buffer)): - _LOG.error(f"[Autorest44] {output_buffer[i]}") + _LOGGER.error(f"[Autorest111] {output_buffer[i]}") # print(f"[Autorest22] {output_buffer[i]}") raise subprocess.CalledProcessError(process.returncode, cmd_line, output) return output except Exception as err: - _LOG.error(err) + _LOGGER.error(err) raise else: _LOGGER.info("Return code: %s", process.returncode) From 3106a700e5489b5826620862d1471741d376ed5d Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 17:13:50 +0800 Subject: [PATCH 49/78] test --- scripts/auto_release/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index c502bad1f949..0c2d816fd0f4 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -140,7 +140,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): time.sleep(20) for line in process.stdout: output_buffer.append(line.rstrip()) - _LOGGER.info(f"==[autorest222]" + output_buffer[-1]) + _LOG.info(f"==[autorest222]" + output_buffer[-1]) process.wait() output = "\n".join(output_buffer) @@ -148,7 +148,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): print(f'++++ {process.returncode}') # print necessary error info for i in range(len(output_buffer)): - _LOGGER.error(f"[Autorest111] {output_buffer[i]}") + _LOG.error(f"[Autorest111] {output_buffer[i]}") # print(f"[Autorest22] {output_buffer[i]}") raise subprocess.CalledProcessError(process.returncode, cmd_line, output) return output From a76690e36d75929066a706d14a59d43daae10dae Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Wed, 23 Mar 2022 17:22:19 +0800 Subject: [PATCH 50/78] test --- scripts/auto_release/main.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 0c2d816fd0f4..4ea4e8e73f07 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -137,10 +137,13 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): encoding='utf-8' ) output_buffer = [] - time.sleep(20) + time.sleep(30) + print("process.stdout") + print(list(process.stdout)) + print("***************") for line in process.stdout: output_buffer.append(line.rstrip()) - _LOG.info(f"==[autorest222]" + output_buffer[-1]) + _LOG.info(f"==[autorest22]" + output_buffer[-1]) process.wait() output = "\n".join(output_buffer) @@ -148,12 +151,12 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): print(f'++++ {process.returncode}') # print necessary error info for i in range(len(output_buffer)): - _LOG.error(f"[Autorest111] {output_buffer[i]}") + _LOG.error(f"[Autorest11] {output_buffer[i]}") # print(f"[Autorest22] {output_buffer[i]}") raise subprocess.CalledProcessError(process.returncode, cmd_line, output) return output except Exception as err: - _LOGGER.error(err) + _LOG.error(err) raise else: _LOGGER.info("Return code: %s", process.returncode) From 84ad7c44aed29c29b2bb4b82e80fcbd650a4ee85 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Thu, 24 Mar 2022 13:50:11 +0800 Subject: [PATCH 51/78] test --- scripts/auto_release/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 4ea4e8e73f07..cab1607557c8 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -128,6 +128,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: process = subprocess.Popen( cmd_line, + bufsize=5, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True, From 295641d722eceb34a5230179986a18f83ff3593c Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Thu, 24 Mar 2022 14:26:57 +0800 Subject: [PATCH 52/78] test --- scripts/auto_release/main.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index cab1607557c8..a0c9a486288a 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -138,14 +138,15 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): encoding='utf-8' ) output_buffer = [] - time.sleep(30) + process.wait() print("process.stdout") - print(list(process.stdout)) + for line in list(process.stdout): + print(line, end='') print("***************") for line in process.stdout: output_buffer.append(line.rstrip()) _LOG.info(f"==[autorest22]" + output_buffer[-1]) - process.wait() + output = "\n".join(output_buffer) if process.returncode: From f852b38327513c9c246795966d4e81b1b40f3b90 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Thu, 24 Mar 2022 14:37:31 +0800 Subject: [PATCH 53/78] test --- scripts/auto_release/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index a0c9a486288a..e70bf1bc343b 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -128,9 +128,8 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: process = subprocess.Popen( cmd_line, - bufsize=5, - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, + # stderr=subprocess.STDOUT, + # stdout=subprocess.PIPE, universal_newlines=True, cwd=cwd, shell=shell, From ee7893e6ebe5aed182f65f4dbaf574071ae7c642 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Thu, 24 Mar 2022 15:47:12 +0800 Subject: [PATCH 54/78] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index a9b389822f09..46decc7b7dcd 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -21,7 +21,7 @@ variables: jobs: - job: LiveTestPython38 displayName: Live Test Python 3.8 - timeoutInMinutes: 1500 + timeoutInMinutes: 3000 strategy: maxParallel: 5 pool: From e26375c8eb3f4372ab08ea4e516728898bbbebca Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Thu, 24 Mar 2022 15:56:39 +0800 Subject: [PATCH 55/78] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 46decc7b7dcd..331a04efe092 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -21,9 +21,9 @@ variables: jobs: - job: LiveTestPython38 displayName: Live Test Python 3.8 - timeoutInMinutes: 3000 + timeoutInMinutes: 5000 strategy: - maxParallel: 5 + maxParallel: 7 pool: vmImage: 'ubuntu-20.04' steps: From d1dae793f36a5e45085dbbdc2cc17a1d6dd901b9 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Thu, 24 Mar 2022 16:15:08 +0800 Subject: [PATCH 56/78] test --- scripts/auto_release/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index e70bf1bc343b..7fb9e0b4bded 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -126,6 +126,11 @@ def start_test_proxy(): def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: + c = ' '.join(cmd_line) + print(f'{c=}') + p = subprocess.getoutput(c) + print(p) + process = subprocess.Popen( cmd_line, # stderr=subprocess.STDOUT, From f1060c1f90b921bced60ac97eb1dcca12614f596 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 10:27:35 +0800 Subject: [PATCH 57/78] test --- scripts/auto_release/main.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 7fb9e0b4bded..e00da20d3cbe 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -127,11 +127,23 @@ def start_test_proxy(): def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: c = ' '.join(cmd_line) - print(f'{c=}') - p = subprocess.getoutput(c) - print(p) + print(f'c={c}') + data = subprocess.check_output(c, shell=True, text=True, stderr=subprocess.STDOUT) + ps = data.split('\n') + print('+++++++++++++++++++') + for line in ps[len(ps) - 6:len(ps)]: + print(line) + print('++++++++++++++++++++') + except subprocess.CalledProcessError as ex: + data = ex.output + if data[-1:] == '\n': + data = data[:-1] + print('+++++++++++++++++++') + print(data.split('\n')) + print('++++++++++++++++++++') - process = subprocess.Popen( + try: + ss = subprocess.Popen( cmd_line, # stderr=subprocess.STDOUT, # stdout=subprocess.PIPE, From 367fd9fd4a11b588f80d796acbddca91bb2a357d Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 10:40:59 +0800 Subject: [PATCH 58/78] test --- scripts/auto_release/main.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index e00da20d3cbe..2557c07d988e 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -136,14 +136,13 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): print('++++++++++++++++++++') except subprocess.CalledProcessError as ex: data = ex.output - if data[-1:] == '\n': - data = data[:-1] - print('+++++++++++++++++++') - print(data.split('\n')) - print('++++++++++++++++++++') + print('-------------------------') + for line in data.split(): + print(line) + print('-------------------------') try: - ss = subprocess.Popen( + process = subprocess.Popen( cmd_line, # stderr=subprocess.STDOUT, # stdout=subprocess.PIPE, From 75f2b5d746acd256a6bdc7ac1a57f646c0a63fb2 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 10:47:25 +0800 Subject: [PATCH 59/78] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 2557c07d988e..31428a59a00f 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -137,7 +137,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): except subprocess.CalledProcessError as ex: data = ex.output print('-------------------------') - for line in data.split(): + for line in data.split('\n'): print(line) print('-------------------------') From ed3f34fa14b872de92cf7b6b285cf8507c493c5e Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 13:54:12 +0800 Subject: [PATCH 60/78] test --- scripts/auto_release/main.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 31428a59a00f..a424077ed68e 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -125,27 +125,27 @@ def start_test_proxy(): print_check('pwsh eng/common/testproxy/docker-start-proxy.ps1 \"start\"') def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): - try: - c = ' '.join(cmd_line) - print(f'c={c}') - data = subprocess.check_output(c, shell=True, text=True, stderr=subprocess.STDOUT) - ps = data.split('\n') - print('+++++++++++++++++++') - for line in ps[len(ps) - 6:len(ps)]: - print(line) - print('++++++++++++++++++++') - except subprocess.CalledProcessError as ex: - data = ex.output - print('-------------------------') - for line in data.split('\n'): - print(line) - print('-------------------------') + # try: + # c = ' '.join(cmd_line) + # print(f'c={c}') + # data = subprocess.check_output(c, shell=True, text=True, stderr=subprocess.STDOUT) + # ps = data.split('\n') + # print('+++++++++++++++++++') + # for line in ps[len(ps) - 6:len(ps)]: + # print(line) + # print('++++++++++++++++++++') + # except subprocess.CalledProcessError as ex: + # data = ex.output + # print('-------------------------') + # for line in data.split('\n'): + # print(line) + # print('-------------------------') try: process = subprocess.Popen( cmd_line, - # stderr=subprocess.STDOUT, - # stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, universal_newlines=True, cwd=cwd, shell=shell, @@ -153,7 +153,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): encoding='utf-8' ) output_buffer = [] - process.wait() + process.communicate() print("process.stdout") for line in list(process.stdout): print(line, end='') From 3ab5308070cf370dc5549877a4d75538305bdb4c Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 14:01:52 +0800 Subject: [PATCH 61/78] test --- scripts/auto_release/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index a424077ed68e..ce6f40bb832c 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -153,10 +153,10 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): encoding='utf-8' ) output_buffer = [] - process.communicate() + stdout, stderr = process.communicate() print("process.stdout") - for line in list(process.stdout): - print(line, end='') + print("stdout: ", stdout) + print("stderr: ", stderr) print("***************") for line in process.stdout: output_buffer.append(line.rstrip()) From 9f0b8da09fa3d27d8c5f9e27ebc35f3bb28072a8 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 14:21:57 +0800 Subject: [PATCH 62/78] test --- scripts/auto_release/main.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index ce6f40bb832c..0996cf856a3c 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -153,10 +153,10 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): encoding='utf-8' ) output_buffer = [] - stdout, stderr = process.communicate() + process.wait() print("process.stdout") - print("stdout: ", stdout) - print("stderr: ", stderr) + print("stdout: ", process.stdout) + print("stderr: ", process.stderr) print("***************") for line in process.stdout: output_buffer.append(line.rstrip()) From 375ccd5163cce035d5b06050b7dc26ee52473f59 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 14:33:06 +0800 Subject: [PATCH 63/78] test --- scripts/auto_release/main.py | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 0996cf856a3c..9389544a36f9 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -155,23 +155,33 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): output_buffer = [] process.wait() print("process.stdout") - print("stdout: ", process.stdout) - print("stderr: ", process.stderr) + print("stdout: ", list(process.stdout)) + print("stderr: ", list(process.stderr)) print("***************") for line in process.stdout: output_buffer.append(line.rstrip()) _LOG.info(f"==[autorest22]" + output_buffer[-1]) output = "\n".join(output_buffer) + if not process.returncode: + return output + except subprocess.CalledProcessError as ex: + data = ex.output + print('-------------------------') + for line in data.split('\n'): + print(line) + print('-------------------------') + raise subprocess.CalledProcessError(process.returncode, cmd_line) + + + # if process.returncode: + # print(f'++++ {process.returncode}') + # # print necessary error info + # for i in range(len(output_buffer)): + # _LOG.error(f"[Autorest11] {output_buffer[i]}") + # # print(f"[Autorest22] {output_buffer[i]}") + # raise subprocess.CalledProcessError(process.returncode, cmd_line, output) - if process.returncode: - print(f'++++ {process.returncode}') - # print necessary error info - for i in range(len(output_buffer)): - _LOG.error(f"[Autorest11] {output_buffer[i]}") - # print(f"[Autorest22] {output_buffer[i]}") - raise subprocess.CalledProcessError(process.returncode, cmd_line, output) - return output except Exception as err: _LOG.error(err) raise From e378ec81b4bcb860914c7019f5a9e0b38a0959ee Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 14:39:19 +0800 Subject: [PATCH 64/78] test --- scripts/auto_release/main.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 9389544a36f9..743988c7833c 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -163,8 +163,13 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): _LOG.info(f"==[autorest22]" + output_buffer[-1]) output = "\n".join(output_buffer) - if not process.returncode: - return output + if process.returncode: + print(f'++++ {process.returncode}') + # print necessary error info + for i in range(len(output_buffer)): + _LOG.error(f"[Autorest11] {output_buffer[i]}") + # print(f"[Autorest22] {output_buffer[i]}") + raise subprocess.CalledProcessError(process.returncode, cmd_line, output) except subprocess.CalledProcessError as ex: data = ex.output print('-------------------------') @@ -172,21 +177,11 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): print(line) print('-------------------------') raise subprocess.CalledProcessError(process.returncode, cmd_line) - - - # if process.returncode: - # print(f'++++ {process.returncode}') - # # print necessary error info - # for i in range(len(output_buffer)): - # _LOG.error(f"[Autorest11] {output_buffer[i]}") - # # print(f"[Autorest22] {output_buffer[i]}") - # raise subprocess.CalledProcessError(process.returncode, cmd_line, output) - except Exception as err: _LOG.error(err) raise else: - _LOGGER.info("Return code: %s", process.returncode) + _LOG.info("Return code: %s", process.returncode) class CodegenTestPR: From 86225ad7b9014c24d89f4415d7d023da866f3c5a Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 14:48:54 +0800 Subject: [PATCH 65/78] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 743988c7833c..8e1b5753bfa7 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -169,7 +169,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): for i in range(len(output_buffer)): _LOG.error(f"[Autorest11] {output_buffer[i]}") # print(f"[Autorest22] {output_buffer[i]}") - raise subprocess.CalledProcessError(process.returncode, cmd_line, output) + raise subprocess.CalledProcessError(process.returncode, cmd_line, output=process.stdout) except subprocess.CalledProcessError as ex: data = ex.output print('-------------------------') From 45938b9a3b199b909f7b0dba8227aa9affeeabfe Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 14:55:14 +0800 Subject: [PATCH 66/78] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 8e1b5753bfa7..0a44d573135a 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -173,7 +173,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): except subprocess.CalledProcessError as ex: data = ex.output print('-------------------------') - for line in data.split('\n'): + for line in data: print(line) print('-------------------------') raise subprocess.CalledProcessError(process.returncode, cmd_line) From 5c25421d76eae9e5de4fa75ae0868afc3d45479b Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 15:02:59 +0800 Subject: [PATCH 67/78] test --- scripts/auto_release/main.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 0a44d573135a..21aa84fe6fb1 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -154,20 +154,20 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): ) output_buffer = [] process.wait() - print("process.stdout") - print("stdout: ", list(process.stdout)) - print("stderr: ", list(process.stderr)) + # print("process.stdout") + # print("stdout: ", list(process.stdout)) + # print("stderr: ", list(process.stderr)) print("***************") - for line in process.stdout: - output_buffer.append(line.rstrip()) - _LOG.info(f"==[autorest22]" + output_buffer[-1]) + # for line in process.stdout: + # output_buffer.append(line.rstrip()) + # _LOG.info(f"==[autorest22]" + output_buffer[-1]) - output = "\n".join(output_buffer) + # output = "\n".join(output_buffer) if process.returncode: - print(f'++++ {process.returncode}') - # print necessary error info - for i in range(len(output_buffer)): - _LOG.error(f"[Autorest11] {output_buffer[i]}") + # print(f'++++ {process.returncode}') + # # print necessary error info + # for i in range(len(output_buffer)): + # _LOG.error(f"[Autorest11] {output_buffer[i]}") # print(f"[Autorest22] {output_buffer[i]}") raise subprocess.CalledProcessError(process.returncode, cmd_line, output=process.stdout) except subprocess.CalledProcessError as ex: From 63939d485439e13bcbaf7bc12a9d83cc749eeb02 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 15:14:57 +0800 Subject: [PATCH 68/78] test --- scripts/auto_release/main.py | 38 ++++++++++++++++++------------------ 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 21aa84fe6fb1..fda8669a0661 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -125,21 +125,21 @@ def start_test_proxy(): print_check('pwsh eng/common/testproxy/docker-start-proxy.ps1 \"start\"') def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): - # try: - # c = ' '.join(cmd_line) - # print(f'c={c}') - # data = subprocess.check_output(c, shell=True, text=True, stderr=subprocess.STDOUT) - # ps = data.split('\n') - # print('+++++++++++++++++++') - # for line in ps[len(ps) - 6:len(ps)]: - # print(line) - # print('++++++++++++++++++++') - # except subprocess.CalledProcessError as ex: - # data = ex.output - # print('-------------------------') - # for line in data.split('\n'): - # print(line) - # print('-------------------------') + try: + c = ' '.join(cmd_line) + print(f'c={c}') + data = subprocess.check_output(c, shell=True, text=True, stderr=subprocess.STDOUT) + ps = data.split('\n') + print('+++++++++++++++++++') + for line in ps[len(ps) - 6:len(ps)]: + print(line) + print('++++++++++++++++++++') + except subprocess.CalledProcessError as ex: + data = ex.output + print('-------------------------') + for line in data.split('\n'): + print(line) + print('-------------------------') try: process = subprocess.Popen( @@ -172,10 +172,10 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): raise subprocess.CalledProcessError(process.returncode, cmd_line, output=process.stdout) except subprocess.CalledProcessError as ex: data = ex.output - print('-------------------------') - for line in data: - print(line) - print('-------------------------') + # print('-------------------------') + # for line in data: + # print(line) + # print('-------------------------') raise subprocess.CalledProcessError(process.returncode, cmd_line) except Exception as err: _LOG.error(err) From 849ea9f4925bd2efff8f8761b681a3684a8c30a3 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 15:30:59 +0800 Subject: [PATCH 69/78] test --- scripts/auto_release/main.py | 43 +++++++++++------------------------- 1 file changed, 13 insertions(+), 30 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index fda8669a0661..3b86c4dac189 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -128,12 +128,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: c = ' '.join(cmd_line) print(f'c={c}') - data = subprocess.check_output(c, shell=True, text=True, stderr=subprocess.STDOUT) - ps = data.split('\n') - print('+++++++++++++++++++') - for line in ps[len(ps) - 6:len(ps)]: - print(line) - print('++++++++++++++++++++') + data = subprocess.getoutput(c) except subprocess.CalledProcessError as ex: data = ex.output print('-------------------------') @@ -144,44 +139,32 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: process = subprocess.Popen( cmd_line, - stderr=subprocess.PIPE, + stderr=subprocess.STDOUT, stdout=subprocess.PIPE, universal_newlines=True, cwd=cwd, shell=shell, env=env, - encoding='utf-8' + encoding="utf-8", ) output_buffer = [] + for line in process.stdout: + output_buffer.append(line.rstrip()) + _LOG.info(f"==[autorest55]" + output_buffer[-1]) process.wait() - # print("process.stdout") - # print("stdout: ", list(process.stdout)) - # print("stderr: ", list(process.stderr)) - print("***************") - # for line in process.stdout: - # output_buffer.append(line.rstrip()) - # _LOG.info(f"==[autorest22]" + output_buffer[-1]) - - # output = "\n".join(output_buffer) + output = "\n".join(output_buffer) if process.returncode: - # print(f'++++ {process.returncode}') - # # print necessary error info - # for i in range(len(output_buffer)): - # _LOG.error(f"[Autorest11] {output_buffer[i]}") + # print necessary error info + for i in range(len(output_buffer)): + _LOG.error(f"[Autorest66] {output_buffer[i]}") # print(f"[Autorest22] {output_buffer[i]}") - raise subprocess.CalledProcessError(process.returncode, cmd_line, output=process.stdout) - except subprocess.CalledProcessError as ex: - data = ex.output - # print('-------------------------') - # for line in data: - # print(line) - # print('-------------------------') - raise subprocess.CalledProcessError(process.returncode, cmd_line) + raise subprocess.CalledProcessError(process.returncode, cmd_line, output) + return output except Exception as err: _LOG.error(err) raise else: - _LOG.info("Return code: %s", process.returncode) + _LOGGER.info("Return code: %s", process.returncode) class CodegenTestPR: From f00bc560f8f6208bee1318bafab87c805e8cb2c4 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 16:01:39 +0800 Subject: [PATCH 70/78] test --- scripts/auto_release/main.py | 58 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 3b86c4dac189..46a3f6b9f0eb 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -136,35 +136,35 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): print(line) print('-------------------------') - try: - process = subprocess.Popen( - cmd_line, - stderr=subprocess.STDOUT, - stdout=subprocess.PIPE, - universal_newlines=True, - cwd=cwd, - shell=shell, - env=env, - encoding="utf-8", - ) - output_buffer = [] - for line in process.stdout: - output_buffer.append(line.rstrip()) - _LOG.info(f"==[autorest55]" + output_buffer[-1]) - process.wait() - output = "\n".join(output_buffer) - if process.returncode: - # print necessary error info - for i in range(len(output_buffer)): - _LOG.error(f"[Autorest66] {output_buffer[i]}") - # print(f"[Autorest22] {output_buffer[i]}") - raise subprocess.CalledProcessError(process.returncode, cmd_line, output) - return output - except Exception as err: - _LOG.error(err) - raise - else: - _LOGGER.info("Return code: %s", process.returncode) + # try: + # process = subprocess.Popen( + # cmd_line, + # stderr=subprocess.STDOUT, + # stdout=subprocess.PIPE, + # universal_newlines=True, + # cwd=cwd, + # shell=shell, + # env=env, + # encoding="utf-8", + # ) + # output_buffer = [] + # for line in process.stdout: + # output_buffer.append(line.rstrip()) + # _LOG.info(f"==[autorest55]" + output_buffer[-1]) + # process.wait() + # output = "\n".join(output_buffer) + # if process.returncode: + # # print necessary error info + # for i in range(len(output_buffer)): + # _LOG.error(f"[Autorest66] {output_buffer[i]}") + # # print(f"[Autorest22] {output_buffer[i]}") + # raise subprocess.CalledProcessError(process.returncode, cmd_line, output) + # return output + # except Exception as err: + # _LOG.error(err) + # raise + # else: + # _LOGGER.info("Return code: %s", process.returncode) class CodegenTestPR: From dbf6a6ecbba1fd2a0ec6fdb933502c66d85c6c2b Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 16:10:50 +0800 Subject: [PATCH 71/78] test --- scripts/auto_release/main.py | 63 ++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 46a3f6b9f0eb..fc0af978d511 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -126,8 +126,9 @@ def start_test_proxy(): def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: - c = ' '.join(cmd_line) - print(f'c={c}') + # c = ' '.join(cmd_line) + # print(f'c={c}') + c = 'autorest --version' data = subprocess.getoutput(c) except subprocess.CalledProcessError as ex: data = ex.output @@ -136,35 +137,35 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): print(line) print('-------------------------') - # try: - # process = subprocess.Popen( - # cmd_line, - # stderr=subprocess.STDOUT, - # stdout=subprocess.PIPE, - # universal_newlines=True, - # cwd=cwd, - # shell=shell, - # env=env, - # encoding="utf-8", - # ) - # output_buffer = [] - # for line in process.stdout: - # output_buffer.append(line.rstrip()) - # _LOG.info(f"==[autorest55]" + output_buffer[-1]) - # process.wait() - # output = "\n".join(output_buffer) - # if process.returncode: - # # print necessary error info - # for i in range(len(output_buffer)): - # _LOG.error(f"[Autorest66] {output_buffer[i]}") - # # print(f"[Autorest22] {output_buffer[i]}") - # raise subprocess.CalledProcessError(process.returncode, cmd_line, output) - # return output - # except Exception as err: - # _LOG.error(err) - # raise - # else: - # _LOGGER.info("Return code: %s", process.returncode) + try: + process = subprocess.Popen( + cmd_line, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE, + universal_newlines=True, + cwd=cwd, + shell=shell, + env=env, + encoding="utf-8", + ) + output_buffer = [] + for line in process.stdout: + output_buffer.append(line.rstrip()) + _LOG.info(f"==[autorest55]" + output_buffer[-1]) + process.wait() + output = "\n".join(output_buffer) + if process.returncode: + # print necessary error info + for i in range(len(output_buffer)): + _LOG.error(f"[Autorest66] {output_buffer[i]}") + # print(f"[Autorest22] {output_buffer[i]}") + raise subprocess.CalledProcessError(process.returncode, cmd_line, output) + return output + except Exception as err: + _LOG.error(err) + raise + else: + _LOGGER.info("Return code: %s", process.returncode) class CodegenTestPR: From f88fe1850cfd39561252c2691e157616b7a9122a Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Fri, 25 Mar 2022 16:22:14 +0800 Subject: [PATCH 72/78] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 331a04efe092..8975d5e1ecb7 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -57,6 +57,8 @@ jobs: export PATH=/usr/local/n/versions/node/14.15.0/bin:$PATH sudo npm init -f sudo npm install -g autorest --save + sudo npm install autorest.python@5.12.0 --save + sudo npm install autorest.modelerfour@4.19.3 --save mkdir temp_folder From bb26d2c9927e393d7e360f5a5ac53782762f49d0 Mon Sep 17 00:00:00 2001 From: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> Date: Fri, 25 Mar 2022 16:27:34 +0800 Subject: [PATCH 73/78] Update PythonSdkLiveTest.yml for Azure Pipelines --- scripts/auto_release/PythonSdkLiveTest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/auto_release/PythonSdkLiveTest.yml b/scripts/auto_release/PythonSdkLiveTest.yml index 8975d5e1ecb7..82d3f009e075 100644 --- a/scripts/auto_release/PythonSdkLiveTest.yml +++ b/scripts/auto_release/PythonSdkLiveTest.yml @@ -57,8 +57,8 @@ jobs: export PATH=/usr/local/n/versions/node/14.15.0/bin:$PATH sudo npm init -f sudo npm install -g autorest --save - sudo npm install autorest.python@5.12.0 --save - sudo npm install autorest.modelerfour@4.19.3 --save + sudo npm install @autorest/python@5.12.0 --save + sudo npm install @autorest/modelerfour@4.19.3 --save mkdir temp_folder From 24080e75903eb9a872f3ff775f4f23af6c326322 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 16:48:25 +0800 Subject: [PATCH 74/78] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index fc0af978d511..1dd47e609416 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -128,7 +128,7 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): try: # c = ' '.join(cmd_line) # print(f'c={c}') - c = 'autorest --version' + c = 'npm install @autorest/modelerfour@4.19.3 --save' data = subprocess.getoutput(c) except subprocess.CalledProcessError as ex: data = ex.output From 8628f1947646c6d664cd5f015a4dce874f5c51ba Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 17:09:55 +0800 Subject: [PATCH 75/78] test --- scripts/auto_release/main.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 1dd47e609416..7cefb016f496 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -125,19 +125,19 @@ def start_test_proxy(): print_check('pwsh eng/common/testproxy/docker-start-proxy.ps1 \"start\"') def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): - try: - # c = ' '.join(cmd_line) - # print(f'c={c}') - c = 'npm install @autorest/modelerfour@4.19.3 --save' - data = subprocess.getoutput(c) - except subprocess.CalledProcessError as ex: - data = ex.output - print('-------------------------') - for line in data.split('\n'): - print(line) - print('-------------------------') - - try: + # try: + # # c = ' '.join(cmd_line) + # # print(f'c={c}') + # c = 'npm install @autorest/modelerfour@4.19.3 --save' + # data = subprocess.getoutput(c) + # except subprocess.CalledProcessError as ex: + # data = ex.output + # print('-------------------------') + # for line in data.split('\n'): + # print(line) + # print('-------------------------') + + def sp_run(): process = subprocess.Popen( cmd_line, stderr=subprocess.STDOUT, @@ -161,11 +161,16 @@ def execute_simple_command(cmd_line, cwd=None, shell=False, env=None): # print(f"[Autorest22] {output_buffer[i]}") raise subprocess.CalledProcessError(process.returncode, cmd_line, output) return output + + try: + sp_run() + except subprocess.CalledProcessError as ex: + sp_run() except Exception as err: _LOG.error(err) raise else: - _LOGGER.info("Return code: %s", process.returncode) + _LOG.info("Return code: %s", 1) class CodegenTestPR: From 5e72f8b3e46c2357c1be25082458f94e18c78cc1 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 17:22:13 +0800 Subject: [PATCH 76/78] test --- scripts/auto_release/main.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 7cefb016f496..a04607970b30 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -165,7 +165,13 @@ def sp_run(): try: sp_run() except subprocess.CalledProcessError as ex: - sp_run() + try: + sp_run() + except subprocess.CalledProcessError as ex: + try: + sp_run() + except subprocess.CalledProcessError as ex: + sp_run() except Exception as err: _LOG.error(err) raise From debf2fa0deff9365a61a4e6cfe2fa83b19241a11 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 17:22:35 +0800 Subject: [PATCH 77/78] test --- scripts/auto_release/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index a04607970b30..5e10f4653854 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -171,7 +171,7 @@ def sp_run(): try: sp_run() except subprocess.CalledProcessError as ex: - sp_run() + sp_run() except Exception as err: _LOG.error(err) raise From c9c9790c1be9bc62b5f2848b01e3fd67e0f9ec69 Mon Sep 17 00:00:00 2001 From: "Jiefeng Chen (WICRESOFT NORTH AMERICA LTD)" Date: Fri, 25 Mar 2022 17:47:54 +0800 Subject: [PATCH 78/78] test --- scripts/auto_release/main.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/scripts/auto_release/main.py b/scripts/auto_release/main.py index 5e10f4653854..34282c4b8e76 100644 --- a/scripts/auto_release/main.py +++ b/scripts/auto_release/main.py @@ -156,22 +156,15 @@ def sp_run(): output = "\n".join(output_buffer) if process.returncode: # print necessary error info - for i in range(len(output_buffer)): - _LOG.error(f"[Autorest66] {output_buffer[i]}") - # print(f"[Autorest22] {output_buffer[i]}") + for i in range(-min(len(output_buffer), 5), 0): + print(f"[Autorest22] {output_buffer[i]}") raise subprocess.CalledProcessError(process.returncode, cmd_line, output) return output try: sp_run() except subprocess.CalledProcessError as ex: - try: - sp_run() - except subprocess.CalledProcessError as ex: - try: - sp_run() - except subprocess.CalledProcessError as ex: - sp_run() + sp_run() except Exception as err: _LOG.error(err) raise