From 397d069c3f05154549aef86bf33c3a03360bee92 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Thu, 28 Jul 2022 17:19:15 +0800 Subject: [PATCH] [DPG] integration into swagger pipeline (#25445) * add PyYAML * code * README.md format * multi client * code * gen multi client * single client gen * compatible with meta storage code * add log to lookup readme * dpg package does not need _meta.json in MAINFEST.in * add language, apiviewartifacts * optimize * Update setup.py * sdk generation pipeline support dpg Signed-off-by: Zhou Zheng * auto codegen don't use specFolder to define swagger repo's location Signed-off-by: Zhou Zheng * fetch automation_generate.sh recent change to sdk_generate.sh Signed-off-by: Zhou Zheng * set default spec flord as global constant Signed-off-by: Zhou Zheng * set DEFAULT_SPEC_FOLDER in generate_utils Signed-off-by: Zhou Zheng * code * Update sdk_generate.sh * Update sdk_init.sh * Update generate_utils.py Co-authored-by: Zhou Zheng --- .../packaging_tools/auto_codegen.py | 5 ++-- .../packaging_tools/generate_utils.py | 25 +++++++++++-------- .../packaging_tools/sdk_generator.py | 19 ++++++++++---- .../packaging_tools/sdk_package.py | 11 ++++++++ 4 files changed, 41 insertions(+), 19 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py index 14f87f40a813a..de2d39ace0857 100644 --- a/tools/azure-sdk-tools/packaging_tools/auto_codegen.py +++ b/tools/azure-sdk-tools/packaging_tools/auto_codegen.py @@ -7,11 +7,10 @@ from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, - format_samples, gen_dpg) + format_samples, gen_dpg, dpg_relative_folder) _LOGGER = logging.getLogger(__name__) - def main(generate_input, generate_output): with open(generate_input, "r") as reader: data = json.load(reader) @@ -28,7 +27,7 @@ def main(generate_input, generate_output): config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, python_tag=python_tag) else: - config = gen_dpg(input_readme, data.get('autorestConfig', '')) + config = gen_dpg(input_readme, data.get('autorestConfig', ''), dpg_relative_folder(spec_folder)) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index c565d7f0520ef..6112d827ecf6f 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -21,6 +21,9 @@ DEFAULT_DEST_FOLDER = "./dist" _DPG_README = "README.md" +def dpg_relative_folder(spec_folder: str) -> str: + return ("../" * 4) + spec_folder + "/" + def get_package_names(sdk_folder): files = get_add_diff_file_list(sdk_folder) matches = {_SDK_FOLDER_RE.search(f) for f in files} @@ -162,12 +165,12 @@ def gen_package_name(origin_config: Dict[str, Any]) -> str: return Path(origin_config["output-folder"]).parts[-1] -def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: +def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str) -> Dict[str, Any]: return { "package-name": gen_package_name(origin_config), "license-header": "MICROSOFT_MIT_NO_VERSION", "package-version": origin_config.get("package-version", "1.0.0b1"), - "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]], + "require": [spec_root + line for line in origin_config["require"]], "package-mode": "dataplane", "output-folder": "../", } @@ -177,9 +180,9 @@ def gen_general_namespace(package_name: str) -> str: return package_name.replace('-', '.') -def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str: +def gen_dpg_config_single_client(origin_config: Dict[str, Any], spec_folder: str) -> str: package_name = Path(origin_config["output-folder"]).parts[-1] - readme_config = gen_basic_config(origin_config) + readme_config = gen_basic_config(origin_config, spec_folder) readme_config.update({ "namespace": gen_general_namespace(package_name), }) @@ -208,9 +211,9 @@ def gen_batch_config(origin_config: Dict[str, Any]) -> Dict[str, Any]: return {"batch": batch_config} -def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: +def gen_dpg_config_multi_client(origin_config: Dict[str, Any], spec_folder: str) -> str: # generate config - basic_config = gen_basic_config(origin_config) + basic_config = gen_basic_config(origin_config, spec_folder) batch_config = gen_batch_config(origin_config) tag_config = gen_tag_config(origin_config) @@ -228,7 +231,7 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str: # generate swagger/README.md and return relative path based on SDK repo root path -def gen_dpg_config(autorest_config: str) -> str: +def gen_dpg_config(autorest_config: str, spec_folder: str) -> str: # remove useless lines autorest_config = extract_yaml_content(autorest_config) _LOGGER.info(f"autorestConfig after remove useless lines:\n{autorest_config}") @@ -242,9 +245,9 @@ def gen_dpg_config(autorest_config: str) -> str: # generate autorest configuration if "batch:" in autorest_config: - readme_content = gen_dpg_config_multi_client(origin_config) + readme_content = gen_dpg_config_multi_client(origin_config, spec_folder) else: - readme_content = gen_dpg_config_single_client(origin_config) + readme_content = gen_dpg_config_single_client(origin_config, spec_folder) # output autorest configuration swagger_readme = str(Path(swagger_folder, _DPG_README)) @@ -265,10 +268,10 @@ def lookup_swagger_readme(rest_readme_path: str) -> str: return "" -def gen_dpg(rest_readme_path: str, autorest_config: str) -> Dict[str, Any]: +def gen_dpg(rest_readme_path: str, autorest_config: str, spec_folder: str) -> Dict[str, Any]: # generate or find swagger/README.md if autorest_config: - swagger_readme = gen_dpg_config(autorest_config) + swagger_readme = gen_dpg_config(autorest_config, spec_folder) else: swagger_readme = lookup_swagger_readme(rest_readme_path) if not swagger_readme: diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py index c2b06e51322e5..0ef787271d340 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py @@ -4,9 +4,10 @@ from pathlib import Path from subprocess import check_call -from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG +from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE from .generate_sdk import generate -from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview +from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview, + format_samples, gen_dpg) _LOGGER = logging.getLogger(__name__) @@ -19,13 +20,17 @@ def main(generate_input, generate_output): spec_folder = data["specFolder"] sdk_folder = "." result = {} + python_tag = data.get('python_tag') package_total = set() input_readme = data["relatedReadmeMdFile"] relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") - config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG - config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True) + if 'resource-manager' in input_readme: + config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True, + python_tag=python_tag) + else: + config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") @@ -39,19 +44,22 @@ def main(generate_input, generate_output): package_entry = {} package_entry["packageName"] = package_name package_entry["path"] = [folder_name] + package_entry["readmeMd"] = [input_readme] package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path) result[package_name] = package_entry else: result[package_name]["path"].append(folder_name) + result[package_name]["readmeMd"].append(input_readme) # Generate some necessary file for new service init_new_service(package_name, folder_name) + format_samples(sdk_code_path) # Update metadata try: update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme) except Exception as e: - _LOGGER.info(str(e)) + _LOGGER.info(f"fail to update meta: {str(e)}") # Setup package locally check_call( @@ -62,6 +70,7 @@ def main(generate_input, generate_output): # remove duplicates for value in result.values(): value["path"] = list(set(value["path"])) + value["readmeMd"] = list(set(value["readmeMd"])) with open(generate_output, "w") as writer: json.dump(result, writer) diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_package.py b/tools/azure-sdk-tools/packaging_tools/sdk_package.py index cbf5c5d799bf0..fdb0d2e9a70da 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_package.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_package.py @@ -13,6 +13,7 @@ def main(generate_input, generate_output): with open(generate_input, "r") as reader: data = json.load(reader) + _LOGGER.info(f"auto_package input: {data}") sdk_folder = "." result = {"packages": []} @@ -38,9 +39,19 @@ def main(generate_input, generate_output): dist_path = Path(sdk_folder, folder_name, package_name, "dist") package["artifacts"] = [str(dist_path / package_file) for package_file in os.listdir(dist_path)] package["result"] = "succeeded" + # Installation package + package["installInstructions"] = { + "full": "You can install the use using pip install of the artificats.", + "lite": f"pip install {package_name}", + } # to distinguish with track1 if 'azure-mgmt-' in package_name: package["packageName"] = "track2_" + package["packageName"] + for artifact in package["artifacts"]: + if ".whl" in artifact: + package["apiViewArtifact"] = artifact + package["language"] = "Python" + break package["packageFolder"] = package["path"][0] result["packages"].append(package)