From d0f5c86fff2c3dd79242b4c455c0766a65145fb9 Mon Sep 17 00:00:00 2001 From: Yuchao Yan Date: Wed, 19 Oct 2022 13:27:09 +0800 Subject: [PATCH] Cadl automation pipeline (#26912) * cdoe * fix * code * fix * code * meta.json * meta.json * compatible --- .../packaging_tools/generate_utils.py | 77 +++++++++++++++---- .../packaging_tools/sdk_generator.py | 23 ++++-- 2 files changed, 77 insertions(+), 23 deletions(-) diff --git a/tools/azure-sdk-tools/packaging_tools/generate_utils.py b/tools/azure-sdk-tools/packaging_tools/generate_utils.py index 77483471890f0..049d45906c8a6 100644 --- a/tools/azure-sdk-tools/packaging_tools/generate_utils.py +++ b/tools/azure-sdk-tools/packaging_tools/generate_utils.py @@ -6,7 +6,7 @@ from azure_devtools.ci_tools.git_tools import get_add_diff_file_list from pathlib import Path -from subprocess import check_call +from subprocess import check_call, getoutput from typing import List, Dict, Any from glob import glob import yaml @@ -52,26 +52,32 @@ def init_new_service(package_name, folder_name): def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme): - readme_file = str(Path(spec_folder, input_readme)) - global_conf = config["meta"] - local_conf = config.get("projects", {}).get(readme_file, {}) - - if "resource-manager" in input_readme: - cmd = ["autorest", input_readme] - else: - # autorest for DPG will be executed in package folder like: sdk/deviceupdate/azure-iot-deviceupdate/swagger - cmd = ["autorest", _DPG_README] - cmd += build_autorest_options(global_conf, local_conf) - - # metadata metadata = { - "autorest": global_conf["autorest_options"]["version"], - "use": global_conf["autorest_options"]["use"], "commit": data["headSha"], "repository_url": data["repoHttpsUrl"], - "autorest_command": " ".join(cmd), - "readme": input_readme, } + if "meta" in config: + readme_file = str(Path(spec_folder, input_readme)) + global_conf = config["meta"] + local_conf = config.get("projects", {}).get(readme_file, {}) + + if "resource-manager" in input_readme: + cmd = ["autorest", input_readme] + else: + # autorest for DPG will be executed in package folder like: sdk/deviceupdate/azure-iot-deviceupdate/swagger + cmd = ["autorest", _DPG_README] + cmd += build_autorest_options(global_conf, local_conf) + + # metadata + metadata.update({ + "autorest": global_conf["autorest_options"]["version"], + "use": global_conf["autorest_options"]["use"], + "autorest_command": " ".join(cmd), + "readme": input_readme, + }) + else: + metadata["cadl_src"] = input_readme + metadata.update(config) _LOGGER.info("Metadata json:\n {}".format(json.dumps(metadata, indent=2))) @@ -322,3 +328,40 @@ def format_samples(sdk_code_path) -> None: fw.write(file_content) _LOGGER.info(f"format generated_samples successfully") + +def gen_cadl(cadl_relative_path: str, spec_folder: str) -> Dict[str, Any]: + # update config file + cadl_python = "@azure-tools/cadl-python" + project_yaml_path = Path(spec_folder) / cadl_relative_path / "cadl-project.yaml" + with open(project_yaml_path, "r") as file_in: + project_yaml = yaml.safe_load(file_in) + if not project_yaml.get("emitters", {}).get(cadl_python): + return + if not project_yaml["emitters"][cadl_python].get("sdk-folder"): + raise Exception("no sdk-folder is defined") + output_path = Path(os.getcwd()) / project_yaml["emitters"][cadl_python]["sdk-folder"] + if not output_path.exists(): + os.makedirs(output_path) + + project_yaml["emitters"][cadl_python].pop("sdk-folder") + project_yaml["emitters"][cadl_python]["output-path"] = str(output_path) + with open(project_yaml_path, "w") as file_out: + yaml.safe_dump(project_yaml, file_out) + + # npm install tool + origin_path = os.getcwd() + os.chdir(Path(spec_folder) / cadl_relative_path) + check_call("npm install", shell=True) + + # generate code + check_call(f"npx cadl compile . --emit {cadl_python}", shell=True) + if Path(output_path / "output.yaml").exists(): + os.remove(Path(output_path / "output.yaml")) + + # get version of @azure-tools/cadl-python used in generation + cadl_python_version = getoutput(f"npm view {cadl_python} version").split('\n')[-1] + + # return to original folder + os.chdir(origin_path) + + return {cadl_python: cadl_python_version} \ No newline at end of file diff --git a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py index 34bdf40321cdb..1fdd1c75cffcf 100644 --- a/tools/azure-sdk-tools/packaging_tools/sdk_generator.py +++ b/tools/azure-sdk-tools/packaging_tools/sdk_generator.py @@ -16,6 +16,7 @@ format_samples, gen_dpg, dpg_relative_folder, + gen_cadl ) _LOGGER = logging.getLogger(__name__) @@ -30,20 +31,28 @@ def main(generate_input, generate_output): result = {} python_tag = data.get("python_tag") package_total = set() + spec_word = "readmeMd" if "relatedReadmeMdFiles" in data: readme_files = data["relatedReadmeMdFiles"] - else: + elif "relatedReadmeMdFile" in data: input_readme = data["relatedReadmeMdFile"] if "specification" in spec_folder: spec_folder = str(Path(spec_folder.split("specification")[0])) if "specification" not in input_readme: input_readme = str(Path("specification") / input_readme) readme_files = [input_readme] + else: + # ["specification/confidentialledger/ConfientialLedger"] + if isinstance(data["relatedCadlProjectFolder"], str): + readme_files = [data["relatedCadlProjectFolder"]] + else: + readme_files = data["relatedCadlProjectFolder"] + spec_word = "cadlProject" for input_readme in readme_files: - relative_path_readme = str(Path(spec_folder, input_readme)) _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin") if "resource-manager" in input_readme: + relative_path_readme = str(Path(spec_folder, input_readme)) config = generate( CONFIG_FILE, sdk_folder, @@ -53,8 +62,10 @@ def main(generate_input, generate_output): force_generation=True, python_tag=python_tag, ) - else: + elif "data-plane" in input_readme: config = gen_dpg(input_readme, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder)) + else: + config = gen_cadl(input_readme, spec_folder) package_names = get_package_names(sdk_folder) _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]") @@ -68,12 +79,12 @@ def main(generate_input, generate_output): package_entry = {} package_entry["packageName"] = package_name package_entry["path"] = [folder_name] - package_entry["readmeMd"] = [input_readme] + package_entry[spec_word] = [input_readme] package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path) result[package_name] = package_entry else: result[package_name]["path"].append(folder_name) - result[package_name]["readmeMd"].append(input_readme) + result[package_name][spec_word].append(input_readme) # Generate some necessary file for new service init_new_service(package_name, folder_name) @@ -102,7 +113,7 @@ def main(generate_input, generate_output): # remove duplicates for value in result.values(): value["path"] = list(set(value["path"])) - value["readmeMd"] = list(set(value["readmeMd"])) + value[spec_word] = list(set(value[spec_word])) with open(generate_output, "w") as writer: json.dump(result, writer)