[pipeline][cadl] add global config (Azure#27847)

* add global config

* fix path

* Update generate_utils.py

* update

* update output-folder

* update init new service

* review

* review

* use latest version

* pin version

cadl_to_sdk_config.json

@@ -0,0 +1,13 @@
+{
+  "dependencies": {
+    "@azure-tools/cadl-python": "0.4.16",
+    "@autorest/python": "6.2.15",
+    "@azure-tools/cadl-azure-core": "~0.24.0",
+    "@azure-tools/cadl-dpg": "~0.25.0-dev.8",
+    "@cadl-lang/compiler": "~0.38.5",
+    "@cadl-lang/eslint-config-cadl": "~0.5.0",
+    "@cadl-lang/openapi": "~0.38.0",
+    "@cadl-lang/rest": "~0.38.1",
+    "@cadl-lang/versioning": "~0.38.0"
+  }
+}

tools/azure-sdk-tools/packaging_tools/generate_utils.py

@@ -34,21 +34,34 @@ def get_package_names(sdk_folder):
     return package_names
 
 
-def init_new_service(package_name, folder_name):
-    setup = Path(folder_name, package_name, "setup.py")
-    if not setup.exists():
-        check_call(
-            f"python -m packaging_tools --build-conf {package_name} -o {folder_name}",
-            shell=True,
+def init_new_service(package_name, folder_name, is_cadl = False):
+    if not is_cadl:
+        setup = Path(folder_name, package_name, "setup.py")
+        if not setup.exists():
+            check_call(
+                f"python -m packaging_tools --build-conf {package_name} -o {folder_name}",
+                shell=True,
+            )
+            ci = Path(folder_name, "ci.yml")
+            if not ci.exists():
+                with open("ci_template.yml", "r") as file_in:
+                    content = file_in.readlines()
+                name = package_name.replace("azure-", "").replace("mgmt-", "")
+                content = [line.replace("MyService", name) for line in content]
+                with open(str(ci), "w") as file_out:
+                    file_out.writelines(content)
+    else:
+        output_path = Path(folder_name) / package_name
+        if not (output_path / "sdk_packaging.toml").exists():
+            with open(output_path / "sdk_packaging.toml", "w") as file_out:
+                file_out.write("[packaging]\nauto_update = false")
+
+        # add ci.yaml
+        generate_ci(
+            template_path=Path("scripts/quickstart_tooling_dpg/template_ci"),
+            folder_path=Path(folder_name),
+            package_name=package_name
         )
-        ci = Path(folder_name, "ci.yml")
-        if not ci.exists():
-            with open("ci_template.yml", "r") as file_in:
-                content = file_in.readlines()
-            name = package_name.replace("azure-", "").replace("mgmt-", "")
-            content = [line.replace("MyService", name) for line in content]
-            with open(str(ci), "w") as file_out:
-                file_out.writelines(content)
 
 
 def update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme):
@@ -347,48 +360,32 @@ def generate_ci(template_path: Path, folder_path: Path, package_name: str) -> No
         file_out.writelines(content)
 
 def gen_cadl(cadl_relative_path: str, spec_folder: str) -> Dict[str, Any]:
-    # update config file
     cadl_python = "@azure-tools/cadl-python"
     autorest_python = "@autorest/python"
-    project_yaml_path = Path(spec_folder) / cadl_relative_path / "cadl-project.yaml"
-    with open(project_yaml_path, "r") as file_in:
-        project_yaml = yaml.safe_load(file_in)
-    if not project_yaml.get("emitters", {}).get(cadl_python):
-        return
-    if not project_yaml["emitters"][cadl_python].get("sdk-folder"):
-        raise Exception("no sdk-folder is defined")
-    output_path = Path(os.getcwd()) / project_yaml["emitters"][cadl_python]["sdk-folder"]
-    if not output_path.exists():
-        os.makedirs(output_path)
-
-    project_yaml["emitters"][cadl_python].pop("sdk-folder")
-    with open(project_yaml_path, "w") as file_out:
-        yaml.safe_dump(project_yaml, file_out)
 
     # npm install tool
     origin_path = os.getcwd()
+    with open("cadl_to_sdk_config.json", "r") as file_in:
+        cadl_python_dep = json.load(file_in)
     os.chdir(Path(spec_folder) / cadl_relative_path)
+    if Path("package.json").exists():
+        with open("package.json", "r") as file_in:
+            cadl_tools = json.load(file_in)
+    else:
+        cadl_tools = {"dependencies:{}"}
+    cadl_tools["dependencies"].update(cadl_python_dep["dependencies"])
+    with open("package.json", "w") as file_out:
+        json.dump(cadl_tools, file_out)
     check_call("npm install", shell=True)
 
     # generate code
-    check_call(f"npx cadl compile . --emit {cadl_python} --output-path={str(output_path)}", shell=True)
-    if (output_path / "output.yaml").exists():
-        os.remove(output_path / "output.yaml")
-    if not (output_path / "sdk_packaging.toml").exists():
-        with open(output_path / "sdk_packaging.toml", "w") as file_out:
-            file_out.write("[packaging]\nauto_update = false")
+    cadl_file = "client.cadl" if Path("client.cadl").exists() else "."
+    check_call(f"npx cadl compile {cadl_file} --emit {cadl_python} --arg \"python-sdk-folder={origin_path}\" ", shell=True)
 
     # get version of codegen used in generation
     npm_package_verstion = get_npm_package_version(autorest_python)
 
     # return to original folder
     os.chdir(origin_path)
 
-    # add ci.yaml
-    generate_ci(
-        template_path=Path("scripts/quickstart_tooling_dpg/template_ci"),
-        folder_path=output_path.parent,
-        package_name=project_yaml["emitters"][cadl_python]["package-name"]
-    )
-
-    return npm_package_verstion
+    return npm_package_verstion

tools/azure-sdk-tools/packaging_tools/sdk_generator.py

@@ -76,6 +76,7 @@ def main(generate_input, generate_output):
 
     for input_readme in readme_files:
         _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin")
+        is_cadl = False
         if "resource-manager" in input_readme:
             relative_path_readme = str(Path(spec_folder, input_readme))
             del_outdated_samples(relative_path_readme)
@@ -92,6 +93,7 @@ def main(generate_input, generate_output):
             config = gen_dpg(input_readme, data.get("autorestConfig", ""), dpg_relative_folder(spec_folder))
         else:
             config = gen_cadl(input_readme, spec_folder)
+            is_cadl = True
         package_names = get_package_names(sdk_folder)
         _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]")
 
@@ -113,7 +115,7 @@ def main(generate_input, generate_output):
                 result[package_name][spec_word].append(input_readme)
 
             # Generate some necessary file for new service
-            init_new_service(package_name, folder_name)
+            init_new_service(package_name, folder_name, is_cadl)
             format_samples(sdk_code_path)
 
             # Update metadata