[DPG] integration into swagger pipeline (Azure#25445)

* add PyYAML

* code

* README.md format

* multi client

* code

* gen multi client

* single client gen

* compatible with meta storage code

* add log to lookup readme

* dpg package does not need _meta.json in MAINFEST.in

* add language, apiviewartifacts

* optimize

* Update setup.py

* sdk generation pipeline support dpg

Signed-off-by: Zhou Zheng <[email protected]>

* auto codegen don't use specFolder to define swagger repo's location

Signed-off-by: Zhou Zheng <[email protected]>

* fetch automation_generate.sh recent change to sdk_generate.sh

Signed-off-by: Zhou Zheng <[email protected]>

* set default spec flord as global constant

Signed-off-by: Zhou Zheng <[email protected]>

* set DEFAULT_SPEC_FOLDER in generate_utils

Signed-off-by: Zhou Zheng <[email protected]>

* code

* Update sdk_generate.sh

* Update sdk_init.sh

* Update generate_utils.py

Co-authored-by: Zhou Zheng <[email protected]>

tools/azure-sdk-tools/packaging_tools/auto_codegen.py

@@ -7,11 +7,10 @@
 from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE
 from .generate_sdk import generate
 from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview,
-                             format_samples, gen_dpg)
+                             format_samples, gen_dpg, dpg_relative_folder)
 
 _LOGGER = logging.getLogger(__name__)
 
-
 def main(generate_input, generate_output):
     with open(generate_input, "r") as reader:
         data = json.load(reader)
@@ -28,7 +27,7 @@ def main(generate_input, generate_output):
             config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True,
                               python_tag=python_tag)
         else:
-            config = gen_dpg(input_readme, data.get('autorestConfig', ''))
+            config = gen_dpg(input_readme, data.get('autorestConfig', ''), dpg_relative_folder(spec_folder))
         package_names = get_package_names(sdk_folder)
         _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]")
 

tools/azure-sdk-tools/packaging_tools/generate_utils.py

@@ -21,6 +21,9 @@
 DEFAULT_DEST_FOLDER = "./dist"
 _DPG_README = "README.md"
 
+def dpg_relative_folder(spec_folder: str) -> str:
+    return ("../" * 4) + spec_folder + "/"
+
 def get_package_names(sdk_folder):
     files = get_add_diff_file_list(sdk_folder)
     matches = {_SDK_FOLDER_RE.search(f) for f in files}
@@ -162,12 +165,12 @@ def gen_package_name(origin_config: Dict[str, Any]) -> str:
     return Path(origin_config["output-folder"]).parts[-1]
 
 
-def gen_basic_config(origin_config: Dict[str, Any]) -> Dict[str, Any]:
+def gen_basic_config(origin_config: Dict[str, Any], spec_folder: str) -> Dict[str, Any]:
     return {
         "package-name": gen_package_name(origin_config),
         "license-header": "MICROSOFT_MIT_NO_VERSION",
         "package-version": origin_config.get("package-version", "1.0.0b1"),
-        "require": ["../../../../../azure-rest-api-specs/" + line for line in origin_config["require"]],
+        "require": [spec_root + line for line in origin_config["require"]],
         "package-mode": "dataplane",
         "output-folder": "../",
     }
@@ -177,9 +180,9 @@ def gen_general_namespace(package_name: str) -> str:
     return package_name.replace('-', '.')
 
 
-def gen_dpg_config_single_client(origin_config: Dict[str, Any]) -> str:
+def gen_dpg_config_single_client(origin_config: Dict[str, Any], spec_folder: str) -> str:
     package_name = Path(origin_config["output-folder"]).parts[-1]
-    readme_config = gen_basic_config(origin_config)
+    readme_config = gen_basic_config(origin_config, spec_folder)
     readme_config.update({
         "namespace": gen_general_namespace(package_name),
     })
@@ -208,9 +211,9 @@ def gen_batch_config(origin_config: Dict[str, Any]) -> Dict[str, Any]:
     return {"batch": batch_config}
 
 
-def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str:
+def gen_dpg_config_multi_client(origin_config: Dict[str, Any], spec_folder: str) -> str:
     # generate config
-    basic_config = gen_basic_config(origin_config)
+    basic_config = gen_basic_config(origin_config, spec_folder)
     batch_config = gen_batch_config(origin_config)
     tag_config = gen_tag_config(origin_config)
 
@@ -228,7 +231,7 @@ def gen_dpg_config_multi_client(origin_config: Dict[str, Any]) -> str:
 
 
 # generate swagger/README.md and return relative path based on SDK repo root path
-def gen_dpg_config(autorest_config: str) -> str:
+def gen_dpg_config(autorest_config: str, spec_folder: str) -> str:
     # remove useless lines
     autorest_config = extract_yaml_content(autorest_config)
     _LOGGER.info(f"autorestConfig after remove useless lines:\n{autorest_config}")
@@ -242,9 +245,9 @@ def gen_dpg_config(autorest_config: str) -> str:
 
     # generate autorest configuration
     if "batch:" in autorest_config:
-        readme_content = gen_dpg_config_multi_client(origin_config)
+        readme_content = gen_dpg_config_multi_client(origin_config, spec_folder)
     else:
-        readme_content = gen_dpg_config_single_client(origin_config)
+        readme_content = gen_dpg_config_single_client(origin_config, spec_folder)
 
     # output autorest configuration
     swagger_readme = str(Path(swagger_folder, _DPG_README))
@@ -265,10 +268,10 @@ def lookup_swagger_readme(rest_readme_path: str) -> str:
     return ""
 
 
-def gen_dpg(rest_readme_path: str, autorest_config: str) -> Dict[str, Any]:
+def gen_dpg(rest_readme_path: str, autorest_config: str, spec_folder: str) -> Dict[str, Any]:
     # generate or find swagger/README.md
     if autorest_config:
-        swagger_readme = gen_dpg_config(autorest_config)
+        swagger_readme = gen_dpg_config(autorest_config, spec_folder)
     else:
         swagger_readme = lookup_swagger_readme(rest_readme_path)
     if not swagger_readme:

tools/azure-sdk-tools/packaging_tools/sdk_generator.py

@@ -4,9 +4,10 @@
 from pathlib import Path
 from subprocess import check_call
 
-from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE, CONFIG_FILE_DPG
+from .swaggertosdk.SwaggerToSdkCore import CONFIG_FILE
 from .generate_sdk import generate
-from .generate_utils import get_package_names, init_new_service, update_servicemetadata, judge_tag_preview
+from .generate_utils import (get_package_names, init_new_service, update_servicemetadata, judge_tag_preview,
+                             format_samples, gen_dpg)
 
 _LOGGER = logging.getLogger(__name__)
 
@@ -19,13 +20,17 @@ def main(generate_input, generate_output):
     spec_folder = data["specFolder"]
     sdk_folder = "."
     result = {}
+    python_tag = data.get('python_tag')
     package_total = set()
 
     input_readme = data["relatedReadmeMdFile"]
     relative_path_readme = str(Path(spec_folder, input_readme))
     _LOGGER.info(f"[CODEGEN]({input_readme})codegen begin")
-    config_file = CONFIG_FILE if 'resource-manager' in input_readme else CONFIG_FILE_DPG
-    config = generate(config_file, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True)
+    if 'resource-manager' in input_readme:
+        config = generate(CONFIG_FILE, sdk_folder, [], relative_path_readme, spec_folder, force_generation=True,
+                          python_tag=python_tag)
+    else:
+        config = gen_dpg(input_readme, data.get('autorestConfig', ''), spec_folder)
     package_names = get_package_names(sdk_folder)
     _LOGGER.info(f"[CODEGEN]({input_readme})codegen end. [(packages:{str(package_names)})]")
 
@@ -39,19 +44,22 @@ def main(generate_input, generate_output):
             package_entry = {}
             package_entry["packageName"] = package_name
             package_entry["path"] = [folder_name]
+            package_entry["readmeMd"] = [input_readme]
             package_entry["tagIsStable"] = not judge_tag_preview(sdk_code_path)
             result[package_name] = package_entry
         else:
             result[package_name]["path"].append(folder_name)
+            result[package_name]["readmeMd"].append(input_readme)
 
         # Generate some necessary file for new service
         init_new_service(package_name, folder_name)
+        format_samples(sdk_code_path)
 
         # Update metadata
         try:
             update_servicemetadata(sdk_folder, data, config, folder_name, package_name, spec_folder, input_readme)
         except Exception as e:
-            _LOGGER.info(str(e))
+            _LOGGER.info(f"fail to update meta: {str(e)}")
 
         # Setup package locally
         check_call(
@@ -62,6 +70,7 @@ def main(generate_input, generate_output):
     # remove duplicates
     for value in result.values():
         value["path"] = list(set(value["path"]))
+        value["readmeMd"] = list(set(value["readmeMd"]))
 
     with open(generate_output, "w") as writer:
         json.dump(result, writer)

tools/azure-sdk-tools/packaging_tools/sdk_package.py

@@ -13,6 +13,7 @@
 def main(generate_input, generate_output):
     with open(generate_input, "r") as reader:
         data = json.load(reader)
+        _LOGGER.info(f"auto_package input: {data}")
 
     sdk_folder = "."
     result = {"packages": []}
@@ -38,9 +39,19 @@ def main(generate_input, generate_output):
         dist_path = Path(sdk_folder, folder_name, package_name, "dist")
         package["artifacts"] = [str(dist_path / package_file) for package_file in os.listdir(dist_path)]
         package["result"] = "succeeded"
+        # Installation package
+        package["installInstructions"] = {
+            "full": "You can install the use using pip install of the artificats.",
+            "lite": f"pip install {package_name}",
+        }
         # to distinguish with track1
         if 'azure-mgmt-' in package_name:
             package["packageName"] = "track2_" + package["packageName"]
+        for artifact in package["artifacts"]:
+            if ".whl" in artifact:
+                package["apiViewArtifact"] = artifact
+                package["language"] = "Python"
+                break
         package["packageFolder"] = package["path"][0]
         result["packages"].append(package)