[Separation] Separation of devkit and azure

.cspell.json

@@ -25,18 +25,21 @@
     "**/*.txt",
     ".gitignore",
     "scripts/docs/_build/**",
-    "src/promptflow/promptflow/azure/_restclient/flow/**",
-    "src/promptflow/promptflow/azure/_restclient/swagger.json",
-    "src/promptflow/promptflow/azure/_models/**",
+    "src/promptflow-azure/promptflow/azure/_restclient/flow/**",
+    "src/promptflow-azure/promptflow/azure/_restclient/swagger.json",
+    "src/promptflow-azure/promptflow/azure/_models/**",
+    "src/promptflow-azure/tests/**",
     "src/promptflow-core/promptflow/core/_connection_provider/_models/**",
     "src/promptflow/tests/**",
     "src/promptflow-recording/**",
     "src/promptflow-tools/tests/**",
     "**/flow.dag.yaml",
+    "**/pyproject.toml",
     "**/setup.py",
     "scripts/installer/curl_install_pypi/**",
     "scripts/installer/windows/**",
     "src/promptflow/promptflow/_sdk/_service/pfsvc.py",
+    "src/promptflow-devkit/promptflow/_sdk/_service/pfsvc.py",
     ".github/workflows/**",
     ".github/actions/**",
     ".github/pipelines/**"

.github/workflows/promptflow-executor-e2e-test.yml

@@ -106,8 +106,9 @@ jobs:
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
         pip install ${{ github.workspace }}/src/promptflow-tracing
-        pip install ${{ github.workspace }}/src/promptflow-core
-        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}}
+        pip install ${{ github.workspace }}/src/promptflow-core[executor-service]
+        pip install ${{ github.workspace }}/src/promptflow-devkit
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze
     - name: install recording

.github/workflows/promptflow-executor-unit-test.yml

@@ -110,8 +110,9 @@ jobs:
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
         pip install ${{ github.workspace }}/src/promptflow-tracing
-        pip install ${{ github.workspace }}/src/promptflow-core
-        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}}
+        pip install ${{ github.workspace }}/src/promptflow-core[executor-service]
+        pip install ${{ github.workspace }}/src/promptflow-devkit
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze
     - name: install recording

.github/workflows/promptflow-global-config-test.yml

@@ -69,6 +69,8 @@ jobs:
         echo "Installed promptflow-tracing"
         pip uninstall -y promptflow-core
         pip install ${{ github.workspace }}/src/promptflow-core
+        pip uninstall -y promptflow-devkit
+        pip install ${{ github.workspace }}/src/promptflow-devkit
         pip freeze
     - name: Azure Login
       uses: azure/login@v1

.github/workflows/promptflow-sdk-cli-test.yml

@@ -5,6 +5,7 @@ on:
   pull_request:
     paths:
       - src/promptflow-core/**
+      - src/promptflow-devkit/**
       - src/promptflow/**
       - src/promptflow-tracing/**
       - scripts/building/**
@@ -81,6 +82,7 @@ jobs:
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
         pip install ${{ github.workspace }}/src/promptflow-tracing
         pip install ${{ github.workspace }}/src/promptflow-core
+        pip install ${{ github.workspace }}/src/promptflow-devkit
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze

.github/workflows/sdk-cli-azure-test-production.yml

@@ -89,6 +89,7 @@ jobs:
           pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
           pip install ${{ github.workspace }}/src/promptflow-tracing
           pip install ${{ github.workspace }}/src/promptflow-core
+          pip install ${{ github.workspace }}/src/promptflow-devkit
           gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
           gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
           pip freeze

.github/workflows/sdk-cli-azure-test-pull-request.yml

@@ -7,6 +7,7 @@ on:
   pull_request:
     paths:
       - src/promptflow-core/**
+      - src/promptflow-devkit/**
       - src/promptflow/**
       - scripts/building/**
       - src/promptflow-tracing/**
@@ -89,6 +90,7 @@ jobs:
           pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
           pip install ${{ github.workspace }}/src/promptflow-tracing
           pip install ${{ github.workspace }}/src/promptflow-core
+          pip install ${{ github.workspace }}/src/promptflow-devkit
           gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
           gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
           pip freeze

.github/workflows/sdk-cli-azure-test.yml

@@ -88,6 +88,7 @@ jobs:
           pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
           pip install ${{ github.workspace }}/src/promptflow-tracing
           pip install ${{ github.workspace }}/src/promptflow-core
+          pip install ${{ github.workspace }}/src/promptflow-devkit
           gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
           gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
           pip freeze

.github/workflows/sdk-cli-perf-monitor-test.yml

@@ -7,6 +7,8 @@ on:
   pull_request:
     paths:
       - src/promptflow/**
+      - src/promptflow-core/**
+      - src/promptflow-devkit/**
       - scripts/building/**
       - .github/workflows/sdk-cli-perf-monitor-test.yml
 
@@ -81,6 +83,7 @@ jobs:
           pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
           pip install ${{ github.workspace }}/src/promptflow-tracing
           pip install ${{ github.workspace }}/src/promptflow-core
+          pip install ${{ github.workspace }}/src/promptflow-devkit
           gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure,executable,azureml-serving]"}}
           gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
           pip freeze

setup.cfg

@@ -11,7 +11,9 @@ exclude =
     docs/*
     venv,.pytest_cache
     build
-    src/promptflow/promptflow/azure/_restclient
+    src/promptflow-azure/promptflow/azure/_restclient
+    src/promptflow-azure/promptflow/azure/_models
+    src/promptflow/promptflow/core/_connection_provider/_models
     src/promptflow/promptflow/azure/_models
     src/promptflow-core/promptflow/core/_connection_provider/_models
     src/promptflow/tests/test_configs/*

src/promptflow/promptflow/_cli/_pf/_run.py → src/promptflow-devkit/promptflow/_cli/_pf/_run.py

@@ -635,7 +635,7 @@ def _build_run_obj():
             logger.debug(f"resume_from specified, append params override {params_override} to run params.")
             run_params.update(list_of_dict_to_nested_dict(params_override))
         logger.debug(f"Run params: {run_params}")
-        run = resume_func(**run_params, stream=stream)
+        run = resume_func(**run_params)
     else:
         run = create_func(run=_build_run_obj(), stream=stream)
     if stream:

src/promptflow-devkit/promptflow/_cli/data/entry_flow/requirements_txt

@@ -0,0 +1 @@
+promptflow

src/promptflow/promptflow/_cli/data/package_tool/README.md.jinja2 → src/promptflow-devkit/promptflow/_cli/data/package_tool/README.md.jinja2

@@ -11,4 +11,4 @@ The directory structure in the package tool is as follows:
         __init__.py
 ```
 
-Please refer to [tool doc](https://microsoft.github.io/promptflow/how-to-guides/develop-a-tool/index.html) for more details about how to develop a tool.
+Please refer to [tool doc](https://microsoft.github.io/promptflow/how-to-guides/develop-a-tool/index.html) for more details about how to develop a tool.

src/promptflow/promptflow/_proxy/_base_inspector_proxy.py → src/promptflow-devkit/promptflow/_proxy/_base_inspector_proxy.py

@@ -2,7 +2,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 from pathlib import Path
-from typing import Dict, List
+from typing import List
 
 
 class AbstractInspectorProxy:
@@ -22,8 +22,6 @@ class AbstractInspectorProxy:
     def __init__(self):
         pass
 
-    def get_used_connection_names(
-        self, flow_file: Path, working_dir: Path, environment_variables_overrides: Dict[str, str] = None
-    ) -> List[str]:
+    def get_used_connection_names(self, flow_file: Path, working_dir: Path) -> List[str]:
         """Check the type of each node input/attribute and return the connection names used in the flow."""
         raise NotImplementedError()

src/promptflow/promptflow/_proxy/_csharp_inspector_proxy.py → src/promptflow-devkit/promptflow/_proxy/_csharp_inspector_proxy.py

@@ -1,7 +1,7 @@
 import re
 from collections import defaultdict
 from pathlib import Path
-from typing import Dict, List
+from typing import List
 
 import pydash
 
@@ -16,10 +16,7 @@ class CSharpInspectorProxy(AbstractInspectorProxy):
     def __init__(self):
         super().__init__()
 
-    def get_used_connection_names(
-        self, flow_file: Path, working_dir: Path, environment_variables_overrides: Dict[str, str] = None
-    ) -> List[str]:
-        # TODO: support environment_variables_overrides
+    def get_used_connection_names(self, flow_file: Path, working_dir: Path) -> List[str]:
         flow_tools_json_path = working_dir / PROMPT_FLOW_DIR_NAME / FLOW_TOOLS_JSON
         tools_meta = read_json_content(flow_tools_json_path, "meta of tools")
         flow_dag = load_yaml(flow_file)

src/promptflow/promptflow/_proxy/_python_inspector_proxy.py → src/promptflow-devkit/promptflow/_proxy/_python_inspector_proxy.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Dict, List
+from typing import List
 
 from ._base_inspector_proxy import AbstractInspectorProxy
 
@@ -8,12 +8,10 @@ class PythonInspectorProxy(AbstractInspectorProxy):
     def __init__(self):
         super().__init__()
 
-    def get_used_connection_names(
-        self, flow_file: Path, working_dir: Path, environment_variables_overrides: Dict[str, str] = None
-    ) -> List[str]:
+    def get_used_connection_names(self, flow_file: Path, working_dir: Path) -> List[str]:
         from promptflow._utils.context_utils import _change_working_dir
         from promptflow.contracts.flow import Flow as ExecutableFlow
 
         with _change_working_dir(working_dir):
             executable = ExecutableFlow.from_yaml(flow_file=flow_file, working_dir=working_dir)
-        return executable.get_connection_names(environment_variables_overrides=environment_variables_overrides)
+        return executable.get_connection_names()

src/promptflow/promptflow/_sdk/_constants.py → src/promptflow-devkit/promptflow/_sdk/_constants.py

@@ -12,7 +12,7 @@
     PROMPT_FLOW_DIR_NAME,
     ConnectionAuthMode,
     ConnectionType,
-    CustomStrongTypeConnectionConfigs, CONNECTION_SCRUBBED_VALUE_NO_CHANGE,
+    CustomStrongTypeConnectionConfigs,
 )
 
 LOGGER_NAME = "promptflow"
@@ -113,7 +113,7 @@ def _prepare_home_dir() -> Path:
 REFRESH_CONNECTIONS_DIR_LOCK_PATH = (HOME_PROMPT_FLOW_DIR / "refresh_connections_dir.lock").resolve()
 # Note: Use this only for show. Reading input should regard all '*' string as scrubbed, no matter the length.
 SCRUBBED_VALUE = CONNECTION_SCRUBBED_VALUE
-SCRUBBED_VALUE_NO_CHANGE = CONNECTION_SCRUBBED_VALUE_NO_CHANGE
+SCRUBBED_VALUE_NO_CHANGE = "<no-change>"
 SCRUBBED_VALUE_USER_INPUT = "<user-input>"
 WORKSPACE_LINKED_DATASTORE_NAME = "workspaceblobstore"
 LINE_NUMBER = "line_number"

src/promptflow/promptflow/_sdk/_orm/__init__.py → src/promptflow-devkit/promptflow/_sdk/_orm/__init__.py

@@ -4,9 +4,9 @@
 
 __path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore
 
-from promptflow._sdk._orm.run_info import RunInfo
-from promptflow._sdk._orm.orchestrator import Orchestrator
 from promptflow._sdk._orm.experiment_node_run import ExperimentNodeRun
+from promptflow._sdk._orm.orchestrator import Orchestrator
+from promptflow._sdk._orm.run_info import RunInfo
 
 from .connection import Connection
 from .experiment import Experiment

src/promptflow/promptflow/_sdk/_service/README.md → src/promptflow-devkit/promptflow/_sdk/_service/README.md

@@ -20,13 +20,13 @@ If you don't specify a port to start service, pfs will first use the port in the
 If not found port configuration or the port is used, pfs will use a random port to start the service.
 
 ### Swagger of service
-After start the service, it will provide Swagger UI documentation, served from "http://localhost:your-port/v1.0/swagger.json". 
+After start the service, it will provide Swagger UI documentation, served from "http://localhost:your-port/v1.0/swagger.json".
 
 For details, please refer to [swagger.json](./swagger.json).
 
 #### Generate C# client
-1. Right click the project, Add -> Rest API Client... -> Generate with OpenAPI Generator 
-   
+1. Right click the project, Add -> Rest API Client... -> Generate with OpenAPI Generator
+
 2. It will open a dialog, fill in the file name and swagger url, it will generate the client under the project.
 
-For details, please refer to [REST API Client Code Generator](https://marketplace.visualstudio.com/items?itemName=ChristianResmaHelle.ApiClientCodeGenerator2022).
+For details, please refer to [REST API Client Code Generator](https://marketplace.visualstudio.com/items?itemName=ChristianResmaHelle.ApiClientCodeGenerator2022).

src/promptflow/promptflow/_sdk/_service/entry.py → src/promptflow-devkit/promptflow/_sdk/_service/entry.py

@@ -138,7 +138,7 @@ def validate_port(port, force_start):
                 app.logger.setLevel(logging.DEBUG)
             else:
                 app.logger.setLevel(logging.INFO)
-            message = f"Start Prompt Flow Service on {port}, version: {get_promptflow_sdk_version()}."
+            message = f"Start Prompt Flow Service on {port}, version: {get_promptflow_sdk_version()}"
             app.logger.info(message)
             print(message)
             sys.stdout.flush()
@@ -197,7 +197,7 @@ def validate_port(port, force_start):
             subprocess.Popen(cmd, stdout=subprocess.DEVNULL, start_new_session=True)
         is_healthy = check_pfs_service_status(port)
         if is_healthy:
-            message = f"Start Prompt Flow Service on port {port}, version: {get_promptflow_sdk_version()}."
+            message = f"Start Prompt Flow Service on {port}, version: {get_promptflow_sdk_version()}"
             print(message)
             logger.info(message)
         else:

src/promptflow/promptflow/_sdk/_service/pfsvc.py → src/promptflow-devkit/promptflow/_sdk/_service/pfsvc.py

@@ -1,12 +1,13 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
-from promptflow._sdk._service.entry import main
 import sys
 
-import win32serviceutil  # ServiceFramework and commandline helper
-import win32service  # Events
 import servicemanager  # Simple setup and logging
+import win32service  # Events
+import win32serviceutil  # ServiceFramework and commandline helper
+
+from promptflow._sdk._service.entry import main
 
 
 class PromptFlowService:

src/promptflow/promptflow/_sdk/_service/static/assets/index.mjs → src/promptflow-devkit/promptflow/_sdk/_service/static/assets/index.mjs

@@ -631,12 +631,12 @@ function DFe() {
           var Rr = Se._result;
           return Rr === void 0 && fe(`lazy: Expected the result of a dynamic import() call. Instead received: %s
 
-Your code should look like: 
+Your code should look like:
   const MyComponent = lazy(() => import('./MyComponent'))
 
 Did you accidentally put curly braces around the import?`, Rr), "default" in Rr || fe(`lazy: Expected the result of a dynamic import() call. Instead received: %s
 
-Your code should look like: 
+Your code should look like:
   const MyComponent = lazy(() => import('./MyComponent'))`, Rr), Rr.default;
         } else
           throw Se._result;
@@ -4471,7 +4471,7 @@ var QO = [], X$ = void 0;
 function MBe(e) {
   var t = j1.__options, r = 2e3, n = 10;
   t.disableWarnings || (QO.push(e), X$ === void 0 && (X$ = setTimeout(function() {
-    Ab(`Some icons were re-registered. Applications should only call registerIcons for any given icon once. Redefining what an icon is may have unintended consequences. Duplicates include: 
+    Ab(`Some icons were re-registered. Applications should only call registerIcons for any given icon once. Redefining what an icon is may have unintended consequences. Duplicates include:
 ` + QO.slice(0, n).join(", ") + (QO.length > n ? " (+ ".concat(QO.length - n, " more)") : "")), X$ = void 0, QO = [];
   }, r)));
 }
@@ -38766,18 +38766,18 @@ const gze = vze.compose({
   template: QPe,
   styles: pze,
   checkedIndicator: `
-		<svg 
+		<svg
 			part="checked-indicator"
 			class="checked-indicator"
-			width="16" 
-			height="16" 
-			viewBox="0 0 16 16" 
-			xmlns="http://www.w3.org/2000/svg" 
+			width="16"
+			height="16"
+			viewBox="0 0 16 16"
+			xmlns="http://www.w3.org/2000/svg"
 			fill="currentColor"
 		>
-			<path 
-				fill-rule="evenodd" 
-				clip-rule="evenodd" 
+			<path
+				fill-rule="evenodd"
+				clip-rule="evenodd"
 				d="M14.431 3.323l-8.47 10-.79-.036-3.35-4.77.818-.574 2.978 4.24 8.051-9.506.764.646z"
 			/>
 		</svg>
@@ -39034,18 +39034,18 @@ const Nze = Rze.compose({
   template: kje,
   styles: Oze,
   indicator: `
-		<svg 
+		<svg
 			class="select-indicator"
 			part="select-indicator"
-			width="16" 
-			height="16" 
-			viewBox="0 0 16 16" 
-			xmlns="http://www.w3.org/2000/svg" 
+			width="16"
+			height="16"
+			viewBox="0 0 16 16"
+			xmlns="http://www.w3.org/2000/svg"
 			fill="currentColor"
 		>
-			<path 
-				fill-rule="evenodd" 
-				clip-rule="evenodd" 
+			<path
+				fill-rule="evenodd"
+				clip-rule="evenodd"
 				d="M7.976 10.072l4.357-4.357.62.618L8.284 11h-.618L3 6.333l.619-.618 4.357 4.357z"
 			/>
 		</svg>
@@ -69889,7 +69889,7 @@ function Vft(e, t, r, n) {
     if (!o)
       return;
     if (o === t) {
-      Wft.NODE_ENV !== "production" && Pu(!1, `[JSS] A rule tries to extend itself 
+      Wft.NODE_ENV !== "production" && Pu(!1, `[JSS] A rule tries to extend itself
 ` + t.toString());
       return;
     }
@@ -70030,8 +70030,8 @@ function vK(e, t) {
   var i = e.options, o = i.parent;
   if (t[0] === "$") {
     var a = o.getRule(t.substr(1));
-    return a ? a === e ? (xpe.NODE_ENV !== "production" && Pu(!1, `[JSS] Cyclic composition detected. 
-` + e.toString()), !1) : (o.classes[e.key] += " " + o.classes[a.key], !0) : (xpe.NODE_ENV !== "production" && Pu(!1, `[JSS] Referenced rule is not defined. 
+    return a ? a === e ? (xpe.NODE_ENV !== "production" && Pu(!1, `[JSS] Cyclic composition detected.
+` + e.toString()), !1) : (o.classes[e.key] += " " + o.classes[a.key], !0) : (xpe.NODE_ENV !== "production" && Pu(!1, `[JSS] Referenced rule is not defined.
 ` + e.toString()), !1);
   }
   return o.classes[e.key] += " " + t, !0;