Separate promptflow-core package

.cspell.json

@@ -28,7 +28,7 @@
     "src/promptflow/promptflow/azure/_restclient/flow/**",
     "src/promptflow/promptflow/azure/_restclient/swagger.json",
     "src/promptflow/promptflow/azure/_models/**",
-    "src/promptflow/promptflow/core/_connection_provider/_models/**",
+    "src/promptflow-core/promptflow/core/_connection_provider/_models/**",
     "src/promptflow/tests/**",
     "src/promptflow-recording/**",
     "src/promptflow-tools/tests/**",

.github/workflows/build_doc_ci.yml

@@ -2,16 +2,16 @@ name: Build Doc CI
 
 on:
   workflow_dispatch:
-  pull_request:
-    branches:
-      - main
-      - preview/docs
-    paths:
-      - 'README.md'
-      - 'docs/**'
-      - 'scripts/docs/**'
-      - '.github/workflows/build_doc_ci.yml'
-      - 'src/promptflow/promptflow/**'
+# TODO: Enable this after package separation
+#  pull_request:
+#    branches:
+#      - main
+#    paths:
+#      - 'README.md'
+#      - 'docs/**'
+#      - 'scripts/docs/**'
+#      - '.github/workflows/build_doc_ci.yml'
+#      - 'src/promptflow/promptflow/**'
 
 env:
   packageSetupType: promptflow_with_extra

.github/workflows/promptflow-executor-e2e-test.yml

@@ -4,6 +4,7 @@ on:
     - cron: "40 20 * * *" # Every day starting at 4:40 BJT
   pull_request_target:
     paths:
+      - src/promptflow-core/*
       - src/promptflow/*
       - src/promptflow/promptflow/*
       - src/promptflow/promptflow/_core/**
@@ -104,10 +105,10 @@ jobs:
       run: |
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-core
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-        pip uninstall -y promptflow-tracing
-        pip install -e ${{ github.workspace }}/src/promptflow-tracing
         pip freeze
     - name: install recording
       run: |

.github/workflows/promptflow-executor-unit-test.yml

@@ -4,6 +4,7 @@ on:
     - cron: "40 19 * * *" # Every day starting at 3:40 BJT
   pull_request_target:
     paths:
+      - src/promptflow-core/*
       - src/promptflow/*
       - src/promptflow/promptflow/*
       - src/promptflow/promptflow/_core/**
@@ -108,10 +109,10 @@ jobs:
       run: |
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-core
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-        pip uninstall -y promptflow-tracing
-        pip install -e ${{ github.workspace }}/src/promptflow-tracing
         pip freeze
     - name: install recording
       run: |

.github/workflows/promptflow-global-config-test.yml

@@ -4,6 +4,7 @@ on:
     - cron: "40 18 * * *" # Every day starting at 2:40 BJT
   pull_request_target:
     paths:
+      - src/promptflow-core/*
       - src/promptflow/**
       - scripts/building/**
       - .github/workflows/promptflow-global-config-test.yml
@@ -60,11 +61,14 @@ jobs:
       with:
         setupType: ${{ env.packageSetupType }}
         scriptPath: ${{ env.testWorkingDirectory }}
-    - name: Install tracing
+    - name: Install dependency
       shell: pwsh
       run: |
         pip uninstall -y promptflow-tracing
-        pip install -e ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        echo "Installed promptflow-tracing"
+        pip uninstall -y promptflow-core
+        pip install ${{ github.workspace }}/src/promptflow-core
         pip freeze
     - name: Azure Login
       uses: azure/login@v1

.github/workflows/promptflow-sdk-cli-test.yml

@@ -4,6 +4,7 @@ on:
     - cron: "40 18 * * *" # Every day starting at 2:40 BJT
   pull_request:
     paths:
+      - src/promptflow-core/**
       - src/promptflow/**
       - src/promptflow-tracing/**
       - scripts/building/**
@@ -78,10 +79,10 @@ jobs:
       run: |
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+        pip install ${{ github.workspace }}/src/promptflow-tracing
+        pip install ${{ github.workspace }}/src/promptflow-core
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-        pip uninstall -y promptflow-tracing
-        pip install -e ${{ github.workspace }}/src/promptflow-tracing
         pip freeze
     - name: install recording
       run: |

.github/workflows/publish_doc.yml

@@ -2,16 +2,17 @@ name: Publish Promptflow Doc
 
 on:
   workflow_dispatch:
-  push: 
-    branches:
-      - main
-      - preview/docs
-    paths:
-      - 'README.md'
-      - 'docs/**'
-      - 'scripts/docs/**'
-      - '.github/workflows/publish_doc.yml'
-      - 'src/promptflow/promptflow/**'
+    # TODO: Enable this after package separation
+#  push:
+#    branches:
+#      - main
+#      - preview/docs
+#    paths:
+#      - 'README.md'
+#      - 'docs/**'
+#      - 'scripts/docs/**'
+#      - '.github/workflows/publish_doc.yml'
+#      - 'src/promptflow/promptflow/**'
 
 # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
 permissions:

.github/workflows/sdk-cli-azure-test-production.yml

@@ -87,6 +87,8 @@ jobs:
         run: |
           Set-PSDebug -Trace 1
           pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+          pip install ${{ github.workspace }}/src/promptflow-tracing
+          pip install ${{ github.workspace }}/src/promptflow-core
           gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
           gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
           pip freeze

.github/workflows/sdk-cli-azure-test-pull-request.yml

@@ -6,6 +6,7 @@ name: sdk-cli-azure-test-pull-request
 on:
   pull_request:
     paths:
+      - src/promptflow-core/**
       - src/promptflow/**
       - scripts/building/**
       - src/promptflow-tracing/**
@@ -86,10 +87,10 @@ jobs:
         run: |
           Set-PSDebug -Trace 1
           pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+          pip install ${{ github.workspace }}/src/promptflow-tracing
+          pip install ${{ github.workspace }}/src/promptflow-core
           gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
           gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
-          pip uninstall -y promptflow-tracing
-          pip install -e ${{ github.workspace }}/src/promptflow-tracing
           pip freeze
 
       - name: install recording

.github/workflows/sdk-cli-azure-test.yml

@@ -86,6 +86,8 @@ jobs:
         run: |
           Set-PSDebug -Trace 1
           pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+          pip install ${{ github.workspace }}/src/promptflow-tracing
+          pip install ${{ github.workspace }}/src/promptflow-core
           gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
           gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
           pip freeze

.github/workflows/sdk-cli-perf-monitor-test.yml

@@ -58,22 +58,39 @@ jobs:
       - name: Build wheel
         uses: "./.github/actions/step_sdk_setup"
         with:
-          setupType: ${{ env.packageSetupType }}
+          setupType: promptflow_with_extra
           scriptPath: ${{ env.testWorkingDirectory }}
+      - name: Upload Wheel
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheel
+          path: |
+            ${{ github.workspace }}/src/promptflow/dist/*.whl
+            ${{ github.workspace }}/src/promptflow-tools/dist/*.whl
+      - name: Download Artifacts
+        uses: actions/download-artifact@v3
+        with:
+          name: wheel
+          path: artifacts
+      - name: Install wheel
+        shell: pwsh
+        working-directory: artifacts
+        run: |
+          Set-PSDebug -Trace 1
+          pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
+          pip install ${{ github.workspace }}/src/promptflow-tracing
+          pip install ${{ github.workspace }}/src/promptflow-core
+          gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure,executable,azureml-serving]"}}
+          gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
+          pip freeze
 
       - name: install recording
         run:
           pip install vcrpy
           pip install -e .
         working-directory: ${{ env.RECORD_DIRECTORY }}
 
-      - name: Install tracing
-        shell: pwsh
-        run: |
-          pip uninstall -y promptflow-tracing
-          pip install -e ${{ github.workspace }}/src/promptflow-tracing
-          pip freeze
-
       - name: Generate (mock) connections.json
         shell: pwsh
         working-directory: ${{ env.testWorkingDirectory }}

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 
-exclude: '(^docs/)|flows|scripts|src/promptflow/promptflow/azure/_restclient/|src/promptflow/promptflow/core/_connection_provider/_models/|src/promptflow/promptflow/azure/_models/|src/promptflow/tests/test_configs|src/promptflow-tools'
+exclude: '(^docs/)|flows|scripts|src/promptflow/promptflow/azure/_restclient/|src/promptflow-core/promptflow/core/_connection_provider/_models/|src/promptflow/promptflow/azure/_models/|src/promptflow/tests/test_configs|src/promptflow-tools'
 
 repos:
     - repo: https://github.com/pre-commit/pre-commit-hooks

scripts/check_enforcer/check_enforcer.py

@@ -47,11 +47,11 @@
         "scripts/building/**",
         ".github/workflows/promptflow-sdk-cli-test.yml",
     ],
-    "sdk_cli_global_config_tests": [
-        "src/promptflow/**",
-        "scripts/building/**",
-        ".github/workflows/promptflow-global-config-test.yml",
-    ],
+    # "sdk_cli_global_config_tests": [
+    #     "src/promptflow/**",
+    #     "scripts/building/**",
+    #     ".github/workflows/promptflow-global-config-test.yml",
+    # ],
     "sdk_cli_azure_test_replay": [
         "src/promptflow/**",
         "scripts/building/**",

scripts/compliance-check/user_exclusion.xml

@@ -1,7 +1,7 @@
 <PoliCheckExclusions>
   <!-- All strings must be UPPER CASE -->
   <!--index-xxx.js is an auto-generated javascript file - skipped given it's not expected to be readable -->
-  <Exclusion Type="FileName">SRC\PROMPTFLOW\PROMPTFLOW\CORE\_SERVING\STATIC\INDEX.JS</Exclusion>
+  <Exclusion Type="FileName">SRC\PROMPTFLOW-CORE\PROMPTFLOW\CORE\_SERVING\STATIC\INDEX.JS</Exclusion>
   <Exclusion Type="FileType">.MIN.JS</Exclusion>
   <Exclusion Type="FolderPathFull">SRC\PROMPTFLOW\PROMPTFLOW\_SDK\_SERVICE\STATIC\</Exclusion>
 </PoliCheckExclusions>

scripts/dev-setup/main.py

@@ -15,7 +15,7 @@
 PROMPT_FLOW_PKGS = [
     "promptflow-tracing",
     # TODO: uncomment below lines when the packages are ready
-    # "promptflow-core",
+    "promptflow-core",
     # "promptflow-devkit",
     # "promptflow-azure",
     "promptflow[azure]",

setup.cfg

@@ -13,7 +13,7 @@ exclude =
     build
     src/promptflow/promptflow/azure/_restclient
     src/promptflow/promptflow/azure/_models
-    src/promptflow/promptflow/core/_connection_provider/_models
+    src/promptflow-core/promptflow/core/_connection_provider/_models
     src/promptflow/tests/test_configs/*
 import-order-style = google
 

src/promptflow/promptflow/_constants.py → src/promptflow-core/promptflow/_constants.py

@@ -7,6 +7,7 @@
 CONNECTION_NAME_PROPERTY = "__connection_name"
 CONNECTION_SECRET_KEYS = "__secret_keys"
 CONNECTION_SCRUBBED_VALUE = "******"
+CONNECTION_SCRUBBED_VALUE_NO_CHANGE = "<no-change>"
 PROMPTFLOW_CONNECTIONS = "PROMPTFLOW_CONNECTIONS"
 PROMPTFLOW_SECRETS_FILE = "PROMPTFLOW_SECRETS_FILE"
 PF_NO_INTERACTIVE_LOGIN = "PF_NO_INTERACTIVE_LOGIN"
@@ -253,3 +254,6 @@ def is_custom_key(key):
 class ConnectionProviderConfig:
     LOCAL = "local"
     AZUREML = "azureml"
+
+
+CONNECTION_DATA_CLASS_KEY = "DATA_CLASS"

src/promptflow/promptflow/_utils/credential_utils.py → src/promptflow-core/promptflow/_utils/credential_utils.py

@@ -2,10 +2,12 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
+
 def get_default_azure_credential():
     from azure.identity import DefaultAzureCredential
+
     try:
-        from azure.ai.ml._azure_environments import _get_default_cloud_name, EndpointURLS, _get_cloud, AzureEnvironments
+        from azure.ai.ml._azure_environments import AzureEnvironments, EndpointURLS, _get_cloud, _get_default_cloud_name
     except ImportError:
         return DefaultAzureCredential()
     # Support sovereign cloud cases, like mooncake, fairfax.

src/promptflow/promptflow/_utils/exception_utils.py → src/promptflow-core/promptflow/_utils/exception_utils.py

@@ -7,7 +7,7 @@
 from datetime import datetime
 from enum import Enum
 from traceback import TracebackException, format_tb
-from types import TracebackType, FrameType
+from types import FrameType, TracebackType
 
 from promptflow.exceptions import PromptflowException, SystemErrorException, UserErrorException, ValidationException
 from promptflow.tracing._operation_context import OperationContext