From 5e775102429ddea83ef2f987ab245a1e5bf7df09 Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Tue, 26 Mar 2024 18:39:15 +0800 Subject: [PATCH 01/16] Move _core, core and _executor Signed-off-by: Brynn Yin --- .../promptflow/_core/__init__.py | 0 .../promptflow/_core/_errors.py | 0 .../promptflow/_core/cache_manager.py | 0 .../promptflow/_core/connection_manager.py | 0 .../promptflow/_core/flow_execution_context.py | 0 .../promptflow/_core/log_manager.py | 0 .../promptflow/_core/metric_logger.py | 0 .../promptflow/_core/operation_context.py | 0 .../promptflow/_core/run_tracker.py | 0 .../promptflow/_core/token_provider.py | 0 .../promptflow/_core/tool.py | 0 .../promptflow/_core/tool_meta_generator.py | 0 .../promptflow/_core/tool_settings_parser.py | 0 .../promptflow/_core/tool_validation.py | 0 .../promptflow/_core/tools_manager.py | 0 src/promptflow-core/promptflow/core/__init__.py | 11 ++++++++++- .../promptflow/core/_connection.py | 0 .../core/_connection_provider/__init__.py | 0 .../_connection_provider/_connection_provider.py | 0 .../_dict_connection_provider.py | 0 .../core/_connection_provider/_models/__init__.py | 0 .../core/_connection_provider/_models/_models.py | 0 .../core/_connection_provider/_models/_version.py | 0 .../core/_connection_provider/_utils.py | 0 .../_workspace_connection_provider.py | 0 .../promptflow/core/_errors.py | 0 .../promptflow/core/_flow.py | 0 .../promptflow/core/_serving/__init__.py | 0 .../promptflow/core/_serving/_errors.py | 0 .../promptflow/core/_serving/app.py | 0 .../core/_serving/blueprint/__init__.py | 0 .../core/_serving/blueprint/monitor_blueprint.py | 0 .../_serving/blueprint/static_web_blueprint.py | 0 .../promptflow/core/_serving/constants.py | 0 .../core/_serving/extension/__init__.py | 0 .../core/_serving/extension/azureml_extension.py | 0 .../core/_serving/extension/default_extension.py | 0 .../core/_serving/extension/extension_factory.py | 0 .../core/_serving/extension/extension_type.py | 0 .../extension/otel_exporter_provider_factory.py | 0 .../promptflow/core/_serving/flow_invoker.py | 0 .../promptflow/core/_serving/flow_result.py | 0 .../promptflow/core/_serving/monitor/__init__.py | 0 .../core/_serving/monitor/data_collector.py | 0 .../core/_serving/monitor/flow_monitor.py | 0 .../core/_serving/monitor/mdc_exporter.py | 0 .../promptflow/core/_serving/monitor/metrics.py | 0 .../core/_serving/monitor/streaming_monitor.py | 0 .../core/_serving/resources/feedback_swagger.json | 0 .../promptflow/core/_serving/response_creator.py | 0 .../promptflow/core/_serving/static/index.html | 0 .../promptflow/core/_serving/static/index.js | 0 .../promptflow/core/_serving/swagger.py | 0 .../promptflow/core/_serving/utils.py | 0 .../promptflow/core/_utils.py | 0 .../promptflow/executor/__init__.py | 0 .../executor/_assistant_tool_invoker.py | 0 .../promptflow/executor/_async_nodes_scheduler.py | 0 .../promptflow/executor/_dag_manager.py | 0 .../promptflow/executor/_docstring_parser.py | 0 .../promptflow/executor/_errors.py | 0 .../promptflow/executor/_flow_nodes_scheduler.py | 0 .../executor/_input_assignment_parser.py | 0 .../executor/_line_execution_process_pool.py | 0 .../promptflow/executor/_process_manager.py | 0 .../promptflow/executor/_result.py | 0 .../promptflow/executor/_script_executor.py | 0 .../promptflow/executor/_service/__init__.py | 0 .../promptflow/executor/_service/_errors.py | 0 .../promptflow/executor/_service/apis/__init__.py | 0 .../promptflow/executor/_service/apis/batch.py | 0 .../promptflow/executor/_service/apis/common.py | 0 .../executor/_service/apis/execution.py | 0 .../promptflow/executor/_service/apis/tool.py | 0 .../promptflow/executor/_service/app.py | 0 .../executor/_service/contracts/__init__.py | 0 .../executor/_service/contracts/base_request.py | 0 .../executor/_service/contracts/batch_request.py | 0 .../_service/contracts/execution_request.py | 0 .../executor/_service/contracts/tool_request.py | 0 .../executor/_service/utils/__init__.py | 0 .../executor/_service/utils/batch_coordinator.py | 0 .../executor/_service/utils/process_manager.py | 0 .../executor/_service/utils/process_utils.py | 0 .../executor/_service/utils/service_utils.py | 0 .../promptflow/executor/_tool_invoker.py | 0 .../promptflow/executor/_tool_resolver.py | 0 .../promptflow/executor/flow_executor.py | 0 .../promptflow/executor/flow_validator.py | 0 src/promptflow/promptflow/core/__init__.py | 15 --------------- 90 files changed, 10 insertions(+), 16 deletions(-) rename src/{promptflow => promptflow-core}/promptflow/_core/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/_errors.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/cache_manager.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/connection_manager.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/flow_execution_context.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/log_manager.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/metric_logger.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/operation_context.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/run_tracker.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/token_provider.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/tool.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/tool_meta_generator.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/tool_settings_parser.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/tool_validation.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_core/tools_manager.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_connection.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_connection_provider/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_connection_provider/_connection_provider.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_connection_provider/_dict_connection_provider.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_connection_provider/_models/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_connection_provider/_models/_models.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_connection_provider/_models/_version.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_connection_provider/_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_connection_provider/_workspace_connection_provider.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_errors.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_flow.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/_errors.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/app.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/blueprint/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/blueprint/monitor_blueprint.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/blueprint/static_web_blueprint.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/constants.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/extension/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/extension/azureml_extension.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/extension/default_extension.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/extension/extension_factory.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/extension/extension_type.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/extension/otel_exporter_provider_factory.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/flow_invoker.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/flow_result.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/monitor/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/monitor/data_collector.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/monitor/flow_monitor.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/monitor/mdc_exporter.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/monitor/metrics.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/monitor/streaming_monitor.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/resources/feedback_swagger.json (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/response_creator.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/static/index.html (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/static/index.js (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/swagger.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_serving/utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/core/_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_assistant_tool_invoker.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_async_nodes_scheduler.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_dag_manager.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_docstring_parser.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_errors.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_flow_nodes_scheduler.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_input_assignment_parser.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_line_execution_process_pool.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_process_manager.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_result.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_script_executor.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/_errors.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/apis/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/apis/batch.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/apis/common.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/apis/execution.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/apis/tool.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/app.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/contracts/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/contracts/base_request.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/contracts/batch_request.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/contracts/execution_request.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/contracts/tool_request.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/utils/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/utils/batch_coordinator.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/utils/process_manager.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/utils/process_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_service/utils/service_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_tool_invoker.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/_tool_resolver.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/flow_executor.py (100%) rename src/{promptflow => promptflow-core}/promptflow/executor/flow_validator.py (100%) delete mode 100644 src/promptflow/promptflow/core/__init__.py diff --git a/src/promptflow/promptflow/_core/__init__.py b/src/promptflow-core/promptflow/_core/__init__.py similarity index 100% rename from src/promptflow/promptflow/_core/__init__.py rename to src/promptflow-core/promptflow/_core/__init__.py diff --git a/src/promptflow/promptflow/_core/_errors.py b/src/promptflow-core/promptflow/_core/_errors.py similarity index 100% rename from src/promptflow/promptflow/_core/_errors.py rename to src/promptflow-core/promptflow/_core/_errors.py diff --git a/src/promptflow/promptflow/_core/cache_manager.py b/src/promptflow-core/promptflow/_core/cache_manager.py similarity index 100% rename from src/promptflow/promptflow/_core/cache_manager.py rename to src/promptflow-core/promptflow/_core/cache_manager.py diff --git a/src/promptflow/promptflow/_core/connection_manager.py b/src/promptflow-core/promptflow/_core/connection_manager.py similarity index 100% rename from src/promptflow/promptflow/_core/connection_manager.py rename to src/promptflow-core/promptflow/_core/connection_manager.py diff --git a/src/promptflow/promptflow/_core/flow_execution_context.py b/src/promptflow-core/promptflow/_core/flow_execution_context.py similarity index 100% rename from src/promptflow/promptflow/_core/flow_execution_context.py rename to src/promptflow-core/promptflow/_core/flow_execution_context.py diff --git a/src/promptflow/promptflow/_core/log_manager.py b/src/promptflow-core/promptflow/_core/log_manager.py similarity index 100% rename from src/promptflow/promptflow/_core/log_manager.py rename to src/promptflow-core/promptflow/_core/log_manager.py diff --git a/src/promptflow/promptflow/_core/metric_logger.py b/src/promptflow-core/promptflow/_core/metric_logger.py similarity index 100% rename from src/promptflow/promptflow/_core/metric_logger.py rename to src/promptflow-core/promptflow/_core/metric_logger.py diff --git a/src/promptflow/promptflow/_core/operation_context.py b/src/promptflow-core/promptflow/_core/operation_context.py similarity index 100% rename from src/promptflow/promptflow/_core/operation_context.py rename to src/promptflow-core/promptflow/_core/operation_context.py diff --git a/src/promptflow/promptflow/_core/run_tracker.py b/src/promptflow-core/promptflow/_core/run_tracker.py similarity index 100% rename from src/promptflow/promptflow/_core/run_tracker.py rename to src/promptflow-core/promptflow/_core/run_tracker.py diff --git a/src/promptflow/promptflow/_core/token_provider.py b/src/promptflow-core/promptflow/_core/token_provider.py similarity index 100% rename from src/promptflow/promptflow/_core/token_provider.py rename to src/promptflow-core/promptflow/_core/token_provider.py diff --git a/src/promptflow/promptflow/_core/tool.py b/src/promptflow-core/promptflow/_core/tool.py similarity index 100% rename from src/promptflow/promptflow/_core/tool.py rename to src/promptflow-core/promptflow/_core/tool.py diff --git a/src/promptflow/promptflow/_core/tool_meta_generator.py b/src/promptflow-core/promptflow/_core/tool_meta_generator.py similarity index 100% rename from src/promptflow/promptflow/_core/tool_meta_generator.py rename to src/promptflow-core/promptflow/_core/tool_meta_generator.py diff --git a/src/promptflow/promptflow/_core/tool_settings_parser.py b/src/promptflow-core/promptflow/_core/tool_settings_parser.py similarity index 100% rename from src/promptflow/promptflow/_core/tool_settings_parser.py rename to src/promptflow-core/promptflow/_core/tool_settings_parser.py diff --git a/src/promptflow/promptflow/_core/tool_validation.py b/src/promptflow-core/promptflow/_core/tool_validation.py similarity index 100% rename from src/promptflow/promptflow/_core/tool_validation.py rename to src/promptflow-core/promptflow/_core/tool_validation.py diff --git a/src/promptflow/promptflow/_core/tools_manager.py b/src/promptflow-core/promptflow/_core/tools_manager.py similarity index 100% rename from src/promptflow/promptflow/_core/tools_manager.py rename to src/promptflow-core/promptflow/_core/tools_manager.py diff --git a/src/promptflow-core/promptflow/core/__init__.py b/src/promptflow-core/promptflow/core/__init__.py index 555fb331eb2..803a98fe719 100644 --- a/src/promptflow-core/promptflow/core/__init__.py +++ b/src/promptflow-core/promptflow/core/__init__.py @@ -1,7 +1,16 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore +from promptflow._core.metric_logger import log_metric from ._version import __version__ -__all__ = ["__version__"] +# flake8: noqa +from promptflow._core.tool import ToolProvider, tool +from promptflow.core._flow import AsyncFlow, Flow + +# backward compatibility +log_flow_metric = log_metric + +__all__ = ["log_metric", "ToolProvider", "tool", "Flow", "AsyncFlow", "__version__"] diff --git a/src/promptflow/promptflow/core/_connection.py b/src/promptflow-core/promptflow/core/_connection.py similarity index 100% rename from src/promptflow/promptflow/core/_connection.py rename to src/promptflow-core/promptflow/core/_connection.py diff --git a/src/promptflow/promptflow/core/_connection_provider/__init__.py b/src/promptflow-core/promptflow/core/_connection_provider/__init__.py similarity index 100% rename from src/promptflow/promptflow/core/_connection_provider/__init__.py rename to src/promptflow-core/promptflow/core/_connection_provider/__init__.py diff --git a/src/promptflow/promptflow/core/_connection_provider/_connection_provider.py b/src/promptflow-core/promptflow/core/_connection_provider/_connection_provider.py similarity index 100% rename from src/promptflow/promptflow/core/_connection_provider/_connection_provider.py rename to src/promptflow-core/promptflow/core/_connection_provider/_connection_provider.py diff --git a/src/promptflow/promptflow/core/_connection_provider/_dict_connection_provider.py b/src/promptflow-core/promptflow/core/_connection_provider/_dict_connection_provider.py similarity index 100% rename from src/promptflow/promptflow/core/_connection_provider/_dict_connection_provider.py rename to src/promptflow-core/promptflow/core/_connection_provider/_dict_connection_provider.py diff --git a/src/promptflow/promptflow/core/_connection_provider/_models/__init__.py b/src/promptflow-core/promptflow/core/_connection_provider/_models/__init__.py similarity index 100% rename from src/promptflow/promptflow/core/_connection_provider/_models/__init__.py rename to src/promptflow-core/promptflow/core/_connection_provider/_models/__init__.py diff --git a/src/promptflow/promptflow/core/_connection_provider/_models/_models.py b/src/promptflow-core/promptflow/core/_connection_provider/_models/_models.py similarity index 100% rename from src/promptflow/promptflow/core/_connection_provider/_models/_models.py rename to src/promptflow-core/promptflow/core/_connection_provider/_models/_models.py diff --git a/src/promptflow/promptflow/core/_connection_provider/_models/_version.py b/src/promptflow-core/promptflow/core/_connection_provider/_models/_version.py similarity index 100% rename from src/promptflow/promptflow/core/_connection_provider/_models/_version.py rename to src/promptflow-core/promptflow/core/_connection_provider/_models/_version.py diff --git a/src/promptflow/promptflow/core/_connection_provider/_utils.py b/src/promptflow-core/promptflow/core/_connection_provider/_utils.py similarity index 100% rename from src/promptflow/promptflow/core/_connection_provider/_utils.py rename to src/promptflow-core/promptflow/core/_connection_provider/_utils.py diff --git a/src/promptflow/promptflow/core/_connection_provider/_workspace_connection_provider.py b/src/promptflow-core/promptflow/core/_connection_provider/_workspace_connection_provider.py similarity index 100% rename from src/promptflow/promptflow/core/_connection_provider/_workspace_connection_provider.py rename to src/promptflow-core/promptflow/core/_connection_provider/_workspace_connection_provider.py diff --git a/src/promptflow/promptflow/core/_errors.py b/src/promptflow-core/promptflow/core/_errors.py similarity index 100% rename from src/promptflow/promptflow/core/_errors.py rename to src/promptflow-core/promptflow/core/_errors.py diff --git a/src/promptflow/promptflow/core/_flow.py b/src/promptflow-core/promptflow/core/_flow.py similarity index 100% rename from src/promptflow/promptflow/core/_flow.py rename to src/promptflow-core/promptflow/core/_flow.py diff --git a/src/promptflow/promptflow/core/_serving/__init__.py b/src/promptflow-core/promptflow/core/_serving/__init__.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/__init__.py rename to src/promptflow-core/promptflow/core/_serving/__init__.py diff --git a/src/promptflow/promptflow/core/_serving/_errors.py b/src/promptflow-core/promptflow/core/_serving/_errors.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/_errors.py rename to src/promptflow-core/promptflow/core/_serving/_errors.py diff --git a/src/promptflow/promptflow/core/_serving/app.py b/src/promptflow-core/promptflow/core/_serving/app.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/app.py rename to src/promptflow-core/promptflow/core/_serving/app.py diff --git a/src/promptflow/promptflow/core/_serving/blueprint/__init__.py b/src/promptflow-core/promptflow/core/_serving/blueprint/__init__.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/blueprint/__init__.py rename to src/promptflow-core/promptflow/core/_serving/blueprint/__init__.py diff --git a/src/promptflow/promptflow/core/_serving/blueprint/monitor_blueprint.py b/src/promptflow-core/promptflow/core/_serving/blueprint/monitor_blueprint.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/blueprint/monitor_blueprint.py rename to src/promptflow-core/promptflow/core/_serving/blueprint/monitor_blueprint.py diff --git a/src/promptflow/promptflow/core/_serving/blueprint/static_web_blueprint.py b/src/promptflow-core/promptflow/core/_serving/blueprint/static_web_blueprint.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/blueprint/static_web_blueprint.py rename to src/promptflow-core/promptflow/core/_serving/blueprint/static_web_blueprint.py diff --git a/src/promptflow/promptflow/core/_serving/constants.py b/src/promptflow-core/promptflow/core/_serving/constants.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/constants.py rename to src/promptflow-core/promptflow/core/_serving/constants.py diff --git a/src/promptflow/promptflow/core/_serving/extension/__init__.py b/src/promptflow-core/promptflow/core/_serving/extension/__init__.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/extension/__init__.py rename to src/promptflow-core/promptflow/core/_serving/extension/__init__.py diff --git a/src/promptflow/promptflow/core/_serving/extension/azureml_extension.py b/src/promptflow-core/promptflow/core/_serving/extension/azureml_extension.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/extension/azureml_extension.py rename to src/promptflow-core/promptflow/core/_serving/extension/azureml_extension.py diff --git a/src/promptflow/promptflow/core/_serving/extension/default_extension.py b/src/promptflow-core/promptflow/core/_serving/extension/default_extension.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/extension/default_extension.py rename to src/promptflow-core/promptflow/core/_serving/extension/default_extension.py diff --git a/src/promptflow/promptflow/core/_serving/extension/extension_factory.py b/src/promptflow-core/promptflow/core/_serving/extension/extension_factory.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/extension/extension_factory.py rename to src/promptflow-core/promptflow/core/_serving/extension/extension_factory.py diff --git a/src/promptflow/promptflow/core/_serving/extension/extension_type.py b/src/promptflow-core/promptflow/core/_serving/extension/extension_type.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/extension/extension_type.py rename to src/promptflow-core/promptflow/core/_serving/extension/extension_type.py diff --git a/src/promptflow/promptflow/core/_serving/extension/otel_exporter_provider_factory.py b/src/promptflow-core/promptflow/core/_serving/extension/otel_exporter_provider_factory.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/extension/otel_exporter_provider_factory.py rename to src/promptflow-core/promptflow/core/_serving/extension/otel_exporter_provider_factory.py diff --git a/src/promptflow/promptflow/core/_serving/flow_invoker.py b/src/promptflow-core/promptflow/core/_serving/flow_invoker.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/flow_invoker.py rename to src/promptflow-core/promptflow/core/_serving/flow_invoker.py diff --git a/src/promptflow/promptflow/core/_serving/flow_result.py b/src/promptflow-core/promptflow/core/_serving/flow_result.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/flow_result.py rename to src/promptflow-core/promptflow/core/_serving/flow_result.py diff --git a/src/promptflow/promptflow/core/_serving/monitor/__init__.py b/src/promptflow-core/promptflow/core/_serving/monitor/__init__.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/monitor/__init__.py rename to src/promptflow-core/promptflow/core/_serving/monitor/__init__.py diff --git a/src/promptflow/promptflow/core/_serving/monitor/data_collector.py b/src/promptflow-core/promptflow/core/_serving/monitor/data_collector.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/monitor/data_collector.py rename to src/promptflow-core/promptflow/core/_serving/monitor/data_collector.py diff --git a/src/promptflow/promptflow/core/_serving/monitor/flow_monitor.py b/src/promptflow-core/promptflow/core/_serving/monitor/flow_monitor.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/monitor/flow_monitor.py rename to src/promptflow-core/promptflow/core/_serving/monitor/flow_monitor.py diff --git a/src/promptflow/promptflow/core/_serving/monitor/mdc_exporter.py b/src/promptflow-core/promptflow/core/_serving/monitor/mdc_exporter.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/monitor/mdc_exporter.py rename to src/promptflow-core/promptflow/core/_serving/monitor/mdc_exporter.py diff --git a/src/promptflow/promptflow/core/_serving/monitor/metrics.py b/src/promptflow-core/promptflow/core/_serving/monitor/metrics.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/monitor/metrics.py rename to src/promptflow-core/promptflow/core/_serving/monitor/metrics.py diff --git a/src/promptflow/promptflow/core/_serving/monitor/streaming_monitor.py b/src/promptflow-core/promptflow/core/_serving/monitor/streaming_monitor.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/monitor/streaming_monitor.py rename to src/promptflow-core/promptflow/core/_serving/monitor/streaming_monitor.py diff --git a/src/promptflow/promptflow/core/_serving/resources/feedback_swagger.json b/src/promptflow-core/promptflow/core/_serving/resources/feedback_swagger.json similarity index 100% rename from src/promptflow/promptflow/core/_serving/resources/feedback_swagger.json rename to src/promptflow-core/promptflow/core/_serving/resources/feedback_swagger.json diff --git a/src/promptflow/promptflow/core/_serving/response_creator.py b/src/promptflow-core/promptflow/core/_serving/response_creator.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/response_creator.py rename to src/promptflow-core/promptflow/core/_serving/response_creator.py diff --git a/src/promptflow/promptflow/core/_serving/static/index.html b/src/promptflow-core/promptflow/core/_serving/static/index.html similarity index 100% rename from src/promptflow/promptflow/core/_serving/static/index.html rename to src/promptflow-core/promptflow/core/_serving/static/index.html diff --git a/src/promptflow/promptflow/core/_serving/static/index.js b/src/promptflow-core/promptflow/core/_serving/static/index.js similarity index 100% rename from src/promptflow/promptflow/core/_serving/static/index.js rename to src/promptflow-core/promptflow/core/_serving/static/index.js diff --git a/src/promptflow/promptflow/core/_serving/swagger.py b/src/promptflow-core/promptflow/core/_serving/swagger.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/swagger.py rename to src/promptflow-core/promptflow/core/_serving/swagger.py diff --git a/src/promptflow/promptflow/core/_serving/utils.py b/src/promptflow-core/promptflow/core/_serving/utils.py similarity index 100% rename from src/promptflow/promptflow/core/_serving/utils.py rename to src/promptflow-core/promptflow/core/_serving/utils.py diff --git a/src/promptflow/promptflow/core/_utils.py b/src/promptflow-core/promptflow/core/_utils.py similarity index 100% rename from src/promptflow/promptflow/core/_utils.py rename to src/promptflow-core/promptflow/core/_utils.py diff --git a/src/promptflow/promptflow/executor/__init__.py b/src/promptflow-core/promptflow/executor/__init__.py similarity index 100% rename from src/promptflow/promptflow/executor/__init__.py rename to src/promptflow-core/promptflow/executor/__init__.py diff --git a/src/promptflow/promptflow/executor/_assistant_tool_invoker.py b/src/promptflow-core/promptflow/executor/_assistant_tool_invoker.py similarity index 100% rename from src/promptflow/promptflow/executor/_assistant_tool_invoker.py rename to src/promptflow-core/promptflow/executor/_assistant_tool_invoker.py diff --git a/src/promptflow/promptflow/executor/_async_nodes_scheduler.py b/src/promptflow-core/promptflow/executor/_async_nodes_scheduler.py similarity index 100% rename from src/promptflow/promptflow/executor/_async_nodes_scheduler.py rename to src/promptflow-core/promptflow/executor/_async_nodes_scheduler.py diff --git a/src/promptflow/promptflow/executor/_dag_manager.py b/src/promptflow-core/promptflow/executor/_dag_manager.py similarity index 100% rename from src/promptflow/promptflow/executor/_dag_manager.py rename to src/promptflow-core/promptflow/executor/_dag_manager.py diff --git a/src/promptflow/promptflow/executor/_docstring_parser.py b/src/promptflow-core/promptflow/executor/_docstring_parser.py similarity index 100% rename from src/promptflow/promptflow/executor/_docstring_parser.py rename to src/promptflow-core/promptflow/executor/_docstring_parser.py diff --git a/src/promptflow/promptflow/executor/_errors.py b/src/promptflow-core/promptflow/executor/_errors.py similarity index 100% rename from src/promptflow/promptflow/executor/_errors.py rename to src/promptflow-core/promptflow/executor/_errors.py diff --git a/src/promptflow/promptflow/executor/_flow_nodes_scheduler.py b/src/promptflow-core/promptflow/executor/_flow_nodes_scheduler.py similarity index 100% rename from src/promptflow/promptflow/executor/_flow_nodes_scheduler.py rename to src/promptflow-core/promptflow/executor/_flow_nodes_scheduler.py diff --git a/src/promptflow/promptflow/executor/_input_assignment_parser.py b/src/promptflow-core/promptflow/executor/_input_assignment_parser.py similarity index 100% rename from src/promptflow/promptflow/executor/_input_assignment_parser.py rename to src/promptflow-core/promptflow/executor/_input_assignment_parser.py diff --git a/src/promptflow/promptflow/executor/_line_execution_process_pool.py b/src/promptflow-core/promptflow/executor/_line_execution_process_pool.py similarity index 100% rename from src/promptflow/promptflow/executor/_line_execution_process_pool.py rename to src/promptflow-core/promptflow/executor/_line_execution_process_pool.py diff --git a/src/promptflow/promptflow/executor/_process_manager.py b/src/promptflow-core/promptflow/executor/_process_manager.py similarity index 100% rename from src/promptflow/promptflow/executor/_process_manager.py rename to src/promptflow-core/promptflow/executor/_process_manager.py diff --git a/src/promptflow/promptflow/executor/_result.py b/src/promptflow-core/promptflow/executor/_result.py similarity index 100% rename from src/promptflow/promptflow/executor/_result.py rename to src/promptflow-core/promptflow/executor/_result.py diff --git a/src/promptflow/promptflow/executor/_script_executor.py b/src/promptflow-core/promptflow/executor/_script_executor.py similarity index 100% rename from src/promptflow/promptflow/executor/_script_executor.py rename to src/promptflow-core/promptflow/executor/_script_executor.py diff --git a/src/promptflow/promptflow/executor/_service/__init__.py b/src/promptflow-core/promptflow/executor/_service/__init__.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/__init__.py rename to src/promptflow-core/promptflow/executor/_service/__init__.py diff --git a/src/promptflow/promptflow/executor/_service/_errors.py b/src/promptflow-core/promptflow/executor/_service/_errors.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/_errors.py rename to src/promptflow-core/promptflow/executor/_service/_errors.py diff --git a/src/promptflow/promptflow/executor/_service/apis/__init__.py b/src/promptflow-core/promptflow/executor/_service/apis/__init__.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/apis/__init__.py rename to src/promptflow-core/promptflow/executor/_service/apis/__init__.py diff --git a/src/promptflow/promptflow/executor/_service/apis/batch.py b/src/promptflow-core/promptflow/executor/_service/apis/batch.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/apis/batch.py rename to src/promptflow-core/promptflow/executor/_service/apis/batch.py diff --git a/src/promptflow/promptflow/executor/_service/apis/common.py b/src/promptflow-core/promptflow/executor/_service/apis/common.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/apis/common.py rename to src/promptflow-core/promptflow/executor/_service/apis/common.py diff --git a/src/promptflow/promptflow/executor/_service/apis/execution.py b/src/promptflow-core/promptflow/executor/_service/apis/execution.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/apis/execution.py rename to src/promptflow-core/promptflow/executor/_service/apis/execution.py diff --git a/src/promptflow/promptflow/executor/_service/apis/tool.py b/src/promptflow-core/promptflow/executor/_service/apis/tool.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/apis/tool.py rename to src/promptflow-core/promptflow/executor/_service/apis/tool.py diff --git a/src/promptflow/promptflow/executor/_service/app.py b/src/promptflow-core/promptflow/executor/_service/app.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/app.py rename to src/promptflow-core/promptflow/executor/_service/app.py diff --git a/src/promptflow/promptflow/executor/_service/contracts/__init__.py b/src/promptflow-core/promptflow/executor/_service/contracts/__init__.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/contracts/__init__.py rename to src/promptflow-core/promptflow/executor/_service/contracts/__init__.py diff --git a/src/promptflow/promptflow/executor/_service/contracts/base_request.py b/src/promptflow-core/promptflow/executor/_service/contracts/base_request.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/contracts/base_request.py rename to src/promptflow-core/promptflow/executor/_service/contracts/base_request.py diff --git a/src/promptflow/promptflow/executor/_service/contracts/batch_request.py b/src/promptflow-core/promptflow/executor/_service/contracts/batch_request.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/contracts/batch_request.py rename to src/promptflow-core/promptflow/executor/_service/contracts/batch_request.py diff --git a/src/promptflow/promptflow/executor/_service/contracts/execution_request.py b/src/promptflow-core/promptflow/executor/_service/contracts/execution_request.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/contracts/execution_request.py rename to src/promptflow-core/promptflow/executor/_service/contracts/execution_request.py diff --git a/src/promptflow/promptflow/executor/_service/contracts/tool_request.py b/src/promptflow-core/promptflow/executor/_service/contracts/tool_request.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/contracts/tool_request.py rename to src/promptflow-core/promptflow/executor/_service/contracts/tool_request.py diff --git a/src/promptflow/promptflow/executor/_service/utils/__init__.py b/src/promptflow-core/promptflow/executor/_service/utils/__init__.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/utils/__init__.py rename to src/promptflow-core/promptflow/executor/_service/utils/__init__.py diff --git a/src/promptflow/promptflow/executor/_service/utils/batch_coordinator.py b/src/promptflow-core/promptflow/executor/_service/utils/batch_coordinator.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/utils/batch_coordinator.py rename to src/promptflow-core/promptflow/executor/_service/utils/batch_coordinator.py diff --git a/src/promptflow/promptflow/executor/_service/utils/process_manager.py b/src/promptflow-core/promptflow/executor/_service/utils/process_manager.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/utils/process_manager.py rename to src/promptflow-core/promptflow/executor/_service/utils/process_manager.py diff --git a/src/promptflow/promptflow/executor/_service/utils/process_utils.py b/src/promptflow-core/promptflow/executor/_service/utils/process_utils.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/utils/process_utils.py rename to src/promptflow-core/promptflow/executor/_service/utils/process_utils.py diff --git a/src/promptflow/promptflow/executor/_service/utils/service_utils.py b/src/promptflow-core/promptflow/executor/_service/utils/service_utils.py similarity index 100% rename from src/promptflow/promptflow/executor/_service/utils/service_utils.py rename to src/promptflow-core/promptflow/executor/_service/utils/service_utils.py diff --git a/src/promptflow/promptflow/executor/_tool_invoker.py b/src/promptflow-core/promptflow/executor/_tool_invoker.py similarity index 100% rename from src/promptflow/promptflow/executor/_tool_invoker.py rename to src/promptflow-core/promptflow/executor/_tool_invoker.py diff --git a/src/promptflow/promptflow/executor/_tool_resolver.py b/src/promptflow-core/promptflow/executor/_tool_resolver.py similarity index 100% rename from src/promptflow/promptflow/executor/_tool_resolver.py rename to src/promptflow-core/promptflow/executor/_tool_resolver.py diff --git a/src/promptflow/promptflow/executor/flow_executor.py b/src/promptflow-core/promptflow/executor/flow_executor.py similarity index 100% rename from src/promptflow/promptflow/executor/flow_executor.py rename to src/promptflow-core/promptflow/executor/flow_executor.py diff --git a/src/promptflow/promptflow/executor/flow_validator.py b/src/promptflow-core/promptflow/executor/flow_validator.py similarity index 100% rename from src/promptflow/promptflow/executor/flow_validator.py rename to src/promptflow-core/promptflow/executor/flow_validator.py diff --git a/src/promptflow/promptflow/core/__init__.py b/src/promptflow/promptflow/core/__init__.py deleted file mode 100644 index f83d5a2d1b4..00000000000 --- a/src/promptflow/promptflow/core/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# --------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# --------------------------------------------------------- -__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore - -from promptflow._core.metric_logger import log_metric - -# flake8: noqa -from promptflow._core.tool import ToolProvider, tool -from promptflow.core._flow import AsyncFlow, Flow - -# backward compatibility -log_flow_metric = log_metric - -__all__ = ["log_metric", "ToolProvider", "tool", "Flow", "AsyncFlow"] From eae4d65e1402cf63e3d3d5e8cc7c6975368bc18d Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Tue, 26 Mar 2024 20:23:22 +0800 Subject: [PATCH 02/16] Move folder Signed-off-by: Brynn Yin --- .../promptflow-executor-e2e-test.yml | 4 +- .../promptflow-executor-unit-test.yml | 4 +- .../promptflow-global-config-test.yml | 4 +- .github/workflows/promptflow-sdk-cli-test.yml | 4 +- .github/workflows/sdk-cli-azure-test.yml | 2 + .pre-commit-config.yaml | 2 +- setup.cfg | 2 +- .../promptflow/_constants.py | 0 .../promptflow/_utils/__init__.py | 0 .../promptflow/_utils/_errors.py | 0 .../promptflow/_utils/async_utils.py | 0 .../promptflow/_utils/connection_utils.py | 0 .../promptflow/_utils/context_utils.py | 0 .../promptflow/_utils/credential_scrubber.py | 0 .../promptflow/_utils/credential_utils.py | 4 +- .../promptflow/_utils/dataclass_serializer.py | 0 .../promptflow/_utils/exception_utils.py | 2 +- .../promptflow/_utils/execution_utils.py | 64 +++++++++---------- .../promptflow/_utils/feature_utils.py | 0 .../promptflow/_utils/flow_utils.py | 0 .../promptflow/_utils/inputs_mapping_utils.py | 0 .../promptflow/_utils/load_data.py | 0 .../promptflow/_utils/logger_utils.py | 0 .../_utils/multimedia_data_converter.py | 0 .../promptflow/_utils/multimedia_utils.py | 0 .../promptflow/_utils/process_utils.py | 0 .../promptflow/_utils/retry_utils.py | 0 .../promptflow/_utils/run_tracker_utils.py | 0 .../promptflow/_utils/thread_utils.py | 0 .../promptflow/_utils/tool_utils.py | 0 .../promptflow/_utils/user_agent_utils.py | 0 .../promptflow/_utils/utils.py | 0 .../promptflow/_utils/version_hint_utils.py | 0 .../promptflow/_utils/yaml_utils.py | 0 .../promptflow/connections/__init__.py | 6 +- .../promptflow/exceptions.py | 0 .../promptflow/integrations/__init__.py | 0 .../promptflow/integrations/langchain.py | 0 .../promptflow/storage/__init__.py | 0 .../promptflow/storage/_cache_storage.py | 0 .../promptflow/storage/_errors.py | 0 .../promptflow/storage/_queue_run_storage.py | 0 .../promptflow/storage/_run_storage.py | 0 .../promptflow/storage/run_records.py | 0 src/promptflow-core/pyproject.toml | 7 ++ src/promptflow/setup.py | 6 +- 46 files changed, 60 insertions(+), 51 deletions(-) rename src/{promptflow => promptflow-core}/promptflow/_constants.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/_errors.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/async_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/connection_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/context_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/credential_scrubber.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/credential_utils.py (86%) rename src/{promptflow => promptflow-core}/promptflow/_utils/dataclass_serializer.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/exception_utils.py (99%) rename src/{promptflow => promptflow-core}/promptflow/_utils/execution_utils.py (64%) rename src/{promptflow => promptflow-core}/promptflow/_utils/feature_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/flow_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/inputs_mapping_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/load_data.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/logger_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/multimedia_data_converter.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/multimedia_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/process_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/retry_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/run_tracker_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/thread_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/tool_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/user_agent_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/version_hint_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/_utils/yaml_utils.py (100%) rename src/{promptflow => promptflow-core}/promptflow/connections/__init__.py (92%) rename src/{promptflow => promptflow-core}/promptflow/exceptions.py (100%) rename src/{promptflow => promptflow-core}/promptflow/integrations/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/integrations/langchain.py (100%) rename src/{promptflow => promptflow-core}/promptflow/storage/__init__.py (100%) rename src/{promptflow => promptflow-core}/promptflow/storage/_cache_storage.py (100%) rename src/{promptflow => promptflow-core}/promptflow/storage/_errors.py (100%) rename src/{promptflow => promptflow-core}/promptflow/storage/_queue_run_storage.py (100%) rename src/{promptflow => promptflow-core}/promptflow/storage/_run_storage.py (100%) rename src/{promptflow => promptflow-core}/promptflow/storage/run_records.py (100%) diff --git a/.github/workflows/promptflow-executor-e2e-test.yml b/.github/workflows/promptflow-executor-e2e-test.yml index b55ebfc1d29..2db03d77746 100644 --- a/.github/workflows/promptflow-executor-e2e-test.yml +++ b/.github/workflows/promptflow-executor-e2e-test.yml @@ -103,10 +103,10 @@ jobs: run: | Set-PSDebug -Trace 1 pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt + pip install ${{ github.workspace }}/src/promptflow-tracing + pip install ${{ github.workspace }}/src/promptflow-core gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}} gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} - pip uninstall -y promptflow-tracing - pip install -e ${{ github.workspace }}/src/promptflow-tracing pip freeze - name: install recording run: | diff --git a/.github/workflows/promptflow-executor-unit-test.yml b/.github/workflows/promptflow-executor-unit-test.yml index 78f3d19c24d..f1ac02bcfe5 100644 --- a/.github/workflows/promptflow-executor-unit-test.yml +++ b/.github/workflows/promptflow-executor-unit-test.yml @@ -108,10 +108,10 @@ jobs: run: | Set-PSDebug -Trace 1 pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt + pip install ${{ github.workspace }}/src/promptflow-tracing + pip install ${{ github.workspace }}/src/promptflow-core gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}} gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} - pip uninstall -y promptflow-tracing - pip install -e ${{ github.workspace }}/src/promptflow-tracing pip freeze - name: install recording run: | diff --git a/.github/workflows/promptflow-global-config-test.yml b/.github/workflows/promptflow-global-config-test.yml index 5b98bf42acb..caf9c689731 100644 --- a/.github/workflows/promptflow-global-config-test.yml +++ b/.github/workflows/promptflow-global-config-test.yml @@ -64,7 +64,9 @@ jobs: shell: pwsh run: | pip uninstall -y promptflow-tracing - pip install -e ${{ github.workspace }}/src/promptflow-tracing + pip install ${{ github.workspace }}/src/promptflow-tracing + pip uninstall -y promptflow-core + pip install ${{ github.workspace }}/src/promptflow-core pip freeze - name: Azure Login uses: azure/login@v1 diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml index dbf9c4104de..65b45c087ec 100644 --- a/.github/workflows/promptflow-sdk-cli-test.yml +++ b/.github/workflows/promptflow-sdk-cli-test.yml @@ -78,10 +78,10 @@ jobs: run: | Set-PSDebug -Trace 1 pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt + pip install ${{ github.workspace }}/src/promptflow-tracing + pip install ${{ github.workspace }}/src/promptflow-core gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}} gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} - pip uninstall -y promptflow-tracing - pip install -e ${{ github.workspace }}/src/promptflow-tracing pip freeze - name: install recording run: | diff --git a/.github/workflows/sdk-cli-azure-test.yml b/.github/workflows/sdk-cli-azure-test.yml index a960920e058..a11becbc194 100644 --- a/.github/workflows/sdk-cli-azure-test.yml +++ b/.github/workflows/sdk-cli-azure-test.yml @@ -86,6 +86,8 @@ jobs: run: | Set-PSDebug -Trace 1 pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt + pip install ${{ github.workspace }}/src/promptflow-tracing + pip install ${{ github.workspace }}/src/promptflow-core gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}} gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} pip freeze diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c66a59b8653..94cab9a9934 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks -exclude: '(^docs/)|flows|scripts|src/promptflow/promptflow/azure/_restclient/|src/promptflow/promptflow/core/_connection_provider/_models/|src/promptflow/promptflow/azure/_models/|src/promptflow/tests/test_configs|src/promptflow-tools' +exclude: '(^docs/)|flows|scripts|src/promptflow/promptflow/azure/_restclient/|src/promptflow-core/promptflow/core/_connection_provider/_models/|src/promptflow/promptflow/azure/_models/|src/promptflow/tests/test_configs|src/promptflow-tools' repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/setup.cfg b/setup.cfg index 58c328b37aa..1a6b19b1dc7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,7 @@ exclude = build src/promptflow/promptflow/azure/_restclient src/promptflow/promptflow/azure/_models - src/promptflow/promptflow/core/_connection_provider/_models + src/promptflow-core/promptflow/core/_connection_provider/_models src/promptflow/tests/test_configs/* import-order-style = google diff --git a/src/promptflow/promptflow/_constants.py b/src/promptflow-core/promptflow/_constants.py similarity index 100% rename from src/promptflow/promptflow/_constants.py rename to src/promptflow-core/promptflow/_constants.py diff --git a/src/promptflow/promptflow/_utils/__init__.py b/src/promptflow-core/promptflow/_utils/__init__.py similarity index 100% rename from src/promptflow/promptflow/_utils/__init__.py rename to src/promptflow-core/promptflow/_utils/__init__.py diff --git a/src/promptflow/promptflow/_utils/_errors.py b/src/promptflow-core/promptflow/_utils/_errors.py similarity index 100% rename from src/promptflow/promptflow/_utils/_errors.py rename to src/promptflow-core/promptflow/_utils/_errors.py diff --git a/src/promptflow/promptflow/_utils/async_utils.py b/src/promptflow-core/promptflow/_utils/async_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/async_utils.py rename to src/promptflow-core/promptflow/_utils/async_utils.py diff --git a/src/promptflow/promptflow/_utils/connection_utils.py b/src/promptflow-core/promptflow/_utils/connection_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/connection_utils.py rename to src/promptflow-core/promptflow/_utils/connection_utils.py diff --git a/src/promptflow/promptflow/_utils/context_utils.py b/src/promptflow-core/promptflow/_utils/context_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/context_utils.py rename to src/promptflow-core/promptflow/_utils/context_utils.py diff --git a/src/promptflow/promptflow/_utils/credential_scrubber.py b/src/promptflow-core/promptflow/_utils/credential_scrubber.py similarity index 100% rename from src/promptflow/promptflow/_utils/credential_scrubber.py rename to src/promptflow-core/promptflow/_utils/credential_scrubber.py diff --git a/src/promptflow/promptflow/_utils/credential_utils.py b/src/promptflow-core/promptflow/_utils/credential_utils.py similarity index 86% rename from src/promptflow/promptflow/_utils/credential_utils.py rename to src/promptflow-core/promptflow/_utils/credential_utils.py index c9254f03f6f..720fbdea45f 100644 --- a/src/promptflow/promptflow/_utils/credential_utils.py +++ b/src/promptflow-core/promptflow/_utils/credential_utils.py @@ -2,10 +2,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- + def get_default_azure_credential(): from azure.identity import DefaultAzureCredential + try: - from azure.ai.ml._azure_environments import _get_default_cloud_name, EndpointURLS, _get_cloud, AzureEnvironments + from azure.ai.ml._azure_environments import AzureEnvironments, EndpointURLS, _get_cloud, _get_default_cloud_name except ImportError: return DefaultAzureCredential() # Support sovereign cloud cases, like mooncake, fairfax. diff --git a/src/promptflow/promptflow/_utils/dataclass_serializer.py b/src/promptflow-core/promptflow/_utils/dataclass_serializer.py similarity index 100% rename from src/promptflow/promptflow/_utils/dataclass_serializer.py rename to src/promptflow-core/promptflow/_utils/dataclass_serializer.py diff --git a/src/promptflow/promptflow/_utils/exception_utils.py b/src/promptflow-core/promptflow/_utils/exception_utils.py similarity index 99% rename from src/promptflow/promptflow/_utils/exception_utils.py rename to src/promptflow-core/promptflow/_utils/exception_utils.py index a4a5be5e9ef..ee47dad0feb 100644 --- a/src/promptflow/promptflow/_utils/exception_utils.py +++ b/src/promptflow-core/promptflow/_utils/exception_utils.py @@ -7,7 +7,7 @@ from datetime import datetime from enum import Enum from traceback import TracebackException, format_tb -from types import TracebackType, FrameType +from types import FrameType, TracebackType from promptflow.exceptions import PromptflowException, SystemErrorException, UserErrorException, ValidationException from promptflow.tracing._operation_context import OperationContext diff --git a/src/promptflow/promptflow/_utils/execution_utils.py b/src/promptflow-core/promptflow/_utils/execution_utils.py similarity index 64% rename from src/promptflow/promptflow/_utils/execution_utils.py rename to src/promptflow-core/promptflow/_utils/execution_utils.py index f76f4a39398..fcf97e4793f 100644 --- a/src/promptflow/promptflow/_utils/execution_utils.py +++ b/src/promptflow-core/promptflow/_utils/execution_utils.py @@ -75,38 +75,38 @@ def _parse_aggregation_input(nodes_outputs: dict, aggregation_input_property: st def set_batch_input_source_from_inputs_mapping(inputs_mapping): """Infer the batch input source from the input mapping and set it in the OperationContext instance. - This method analyzes the `inputs_mapping` to ascertain the origin of the inputs for a batch operation. - The `inputs_mapping` should be a dictionary with keys representing input names and values specifying the sources - of these inputs. Inputs can originate from direct data or from the outputs of a previous run. - - The `inputs_mapping` is dictated entirely by the external caller. For more details on column mapping, refer to - https://aka.ms/pf/column-mapping. The mapping can include references to both the inputs and outputs of previous - runs, using a reserved source name 'run' to indicate such references. However, this method specifically checks - for references to outputs of previous runs, which are denoted by values starting with "${run.outputs". When such - a reference is found, the `batch_input_source` attribute of the OperationContext instance is set to "Run" to - reflect that the batch operation is utilizing outputs from a prior run. - - If no values in the `inputs_mapping` start with "${run.outputs", it is inferred that the inputs do not derive - from a previous run, and the `batch_input_source` is set to "Data". - - Examples of `inputs_mapping`: - - Referencing a previous run's output: - {'input1': '${run.outputs.some_output}', 'input2': 'direct_data'} - In this case, 'input1' is sourced from a prior run's output, and 'input2' is from direct data. - The `batch_input_source` would be set to "Run". - - - Sourcing directly from data: - {'input1': 'data_source1', 'input2': 'data_source2'} - Since no values start with "${run.outputs", the `batch_input_source` is set to "Data". - - Args: - inputs_mapping (Mapping[str, str]): A dictionary mapping input names to their sources, where the sources - can be either direct data or outputs from a previous run. The structure and content of this mapping are - entirely under the control of the external caller. - - Returns: - None - """ + This method analyzes the `inputs_mapping` to ascertain the origin of the inputs for a batch operation. + The `inputs_mapping` should be a dictionary with keys representing input names and values specifying the sources + of these inputs. Inputs can originate from direct data or from the outputs of a previous run. + + The `inputs_mapping` is dictated entirely by the external caller. For more details on column mapping, refer to + https://aka.ms/pf/column-mapping. The mapping can include references to both the inputs and outputs of previous + runs, using a reserved source name 'run' to indicate such references. However, this method specifically checks + for references to outputs of previous runs, which are denoted by values starting with "${run.outputs". When such + a reference is found, the `batch_input_source` attribute of the OperationContext instance is set to "Run" to + reflect that the batch operation is utilizing outputs from a prior run. + + If no values in the `inputs_mapping` start with "${run.outputs", it is inferred that the inputs do not derive + from a previous run, and the `batch_input_source` is set to "Data". + + Examples of `inputs_mapping`: + - Referencing a previous run's output: + {'input1': '${run.outputs.some_output}', 'input2': 'direct_data'} + In this case, 'input1' is sourced from a prior run's output, and 'input2' is from direct data. + The `batch_input_source` would be set to "Run". + + - Sourcing directly from data: + {'input1': 'data_source1', 'input2': 'data_source2'} + Since no values start with "${run.outputs", the `batch_input_source` is set to "Data". + + Args: + inputs_mapping (Mapping[str, str]): A dictionary mapping input names to their sources, where the sources + can be either direct data or outputs from a previous run. The structure and content of this mapping are + entirely under the control of the external caller. + + Returns: + None + """ if inputs_mapping and any( isinstance(value, str) and value.startswith("${run.outputs") for value in inputs_mapping.values() diff --git a/src/promptflow/promptflow/_utils/feature_utils.py b/src/promptflow-core/promptflow/_utils/feature_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/feature_utils.py rename to src/promptflow-core/promptflow/_utils/feature_utils.py diff --git a/src/promptflow/promptflow/_utils/flow_utils.py b/src/promptflow-core/promptflow/_utils/flow_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/flow_utils.py rename to src/promptflow-core/promptflow/_utils/flow_utils.py diff --git a/src/promptflow/promptflow/_utils/inputs_mapping_utils.py b/src/promptflow-core/promptflow/_utils/inputs_mapping_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/inputs_mapping_utils.py rename to src/promptflow-core/promptflow/_utils/inputs_mapping_utils.py diff --git a/src/promptflow/promptflow/_utils/load_data.py b/src/promptflow-core/promptflow/_utils/load_data.py similarity index 100% rename from src/promptflow/promptflow/_utils/load_data.py rename to src/promptflow-core/promptflow/_utils/load_data.py diff --git a/src/promptflow/promptflow/_utils/logger_utils.py b/src/promptflow-core/promptflow/_utils/logger_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/logger_utils.py rename to src/promptflow-core/promptflow/_utils/logger_utils.py diff --git a/src/promptflow/promptflow/_utils/multimedia_data_converter.py b/src/promptflow-core/promptflow/_utils/multimedia_data_converter.py similarity index 100% rename from src/promptflow/promptflow/_utils/multimedia_data_converter.py rename to src/promptflow-core/promptflow/_utils/multimedia_data_converter.py diff --git a/src/promptflow/promptflow/_utils/multimedia_utils.py b/src/promptflow-core/promptflow/_utils/multimedia_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/multimedia_utils.py rename to src/promptflow-core/promptflow/_utils/multimedia_utils.py diff --git a/src/promptflow/promptflow/_utils/process_utils.py b/src/promptflow-core/promptflow/_utils/process_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/process_utils.py rename to src/promptflow-core/promptflow/_utils/process_utils.py diff --git a/src/promptflow/promptflow/_utils/retry_utils.py b/src/promptflow-core/promptflow/_utils/retry_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/retry_utils.py rename to src/promptflow-core/promptflow/_utils/retry_utils.py diff --git a/src/promptflow/promptflow/_utils/run_tracker_utils.py b/src/promptflow-core/promptflow/_utils/run_tracker_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/run_tracker_utils.py rename to src/promptflow-core/promptflow/_utils/run_tracker_utils.py diff --git a/src/promptflow/promptflow/_utils/thread_utils.py b/src/promptflow-core/promptflow/_utils/thread_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/thread_utils.py rename to src/promptflow-core/promptflow/_utils/thread_utils.py diff --git a/src/promptflow/promptflow/_utils/tool_utils.py b/src/promptflow-core/promptflow/_utils/tool_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/tool_utils.py rename to src/promptflow-core/promptflow/_utils/tool_utils.py diff --git a/src/promptflow/promptflow/_utils/user_agent_utils.py b/src/promptflow-core/promptflow/_utils/user_agent_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/user_agent_utils.py rename to src/promptflow-core/promptflow/_utils/user_agent_utils.py diff --git a/src/promptflow/promptflow/_utils/utils.py b/src/promptflow-core/promptflow/_utils/utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/utils.py rename to src/promptflow-core/promptflow/_utils/utils.py diff --git a/src/promptflow/promptflow/_utils/version_hint_utils.py b/src/promptflow-core/promptflow/_utils/version_hint_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/version_hint_utils.py rename to src/promptflow-core/promptflow/_utils/version_hint_utils.py diff --git a/src/promptflow/promptflow/_utils/yaml_utils.py b/src/promptflow-core/promptflow/_utils/yaml_utils.py similarity index 100% rename from src/promptflow/promptflow/_utils/yaml_utils.py rename to src/promptflow-core/promptflow/_utils/yaml_utils.py diff --git a/src/promptflow/promptflow/connections/__init__.py b/src/promptflow-core/promptflow/connections/__init__.py similarity index 92% rename from src/promptflow/promptflow/connections/__init__.py rename to src/promptflow-core/promptflow/connections/__init__.py index c9d5b0466c2..70b29858f3a 100644 --- a/src/promptflow/promptflow/connections/__init__.py +++ b/src/promptflow-core/promptflow/connections/__init__.py @@ -4,7 +4,8 @@ from dataclasses import dataclass, is_dataclass from promptflow._core.tools_manager import register_connections -from promptflow._sdk.entities import ( +from promptflow.contracts.types import Secret +from promptflow.core._connection import ( AzureContentSafetyConnection, AzureOpenAIConnection, CognitiveSearchConnection, @@ -14,9 +15,8 @@ OpenAIConnection, SerpConnection, ServerlessConnection, + _Connection, ) -from promptflow._sdk.entities._connection import _Connection -from promptflow.contracts.types import Secret @dataclass diff --git a/src/promptflow/promptflow/exceptions.py b/src/promptflow-core/promptflow/exceptions.py similarity index 100% rename from src/promptflow/promptflow/exceptions.py rename to src/promptflow-core/promptflow/exceptions.py diff --git a/src/promptflow/promptflow/integrations/__init__.py b/src/promptflow-core/promptflow/integrations/__init__.py similarity index 100% rename from src/promptflow/promptflow/integrations/__init__.py rename to src/promptflow-core/promptflow/integrations/__init__.py diff --git a/src/promptflow/promptflow/integrations/langchain.py b/src/promptflow-core/promptflow/integrations/langchain.py similarity index 100% rename from src/promptflow/promptflow/integrations/langchain.py rename to src/promptflow-core/promptflow/integrations/langchain.py diff --git a/src/promptflow/promptflow/storage/__init__.py b/src/promptflow-core/promptflow/storage/__init__.py similarity index 100% rename from src/promptflow/promptflow/storage/__init__.py rename to src/promptflow-core/promptflow/storage/__init__.py diff --git a/src/promptflow/promptflow/storage/_cache_storage.py b/src/promptflow-core/promptflow/storage/_cache_storage.py similarity index 100% rename from src/promptflow/promptflow/storage/_cache_storage.py rename to src/promptflow-core/promptflow/storage/_cache_storage.py diff --git a/src/promptflow/promptflow/storage/_errors.py b/src/promptflow-core/promptflow/storage/_errors.py similarity index 100% rename from src/promptflow/promptflow/storage/_errors.py rename to src/promptflow-core/promptflow/storage/_errors.py diff --git a/src/promptflow/promptflow/storage/_queue_run_storage.py b/src/promptflow-core/promptflow/storage/_queue_run_storage.py similarity index 100% rename from src/promptflow/promptflow/storage/_queue_run_storage.py rename to src/promptflow-core/promptflow/storage/_queue_run_storage.py diff --git a/src/promptflow/promptflow/storage/_run_storage.py b/src/promptflow-core/promptflow/storage/_run_storage.py similarity index 100% rename from src/promptflow/promptflow/storage/_run_storage.py rename to src/promptflow-core/promptflow/storage/_run_storage.py diff --git a/src/promptflow/promptflow/storage/run_records.py b/src/promptflow-core/promptflow/storage/run_records.py similarity index 100% rename from src/promptflow/promptflow/storage/run_records.py rename to src/promptflow-core/promptflow/storage/run_records.py diff --git a/src/promptflow-core/pyproject.toml b/src/promptflow-core/pyproject.toml index 9914d7aafa0..8c8534a9cb3 100644 --- a/src/promptflow-core/pyproject.toml +++ b/src/promptflow-core/pyproject.toml @@ -36,6 +36,13 @@ packages = [ [tool.poetry.dependencies] python = "<4.0,>=3.8" +promptflow-tracing = ">=1.0.0,<2.0.0" +"ruamel.yaml" = ">=0.17.10,<1.0.0" # used to generate connection templates with preserved comments +docutils = "*" # used to generate description for tools +jsonschema = ">=4.0.0,<5.0.0" # used to validate tool +filetype = ">=1.2.0" # used to detect the mime type for mulitmedia input +flask = ">=2.2.3,<4.0.0" # Serving endpoint requirements +fastapi = {extras = ["executor-service"], version = ">=0.109.0,<1.0.0"} [tool.poetry.group.dev.dependencies] pre-commit = "*" diff --git a/src/promptflow/setup.py b/src/promptflow/setup.py index bab2641659d..98efec7c17c 100644 --- a/src/promptflow/setup.py +++ b/src/promptflow/setup.py @@ -23,7 +23,6 @@ REQUIRES = [ "psutil", # get process information when bulk run "httpx>=0.25.1", # used to send http requests asynchronously - "flask>=2.2.3,<4.0.0", # Serving endpoint requirements "sqlalchemy>=1.4.48,<3.0.0", # sqlite requirements # note that pandas 1.5.3 is the only version to test in ci before promptflow 0.1.0b7 is released # and pandas 2.x.x will be the only version to test in ci after that. @@ -42,16 +41,13 @@ "strictyaml>=1.5.0,<2.0.0", # used to identify exact location of validation error "waitress>=2.1.2,<3.0.0", # used to serve local service "azure-monitor-opentelemetry-exporter>=1.0.0b21,<2.0.0", - "ruamel.yaml>=0.17.10,<1.0.0", # used to generate connection templates with preserved comments "pyarrow>=14.0.1,<15.0.0", # used to read parquet file with pandas.read_parquet "pillow>=10.1.0,<11.0.0", # used to generate icon data URI for package tool - "filetype>=1.2.0", # used to detect the mime type for mulitmedia input - "jsonschema>=4.0.0,<5.0.0", # used to validate tool - "docutils", # used to generate description for tools "opentelemetry-exporter-otlp-proto-http>=1.22.0,<2.0.0", # trace support "flask-restx>=1.2.0,<2.0.0", # PFS Swagger "flask-cors>=4.0.0,<5.0.0", # handle PFS CORS "promptflow-tracing>=1.0.0", # tracing capabilities + "promptflow-core", # tracing capabilities ] setup( From a3436651971564e471f0de8d5526d88dcc26931e Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Tue, 26 Mar 2024 20:30:43 +0800 Subject: [PATCH 03/16] Add workflow path Signed-off-by: Brynn Yin --- .github/workflows/promptflow-executor-e2e-test.yml | 1 + .github/workflows/promptflow-executor-unit-test.yml | 1 + .github/workflows/promptflow-global-config-test.yml | 1 + .github/workflows/promptflow-sdk-cli-test.yml | 1 + .github/workflows/sdk-cli-azure-test-production.yml | 2 ++ .github/workflows/sdk-cli-azure-test-pull-request.yml | 5 +++-- 6 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/promptflow-executor-e2e-test.yml b/.github/workflows/promptflow-executor-e2e-test.yml index 2db03d77746..f8e4dcf1a5a 100644 --- a/.github/workflows/promptflow-executor-e2e-test.yml +++ b/.github/workflows/promptflow-executor-e2e-test.yml @@ -4,6 +4,7 @@ on: - cron: "40 20 * * *" # Every day starting at 4:40 BJT pull_request_target: paths: + - src/promptflow-core/* - src/promptflow/* - src/promptflow/promptflow/* - src/promptflow/promptflow/_core/** diff --git a/.github/workflows/promptflow-executor-unit-test.yml b/.github/workflows/promptflow-executor-unit-test.yml index f1ac02bcfe5..b5fdeb5b6a5 100644 --- a/.github/workflows/promptflow-executor-unit-test.yml +++ b/.github/workflows/promptflow-executor-unit-test.yml @@ -4,6 +4,7 @@ on: - cron: "40 19 * * *" # Every day starting at 3:40 BJT pull_request_target: paths: + - src/promptflow-core/* - src/promptflow/* - src/promptflow/promptflow/* - src/promptflow/promptflow/_core/** diff --git a/.github/workflows/promptflow-global-config-test.yml b/.github/workflows/promptflow-global-config-test.yml index caf9c689731..fac5e08bec4 100644 --- a/.github/workflows/promptflow-global-config-test.yml +++ b/.github/workflows/promptflow-global-config-test.yml @@ -4,6 +4,7 @@ on: - cron: "40 18 * * *" # Every day starting at 2:40 BJT pull_request_target: paths: + - src/promptflow-core/* - src/promptflow/** - scripts/building/** - .github/workflows/promptflow-global-config-test.yml diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml index 65b45c087ec..6297c16f4bc 100644 --- a/.github/workflows/promptflow-sdk-cli-test.yml +++ b/.github/workflows/promptflow-sdk-cli-test.yml @@ -4,6 +4,7 @@ on: - cron: "40 18 * * *" # Every day starting at 2:40 BJT pull_request: paths: + - src/promptflow-core/** - src/promptflow/** - src/promptflow-tracing/** - scripts/building/** diff --git a/.github/workflows/sdk-cli-azure-test-production.yml b/.github/workflows/sdk-cli-azure-test-production.yml index aaf59bad765..89d96f4ff9f 100644 --- a/.github/workflows/sdk-cli-azure-test-production.yml +++ b/.github/workflows/sdk-cli-azure-test-production.yml @@ -87,6 +87,8 @@ jobs: run: | Set-PSDebug -Trace 1 pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt + pip install ${{ github.workspace }}/src/promptflow-tracing + pip install ${{ github.workspace }}/src/promptflow-core gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}} gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} pip freeze diff --git a/.github/workflows/sdk-cli-azure-test-pull-request.yml b/.github/workflows/sdk-cli-azure-test-pull-request.yml index 28199f8d358..2955d499d98 100644 --- a/.github/workflows/sdk-cli-azure-test-pull-request.yml +++ b/.github/workflows/sdk-cli-azure-test-pull-request.yml @@ -6,6 +6,7 @@ name: sdk-cli-azure-test-pull-request on: pull_request: paths: + - src/promptflow-core/** - src/promptflow/** - scripts/building/** - src/promptflow-tracing/** @@ -86,10 +87,10 @@ jobs: run: | Set-PSDebug -Trace 1 pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt + pip install ${{ github.workspace }}/src/promptflow-tracing + pip install ${{ github.workspace }}/src/promptflow-core gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}} gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} - pip uninstall -y promptflow-tracing - pip install -e ${{ github.workspace }}/src/promptflow-tracing pip freeze - name: install recording From c5f0a6dd584646536483538e010b343f33ef8b0f Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Tue, 26 Mar 2024 20:43:13 +0800 Subject: [PATCH 04/16] Add echo in workflow Signed-off-by: Brynn Yin --- .github/workflows/promptflow-global-config-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/promptflow-global-config-test.yml b/.github/workflows/promptflow-global-config-test.yml index fac5e08bec4..0e887a5e56e 100644 --- a/.github/workflows/promptflow-global-config-test.yml +++ b/.github/workflows/promptflow-global-config-test.yml @@ -61,11 +61,12 @@ jobs: with: setupType: ${{ env.packageSetupType }} scriptPath: ${{ env.testWorkingDirectory }} - - name: Install tracing + - name: Install dependency shell: pwsh run: | pip uninstall -y promptflow-tracing pip install ${{ github.workspace }}/src/promptflow-tracing + echo "Installed promptflow-tracing" pip uninstall -y promptflow-core pip install ${{ github.workspace }}/src/promptflow-core pip freeze From 916f20a884cedcfaf33495802d49a504dde22117 Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Tue, 26 Mar 2024 20:54:15 +0800 Subject: [PATCH 05/16] Update MANIFEST Signed-off-by: Brynn Yin --- .cspell.json | 2 +- src/promptflow-core/pyproject.toml | 4 ++++ src/promptflow/MANIFEST.in | 2 -- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.cspell.json b/.cspell.json index 953c95a23bf..6f9d9c5c1ec 100644 --- a/.cspell.json +++ b/.cspell.json @@ -28,7 +28,7 @@ "src/promptflow/promptflow/azure/_restclient/flow/**", "src/promptflow/promptflow/azure/_restclient/swagger.json", "src/promptflow/promptflow/azure/_models/**", - "src/promptflow/promptflow/core/_connection_provider/_models/**", + "src/promptflow-core/promptflow/core/_connection_provider/_models/**", "src/promptflow/tests/**", "src/promptflow-recording/**", "src/promptflow-tools/tests/**", diff --git a/src/promptflow-core/pyproject.toml b/src/promptflow-core/pyproject.toml index 8c8534a9cb3..838bedfd994 100644 --- a/src/promptflow-core/pyproject.toml +++ b/src/promptflow-core/pyproject.toml @@ -2,6 +2,10 @@ name = "promptflow-core" version = "0.1.0b1" description = "Prompt flow core" +include = [ + "promptflow/core/_serving/static/*", + "promptflow/core/_serving/resources/*", +] license = "MIT" diff --git a/src/promptflow/MANIFEST.in b/src/promptflow/MANIFEST.in index 3eeecf6c71c..bdbbd599f89 100644 --- a/src/promptflow/MANIFEST.in +++ b/src/promptflow/MANIFEST.in @@ -1,6 +1,4 @@ include promptflow/azure/resources/* -include promptflow/core/_serving/static/* -include promptflow/core/_serving/resources/* include promptflow/_sdk/_service/static/* include promptflow/_sdk/_service/static/assets/* recursive-include promptflow/_cli/data * From a6dfcdc547c79281d6fc20fe0ff8c140a3983be4 Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Wed, 27 Mar 2024 11:24:29 +0800 Subject: [PATCH 06/16] Disable doc ci Signed-off-by: Brynn Yin --- .github/workflows/build_doc_ci.yml | 16 +++++++------- .github/workflows/publish_doc.yml | 21 ++++++++++--------- .../workflows/sdk-cli-perf-monitor-test.yml | 6 ++++-- scripts/check_enforcer/check_enforcer.py | 10 ++++----- scripts/dev-setup/main.py | 2 +- src/promptflow-core/pyproject.toml | 2 +- src/promptflow/setup.py | 2 +- src/promptflow/tests/conftest.py | 2 +- .../sdk_cli_test/e2etests/test_flow_run.py | 2 +- .../sdk_cli_test/e2etests/test_flow_test.py | 4 ++-- 10 files changed, 35 insertions(+), 32 deletions(-) diff --git a/.github/workflows/build_doc_ci.yml b/.github/workflows/build_doc_ci.yml index 58b07d82c95..6d2ce478059 100644 --- a/.github/workflows/build_doc_ci.yml +++ b/.github/workflows/build_doc_ci.yml @@ -4,14 +4,14 @@ on: workflow_dispatch: pull_request: branches: - - main - - preview/docs - paths: - - 'README.md' - - 'docs/**' - - 'scripts/docs/**' - - '.github/workflows/build_doc_ci.yml' - - 'src/promptflow/promptflow/**' +# TODO: Enable this after package separation +# - main +# paths: +# - 'README.md' +# - 'docs/**' +# - 'scripts/docs/**' +# - '.github/workflows/build_doc_ci.yml' +# - 'src/promptflow/promptflow/**' env: packageSetupType: promptflow_with_extra diff --git a/.github/workflows/publish_doc.yml b/.github/workflows/publish_doc.yml index 0138b45850a..6b241bb81b9 100644 --- a/.github/workflows/publish_doc.yml +++ b/.github/workflows/publish_doc.yml @@ -2,16 +2,17 @@ name: Publish Promptflow Doc on: workflow_dispatch: - push: - branches: - - main - - preview/docs - paths: - - 'README.md' - - 'docs/**' - - 'scripts/docs/**' - - '.github/workflows/publish_doc.yml' - - 'src/promptflow/promptflow/**' + push: + # TODO: Enable this after package separation +# branches: +# - main +# - preview/docs +# paths: +# - 'README.md' +# - 'docs/**' +# - 'scripts/docs/**' +# - '.github/workflows/publish_doc.yml' +# - 'src/promptflow/promptflow/**' # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages permissions: diff --git a/.github/workflows/sdk-cli-perf-monitor-test.yml b/.github/workflows/sdk-cli-perf-monitor-test.yml index 6995758ac77..0718ec994d5 100644 --- a/.github/workflows/sdk-cli-perf-monitor-test.yml +++ b/.github/workflows/sdk-cli-perf-monitor-test.yml @@ -67,11 +67,13 @@ jobs: pip install -e . working-directory: ${{ env.RECORD_DIRECTORY }} - - name: Install tracing + - name: Install dependency shell: pwsh run: | pip uninstall -y promptflow-tracing - pip install -e ${{ github.workspace }}/src/promptflow-tracing + pip install ${{ github.workspace }}/src/promptflow-tracing + pip uninstall -y promptflow-core + pip install ${{ github.workspace }}/src/promptflow-core pip freeze - name: Generate (mock) connections.json diff --git a/scripts/check_enforcer/check_enforcer.py b/scripts/check_enforcer/check_enforcer.py index 6755b95f77d..b1a3bf08659 100644 --- a/scripts/check_enforcer/check_enforcer.py +++ b/scripts/check_enforcer/check_enforcer.py @@ -47,11 +47,11 @@ "scripts/building/**", ".github/workflows/promptflow-sdk-cli-test.yml", ], - "sdk_cli_global_config_tests": [ - "src/promptflow/**", - "scripts/building/**", - ".github/workflows/promptflow-global-config-test.yml", - ], + # "sdk_cli_global_config_tests": [ + # "src/promptflow/**", + # "scripts/building/**", + # ".github/workflows/promptflow-global-config-test.yml", + # ], "sdk_cli_azure_test_replay": [ "src/promptflow/**", "scripts/building/**", diff --git a/scripts/dev-setup/main.py b/scripts/dev-setup/main.py index 848e1a2047c..183a9449c55 100644 --- a/scripts/dev-setup/main.py +++ b/scripts/dev-setup/main.py @@ -15,7 +15,7 @@ PROMPT_FLOW_PKGS = [ "promptflow-tracing", # TODO: uncomment below lines when the packages are ready - # "promptflow-core", + "promptflow-core", # "promptflow-devkit", # "promptflow-azure", "promptflow[azure]", diff --git a/src/promptflow-core/pyproject.toml b/src/promptflow-core/pyproject.toml index 838bedfd994..e017a2b05f0 100644 --- a/src/promptflow-core/pyproject.toml +++ b/src/promptflow-core/pyproject.toml @@ -44,7 +44,7 @@ promptflow-tracing = ">=1.0.0,<2.0.0" "ruamel.yaml" = ">=0.17.10,<1.0.0" # used to generate connection templates with preserved comments docutils = "*" # used to generate description for tools jsonschema = ">=4.0.0,<5.0.0" # used to validate tool -filetype = ">=1.2.0" # used to detect the mime type for mulitmedia input +filetype = ">=1.2.0" # used to detect the mime type for mulit-media input flask = ">=2.2.3,<4.0.0" # Serving endpoint requirements fastapi = {extras = ["executor-service"], version = ">=0.109.0,<1.0.0"} diff --git a/src/promptflow/setup.py b/src/promptflow/setup.py index 98efec7c17c..3261e437898 100644 --- a/src/promptflow/setup.py +++ b/src/promptflow/setup.py @@ -47,7 +47,7 @@ "flask-restx>=1.2.0,<2.0.0", # PFS Swagger "flask-cors>=4.0.0,<5.0.0", # handle PFS CORS "promptflow-tracing>=1.0.0", # tracing capabilities - "promptflow-core", # tracing capabilities + "promptflow-core", ] setup( diff --git a/src/promptflow/tests/conftest.py b/src/promptflow/tests/conftest.py index bd16e4fb503..2eadd0c9e23 100644 --- a/src/promptflow/tests/conftest.py +++ b/src/promptflow/tests/conftest.py @@ -17,7 +17,7 @@ from promptflow._constants import PROMPTFLOW_CONNECTIONS from promptflow._core.connection_manager import ConnectionManager from promptflow._utils.context_utils import _change_working_dir -from promptflow.connections import AzureOpenAIConnection +from promptflow._sdk.entities._connection import AzureOpenAIConnection load_dotenv() diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py b/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py index 99282c86df0..4621a4bc3dd 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py +++ b/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_run.py @@ -378,7 +378,7 @@ def test_basic_flow_with_script_tool_with_custom_strong_type_connection( self, install_custom_tool_pkg, local_client, pf ): # Prepare custom connection - from promptflow.connections import CustomConnection + from promptflow._sdk.entities._connection import CustomConnection conn = CustomConnection(name="custom_connection_2", secrets={"api_key": "test"}, configs={"api_url": "test"}) local_client.connections.create_or_update(conn) diff --git a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_test.py b/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_test.py index f1a063e1d30..9c693f1c303 100644 --- a/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_test.py +++ b/src/promptflow/tests/sdk_cli_test/e2etests/test_flow_test.py @@ -63,7 +63,7 @@ def test_pf_test_flow_with_package_tool_with_custom_strong_type_connection(self, def test_pf_test_flow_with_package_tool_with_custom_connection_as_input_value(self, install_custom_tool_pkg): # Prepare custom connection - from promptflow.connections import CustomConnection + from promptflow._sdk.entities._connection import CustomConnection conn = CustomConnection(name="custom_connection_3", secrets={"api_key": "test"}, configs={"api_base": "test"}) _client.connections.create_or_update(conn) @@ -77,7 +77,7 @@ def test_pf_test_flow_with_package_tool_with_custom_connection_as_input_value(se def test_pf_test_flow_with_script_tool_with_custom_strong_type_connection(self): # Prepare custom connection - from promptflow.connections import CustomConnection + from promptflow._sdk.entities._connection import CustomConnection conn = CustomConnection(name="custom_connection_2", secrets={"api_key": "test"}, configs={"api_url": "test"}) _client.connections.create_or_update(conn) From bb93ab6b319c05bb77863fde27e1883a3da18ccc Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Wed, 27 Mar 2024 11:38:19 +0800 Subject: [PATCH 07/16] Update compliance Signed-off-by: Brynn Yin --- .github/workflows/build_doc_ci.yml | 4 ++-- .github/workflows/publish_doc.yml | 2 +- scripts/compliance-check/user_exclusion.xml | 2 +- src/promptflow-core/pyproject.toml | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build_doc_ci.yml b/.github/workflows/build_doc_ci.yml index 6d2ce478059..9a1b2fa47a7 100644 --- a/.github/workflows/build_doc_ci.yml +++ b/.github/workflows/build_doc_ci.yml @@ -2,9 +2,9 @@ name: Build Doc CI on: workflow_dispatch: - pull_request: - branches: # TODO: Enable this after package separation +# pull_request: +# branches: # - main # paths: # - 'README.md' diff --git a/.github/workflows/publish_doc.yml b/.github/workflows/publish_doc.yml index 6b241bb81b9..16e562ad453 100644 --- a/.github/workflows/publish_doc.yml +++ b/.github/workflows/publish_doc.yml @@ -2,8 +2,8 @@ name: Publish Promptflow Doc on: workflow_dispatch: - push: # TODO: Enable this after package separation +# push: # branches: # - main # - preview/docs diff --git a/scripts/compliance-check/user_exclusion.xml b/scripts/compliance-check/user_exclusion.xml index 1db96c0e0ed..24044e16b19 100644 --- a/scripts/compliance-check/user_exclusion.xml +++ b/scripts/compliance-check/user_exclusion.xml @@ -1,7 +1,7 @@ - SRC\PROMPTFLOW\PROMPTFLOW\CORE\_SERVING\STATIC\INDEX.JS + SRC\PROMPTFLOW-CORE\PROMPTFLOW\CORE\_SERVING\STATIC\INDEX.JS .MIN.JS SRC\PROMPTFLOW\PROMPTFLOW\_SDK\_SERVICE\STATIC\ diff --git a/src/promptflow-core/pyproject.toml b/src/promptflow-core/pyproject.toml index e017a2b05f0..63b4729ce8f 100644 --- a/src/promptflow-core/pyproject.toml +++ b/src/promptflow-core/pyproject.toml @@ -44,7 +44,7 @@ promptflow-tracing = ">=1.0.0,<2.0.0" "ruamel.yaml" = ">=0.17.10,<1.0.0" # used to generate connection templates with preserved comments docutils = "*" # used to generate description for tools jsonschema = ">=4.0.0,<5.0.0" # used to validate tool -filetype = ">=1.2.0" # used to detect the mime type for mulit-media input +filetype = ">=1.2.0" # used to detect the mime type for multi-media input flask = ">=2.2.3,<4.0.0" # Serving endpoint requirements fastapi = {extras = ["executor-service"], version = ">=0.109.0,<1.0.0"} From d7ead83113f88a3a0df01661acc89c0ef1a674f6 Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Wed, 27 Mar 2024 13:36:39 +0800 Subject: [PATCH 08/16] Refine is connection value logic Signed-off-by: Brynn Yin --- src/promptflow-core/promptflow/_constants.py | 3 +++ .../promptflow/_sdk/entities/_connection.py | 11 +++++++++++ src/promptflow/promptflow/contracts/tool.py | 7 ++++++- 3 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/promptflow-core/promptflow/_constants.py b/src/promptflow-core/promptflow/_constants.py index e420d0bd3f7..090e0fb6844 100644 --- a/src/promptflow-core/promptflow/_constants.py +++ b/src/promptflow-core/promptflow/_constants.py @@ -253,3 +253,6 @@ def is_custom_key(key): class ConnectionProviderConfig: LOCAL = "local" AZUREML = "azureml" + + +CONNECTION_DATA_CLASS_KEY = "DATA_CLASS" diff --git a/src/promptflow/promptflow/_sdk/entities/_connection.py b/src/promptflow/promptflow/_sdk/entities/_connection.py index 7b3cfda28d5..79c434e8598 100644 --- a/src/promptflow/promptflow/_sdk/entities/_connection.py +++ b/src/promptflow/promptflow/_sdk/entities/_connection.py @@ -265,6 +265,7 @@ def _from_mt_rest_object(cls, mt_rest_obj): class AzureOpenAIConnection(_CoreAzureOpenAIConnection, _StrongTypeConnection): __doc__ = _CoreAzureOpenAIConnection.__doc__ + DATA_CLASS = _CoreAzureOpenAIConnection @classmethod def _get_schema_cls(cls): @@ -273,6 +274,7 @@ def _get_schema_cls(cls): class OpenAIConnection(_CoreOpenAIConnection, _StrongTypeConnection): __doc__ = _CoreOpenAIConnection.__doc__ + DATA_CLASS = _CoreOpenAIConnection @classmethod def _get_schema_cls(cls): @@ -281,6 +283,7 @@ def _get_schema_cls(cls): class ServerlessConnection(_CoreServerlessConnection, _StrongTypeConnection): __doc__ = _CoreServerlessConnection.__doc__ + DATA_CLASS = _CoreServerlessConnection @classmethod def _get_schema_cls(cls): @@ -289,6 +292,7 @@ def _get_schema_cls(cls): class SerpConnection(_CoreSerpConnection, _StrongTypeConnection): __doc__ = _CoreSerpConnection.__doc__ + DATA_CLASS = _CoreSerpConnection @classmethod def _get_schema_cls(cls): @@ -297,6 +301,7 @@ def _get_schema_cls(cls): class QdrantConnection(_CoreQdrantConnection, _StrongTypeConnection): __doc__ = _CoreQdrantConnection.__doc__ + DATA_CLASS = _CoreQdrantConnection @classmethod def _get_schema_cls(cls): @@ -305,6 +310,7 @@ def _get_schema_cls(cls): class WeaviateConnection(_CoreWeaviateConnection, _StrongTypeConnection): __doc__ = _CoreWeaviateConnection.__doc__ + DATA_CLASS = _CoreWeaviateConnection @classmethod def _get_schema_cls(cls): @@ -313,6 +319,7 @@ def _get_schema_cls(cls): class CognitiveSearchConnection(_CoreCognitiveSearchConnection, _StrongTypeConnection): __doc__ = _CoreCognitiveSearchConnection.__doc__ + DATA_CLASS = _CoreCognitiveSearchConnection @classmethod def _get_schema_cls(cls): @@ -321,6 +328,7 @@ def _get_schema_cls(cls): class AzureContentSafetyConnection(_CoreAzureContentSafetyConnection, _StrongTypeConnection): __doc__ = _CoreAzureContentSafetyConnection.__doc__ + DATA_CLASS = _CoreAzureContentSafetyConnection @classmethod def _get_schema_cls(cls): @@ -329,6 +337,7 @@ def _get_schema_cls(cls): class FormRecognizerConnection(_CoreFormRecognizerConnection, AzureContentSafetyConnection): __doc__ = _CoreFormRecognizerConnection.__doc__ + DATA_CLASS = _CoreFormRecognizerConnection @classmethod def _get_schema_cls(cls): @@ -337,6 +346,7 @@ def _get_schema_cls(cls): class CustomStrongTypeConnection(_CoreCustomStrongTypeConnection, _Connection): __doc__ = _CoreCustomStrongTypeConnection.__doc__ + DATA_CLASS = _CoreCustomStrongTypeConnection def _to_orm_object(self) -> ORMConnection: custom_connection = self._convert_to_custom() @@ -417,6 +427,7 @@ def _load_from_dict(cls, data: Dict, context: Dict, additional_message: str = No class CustomConnection(_CoreCustomConnection, _Connection): __doc__ = _CoreCustomConnection.__doc__ + DATA_CLASS = _CoreCustomConnection @classmethod def _get_schema_cls(cls): diff --git a/src/promptflow/promptflow/contracts/tool.py b/src/promptflow/promptflow/contracts/tool.py index deeeb350069..42ad59fdc02 100644 --- a/src/promptflow/promptflow/contracts/tool.py +++ b/src/promptflow/promptflow/contracts/tool.py @@ -8,7 +8,7 @@ from enum import Enum from typing import Any, Dict, List, Optional, Type, TypeVar -from promptflow._constants import CONNECTION_NAME_PROPERTY +from promptflow._constants import CONNECTION_NAME_PROPERTY, CONNECTION_DATA_CLASS_KEY from .multimedia import Image from .types import AssistantDefinition, FilePath, PromptTemplate, Secret @@ -190,6 +190,11 @@ def is_connection_value(val: Any) -> bool: from promptflow._core.tools_manager import connections val = type(val) if not isinstance(val, type) else val + if hasattr(val, CONNECTION_DATA_CLASS_KEY): + # Get the data class for sdk connection object + data_class = getattr(val, CONNECTION_DATA_CLASS_KEY) + logger.debug(f"val {val} has DATA_CLASS key, get the data plane class {data_class}.") + val = data_class return val in connections.values() or ConnectionType.is_custom_strong_type(val) @staticmethod From 26b3228baca6f55a21d49ff6f3b43c69ad0a6144 Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Wed, 27 Mar 2024 13:53:00 +0800 Subject: [PATCH 09/16] Fix tool resolver test Signed-off-by: Brynn Yin --- .../tests/executor/unittests/executor/test_tool_resolver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/promptflow/tests/executor/unittests/executor/test_tool_resolver.py b/src/promptflow/tests/executor/unittests/executor/test_tool_resolver.py index c66f8404d56..f0d3f38952b 100644 --- a/src/promptflow/tests/executor/unittests/executor/test_tool_resolver.py +++ b/src/promptflow/tests/executor/unittests/executor/test_tool_resolver.py @@ -10,7 +10,7 @@ from promptflow._core._errors import InvalidSource from promptflow._core.tools_manager import ToolLoader from promptflow._internal import tool -from promptflow._sdk.entities import CustomConnection, CustomStrongTypeConnection +from promptflow.connections import CustomConnection, CustomStrongTypeConnection from promptflow.connections import AzureOpenAIConnection from promptflow.contracts.flow import InputAssignment, InputValueType, Node, ToolSource, ToolSourceType from promptflow.contracts.tool import AssistantDefinition, InputDefinition, Secret, Tool, ToolType, ValueType From 0df3549eb78200d2993bb7a4a3a3ac033c7cb102 Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Wed, 27 Mar 2024 14:18:52 +0800 Subject: [PATCH 10/16] Fix perf flow Signed-off-by: Brynn Yin --- .github/workflows/sdk-cli-perf-monitor-test.yml | 11 ++++++++--- src/promptflow/promptflow/_sdk/entities/_flow/base.py | 2 +- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sdk-cli-perf-monitor-test.yml b/.github/workflows/sdk-cli-perf-monitor-test.yml index 0718ec994d5..6c953cde88a 100644 --- a/.github/workflows/sdk-cli-perf-monitor-test.yml +++ b/.github/workflows/sdk-cli-perf-monitor-test.yml @@ -55,6 +55,13 @@ jobs: with: pythonVersion: ${{ steps.display_env.outputs.pyVersion }} + - name: Install promptflow-core + shell: pwsh + run: | + pip uninstall -y promptflow-core + pip install ${{ github.workspace }}/src/promptflow-core + pip freeze + - name: Build wheel uses: "./.github/actions/step_sdk_setup" with: @@ -67,13 +74,11 @@ jobs: pip install -e . working-directory: ${{ env.RECORD_DIRECTORY }} - - name: Install dependency + - name: Install promptflow-tracing shell: pwsh run: | pip uninstall -y promptflow-tracing pip install ${{ github.workspace }}/src/promptflow-tracing - pip uninstall -y promptflow-core - pip install ${{ github.workspace }}/src/promptflow-core pip freeze - name: Generate (mock) connections.json diff --git a/src/promptflow/promptflow/_sdk/entities/_flow/base.py b/src/promptflow/promptflow/_sdk/entities/_flow/base.py index 75e1aa12259..be3a5b08745 100644 --- a/src/promptflow/promptflow/_sdk/entities/_flow/base.py +++ b/src/promptflow/promptflow/_sdk/entities/_flow/base.py @@ -58,7 +58,7 @@ def _resolve_connections(self): def _get_connection_obj_name(cls, connection): # create a unique connection name for connection obj # will generate same name if connection has same content - connection_dict = connection._to_dict() + connection_dict = dict(connection) connection_name = f"connection_{hash(json.dumps(connection_dict, sort_keys=True))}" return connection_name From fc2a746f540c0cb2e061671fac5151ec2a8072c9 Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Wed, 27 Mar 2024 14:38:11 +0800 Subject: [PATCH 11/16] Fix test Signed-off-by: Brynn Yin --- .../workflows/sdk-cli-perf-monitor-test.yml | 19 +++++-------------- src/promptflow-core/promptflow/_constants.py | 1 + .../promptflow/core/_connection.py | 15 ++++++++++++++- src/promptflow/promptflow/_sdk/_constants.py | 4 ++-- .../promptflow/_sdk/entities/_connection.py | 14 -------------- 5 files changed, 22 insertions(+), 31 deletions(-) diff --git a/.github/workflows/sdk-cli-perf-monitor-test.yml b/.github/workflows/sdk-cli-perf-monitor-test.yml index 6c953cde88a..be3a4f8f883 100644 --- a/.github/workflows/sdk-cli-perf-monitor-test.yml +++ b/.github/workflows/sdk-cli-perf-monitor-test.yml @@ -55,32 +55,23 @@ jobs: with: pythonVersion: ${{ steps.display_env.outputs.pyVersion }} - - name: Install promptflow-core + - name: Install promptflow shell: pwsh run: | + pip uninstall -y promptflow-tracing + pip install ${{ github.workspace }}/src/promptflow-tracing pip uninstall -y promptflow-core pip install ${{ github.workspace }}/src/promptflow-core + pip uninstall -y promptflow + pip install ${{ github.workspace }}/src/promptflow[azure,executable,azureml-serving,executor-service] pip freeze - - name: Build wheel - uses: "./.github/actions/step_sdk_setup" - with: - setupType: ${{ env.packageSetupType }} - scriptPath: ${{ env.testWorkingDirectory }} - - name: install recording run: pip install vcrpy pip install -e . working-directory: ${{ env.RECORD_DIRECTORY }} - - name: Install promptflow-tracing - shell: pwsh - run: | - pip uninstall -y promptflow-tracing - pip install ${{ github.workspace }}/src/promptflow-tracing - pip freeze - - name: Generate (mock) connections.json shell: pwsh working-directory: ${{ env.testWorkingDirectory }} diff --git a/src/promptflow-core/promptflow/_constants.py b/src/promptflow-core/promptflow/_constants.py index 090e0fb6844..edc3dcbf9a7 100644 --- a/src/promptflow-core/promptflow/_constants.py +++ b/src/promptflow-core/promptflow/_constants.py @@ -7,6 +7,7 @@ CONNECTION_NAME_PROPERTY = "__connection_name" CONNECTION_SECRET_KEYS = "__secret_keys" CONNECTION_SCRUBBED_VALUE = "******" +CONNECTION_SCRUBBED_VALUE_NO_CHANGE = "" PROMPTFLOW_CONNECTIONS = "PROMPTFLOW_CONNECTIONS" PROMPTFLOW_SECRETS_FILE = "PROMPTFLOW_SECRETS_FILE" PF_NO_INTERACTIVE_LOGIN = "PF_NO_INTERACTIVE_LOGIN" diff --git a/src/promptflow-core/promptflow/core/_connection.py b/src/promptflow-core/promptflow/core/_connection.py index 5717473144b..3d980e6f21b 100644 --- a/src/promptflow-core/promptflow/core/_connection.py +++ b/src/promptflow-core/promptflow/core/_connection.py @@ -6,7 +6,7 @@ import types from typing import Dict, List -from promptflow._constants import CONNECTION_SCRUBBED_VALUE as SCRUBBED_VALUE +from promptflow._constants import CONNECTION_SCRUBBED_VALUE as SCRUBBED_VALUE, CONNECTION_SCRUBBED_VALUE_NO_CHANGE from promptflow._constants import ConnectionAuthMode, ConnectionType, CustomStrongTypeConnectionConfigs from promptflow._core.token_provider import AzureTokenProvider from promptflow._utils.logger_utils import LoggerFactory @@ -109,6 +109,19 @@ def _from_execution_connection_dict(cls, name, data) -> "_Connection": return type_cls(name=name, configs=configs, secrets=secrets) return type_cls(name=name, **value_dict) + @classmethod + def _is_scrubbed_value(cls, value): + """For scrubbed value, cli will get original for update, and prompt user to input for create.""" + if value is None or not value: + return True + if all([v == "*" for v in value]): + return True + return value == CONNECTION_SCRUBBED_VALUE_NO_CHANGE + + def _get_scrubbed_secrets(self): + """Return the scrubbed secrets of connection.""" + return {key: val for key, val in self.secrets.items() if self._is_scrubbed_value(val)} + class _StrongTypeConnection(_Connection): @property diff --git a/src/promptflow/promptflow/_sdk/_constants.py b/src/promptflow/promptflow/_sdk/_constants.py index 864216f0abc..c304b7551a0 100644 --- a/src/promptflow/promptflow/_sdk/_constants.py +++ b/src/promptflow/promptflow/_sdk/_constants.py @@ -12,7 +12,7 @@ PROMPT_FLOW_DIR_NAME, ConnectionAuthMode, ConnectionType, - CustomStrongTypeConnectionConfigs, + CustomStrongTypeConnectionConfigs, CONNECTION_SCRUBBED_VALUE_NO_CHANGE, ) LOGGER_NAME = "promptflow" @@ -113,7 +113,7 @@ def _prepare_home_dir() -> Path: REFRESH_CONNECTIONS_DIR_LOCK_PATH = (HOME_PROMPT_FLOW_DIR / "refresh_connections_dir.lock").resolve() # Note: Use this only for show. Reading input should regard all '*' string as scrubbed, no matter the length. SCRUBBED_VALUE = CONNECTION_SCRUBBED_VALUE -SCRUBBED_VALUE_NO_CHANGE = "" +SCRUBBED_VALUE_NO_CHANGE = CONNECTION_SCRUBBED_VALUE_NO_CHANGE SCRUBBED_VALUE_USER_INPUT = "" WORKSPACE_LINKED_DATASTORE_NAME = "workspaceblobstore" LINE_NUMBER = "line_number" diff --git a/src/promptflow/promptflow/_sdk/entities/_connection.py b/src/promptflow/promptflow/_sdk/entities/_connection.py index 79c434e8598..7fe3316ece6 100644 --- a/src/promptflow/promptflow/_sdk/entities/_connection.py +++ b/src/promptflow/promptflow/_sdk/entities/_connection.py @@ -17,7 +17,6 @@ SCHEMA_KEYS_CONTEXT_CONFIG_KEY, SCHEMA_KEYS_CONTEXT_SECRET_KEY, SCRUBBED_VALUE, - SCRUBBED_VALUE_NO_CHANGE, SCRUBBED_VALUE_USER_INPUT, ConfigValueType, ) @@ -79,15 +78,6 @@ def _casting_type(cls, typ): return type_dict.get(typ) return snake_to_camel(typ) - @classmethod - def _is_scrubbed_value(cls, value): - """For scrubbed value, cli will get original for update, and prompt user to input for create.""" - if value is None or not value: - return True - if all([v == "*" for v in value]): - return True - return value == SCRUBBED_VALUE_NO_CHANGE - @classmethod def _is_user_input_value(cls, value): """The value will prompt user to input in cli for both create and update.""" @@ -211,10 +201,6 @@ def _load( ) return connection - def _get_scrubbed_secrets(self): - """Return the scrubbed secrets of connection.""" - return {key: val for key, val in self.secrets.items() if self._is_scrubbed_value(val)} - class _StrongTypeConnection(_CoreStrongTypeConnection, _Connection): def _to_orm_object(self): From 7ae1d381b8136635453944ca25ff8b7bffb7f994 Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Wed, 27 Mar 2024 14:46:40 +0800 Subject: [PATCH 12/16] Fix per test Signed-off-by: Brynn Yin --- .github/workflows/sdk-cli-perf-monitor-test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sdk-cli-perf-monitor-test.yml b/.github/workflows/sdk-cli-perf-monitor-test.yml index be3a4f8f883..8ffdb8eb768 100644 --- a/.github/workflows/sdk-cli-perf-monitor-test.yml +++ b/.github/workflows/sdk-cli-perf-monitor-test.yml @@ -58,6 +58,8 @@ jobs: - name: Install promptflow shell: pwsh run: | + Set-PSDebug -Trace 1 + pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt pip uninstall -y promptflow-tracing pip install ${{ github.workspace }}/src/promptflow-tracing pip uninstall -y promptflow-core From d77c3de5b61df6c2b14aa77f4a9644841afb0cea Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Wed, 27 Mar 2024 14:47:17 +0800 Subject: [PATCH 13/16] Update workflow Signed-off-by: Brynn Yin --- .github/workflows/sdk-cli-perf-monitor-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sdk-cli-perf-monitor-test.yml b/.github/workflows/sdk-cli-perf-monitor-test.yml index 8ffdb8eb768..8cd41d4a7eb 100644 --- a/.github/workflows/sdk-cli-perf-monitor-test.yml +++ b/.github/workflows/sdk-cli-perf-monitor-test.yml @@ -61,10 +61,10 @@ jobs: Set-PSDebug -Trace 1 pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt pip uninstall -y promptflow-tracing - pip install ${{ github.workspace }}/src/promptflow-tracing pip uninstall -y promptflow-core - pip install ${{ github.workspace }}/src/promptflow-core pip uninstall -y promptflow + pip install ${{ github.workspace }}/src/promptflow-tracing + pip install ${{ github.workspace }}/src/promptflow-core pip install ${{ github.workspace }}/src/promptflow[azure,executable,azureml-serving,executor-service] pip freeze From 538a9598aeb7df24400d9bc1c07dc85f195fd761 Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Wed, 27 Mar 2024 14:56:29 +0800 Subject: [PATCH 14/16] Update workflow Signed-off-by: Brynn Yin --- .github/workflows/sdk-cli-perf-monitor-test.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/sdk-cli-perf-monitor-test.yml b/.github/workflows/sdk-cli-perf-monitor-test.yml index 8cd41d4a7eb..7ed9758a97f 100644 --- a/.github/workflows/sdk-cli-perf-monitor-test.yml +++ b/.github/workflows/sdk-cli-perf-monitor-test.yml @@ -55,17 +55,22 @@ jobs: with: pythonVersion: ${{ steps.display_env.outputs.pyVersion }} - - name: Install promptflow + - name: Build wheel + uses: "./.github/actions/step_sdk_setup" + with: + setupType: promptflow_with_extra + scriptPath: ${{ env.testWorkingDirectory }} + + - name: Install wheel shell: pwsh + working-directory: artifacts run: | Set-PSDebug -Trace 1 pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt - pip uninstall -y promptflow-tracing - pip uninstall -y promptflow-core - pip uninstall -y promptflow pip install ${{ github.workspace }}/src/promptflow-tracing pip install ${{ github.workspace }}/src/promptflow-core - pip install ${{ github.workspace }}/src/promptflow[azure,executable,azureml-serving,executor-service] + gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}} + gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} pip freeze - name: install recording From dcb807ea2e8247863ef6f3ce695b758f11875e7c Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Wed, 27 Mar 2024 14:57:26 +0800 Subject: [PATCH 15/16] Update workflow Signed-off-by: Brynn Yin --- .github/workflows/sdk-cli-perf-monitor-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sdk-cli-perf-monitor-test.yml b/.github/workflows/sdk-cli-perf-monitor-test.yml index 7ed9758a97f..c0206907354 100644 --- a/.github/workflows/sdk-cli-perf-monitor-test.yml +++ b/.github/workflows/sdk-cli-perf-monitor-test.yml @@ -69,7 +69,7 @@ jobs: pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt pip install ${{ github.workspace }}/src/promptflow-tracing pip install ${{ github.workspace }}/src/promptflow-core - gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}} + gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure,executable,azureml-serving]"}} gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}} pip freeze From 67c6fe3c19cd3e10b17a060b3808233498011775 Mon Sep 17 00:00:00 2001 From: Brynn Yin Date: Wed, 27 Mar 2024 15:02:01 +0800 Subject: [PATCH 16/16] Update workflow Signed-off-by: Brynn Yin --- .github/workflows/sdk-cli-perf-monitor-test.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sdk-cli-perf-monitor-test.yml b/.github/workflows/sdk-cli-perf-monitor-test.yml index c0206907354..46ee641326a 100644 --- a/.github/workflows/sdk-cli-perf-monitor-test.yml +++ b/.github/workflows/sdk-cli-perf-monitor-test.yml @@ -60,7 +60,19 @@ jobs: with: setupType: promptflow_with_extra scriptPath: ${{ env.testWorkingDirectory }} - + - name: Upload Wheel + if: always() + uses: actions/upload-artifact@v3 + with: + name: wheel + path: | + ${{ github.workspace }}/src/promptflow/dist/*.whl + ${{ github.workspace }}/src/promptflow-tools/dist/*.whl + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + name: wheel + path: artifacts - name: Install wheel shell: pwsh working-directory: artifacts