Bring parity between eval and sim for azure_ai_project

src/promptflow-evals/promptflow/evals/synthetic/README.md

@@ -25,8 +25,7 @@ import asyncio
 azure_ai_project = {
     "subscription_id": <subscription_id>,
     "resource_group_name": <resource_group_name>,
-    "project_name": <project_name>,
-    "credential": DefaultAzureCredential(),
+    "project_name": <project_name>
 }
 
 async def callback(
@@ -90,7 +89,7 @@ Make sure you change the snippets below to remove the `asyncio.run` wrapper and
 ### Adversarial QA:
 ```python
 scenario = AdversarialScenario.ADVERSARIAL_QA
-simulator = AdversarialSimulator(azure_ai_project=azure_ai_project)
+simulator = AdversarialSimulator(azure_ai_project=azure_ai_project, credential=DefaultAzureCredential())
 
 outputs = asyncio.run(
     simulator(

src/promptflow-evals/promptflow/evals/synthetic/adversarial_simulator.py

@@ -8,6 +8,7 @@
 import random
 from typing import Any, Callable, Dict, List
 
+from azure.identity import DefaultAzureCredential
 from tqdm import tqdm
 
 from promptflow._sdk._telemetry import ActivityType, monitor_operation
@@ -56,33 +57,31 @@ class AdversarialSimulator:
     Initializes the adversarial simulator with a project scope.
 
     :param azure_ai_project: Dictionary defining the scope of the project. It must include the following keys:
-
-        * "subscription_id": Azure subscription ID.
-        * "resource_group_name": Name of the Azure resource group.
-        * "project_name": Name of the Azure Machine Learning workspace.
-        * "credential": Azure credentials object for authentication.
+        - "subscription_id": Azure subscription ID.
+        - "resource_group_name": Name of the Azure resource group.
+        - "project_name": Name of the Azure Machine Learning workspace.
+    :param credential: The credential for connecting to Azure AI project.
+    :type credential: TokenCredential
     :type azure_ai_project: Dict[str, Any]
     """
 
-    def __init__(self, *, azure_ai_project: Dict[str, Any]):
+    def __init__(self, *, azure_ai_project: Dict[str, Any], credential=None):
         """Constructor."""
-        # check if azure_ai_project has the keys: subscription_id, resource_group_name, project_name, credential
-        if not all(
-            key in azure_ai_project for key in ["subscription_id", "resource_group_name", "project_name", "credential"]
-        ):
-            raise ValueError(
-                "azure_ai_project must contain keys: subscription_id, resource_group_name, project_name, credential"
-            )
+        # check if azure_ai_project has the keys: subscription_id, resource_group_name and project_name
+        if not all(key in azure_ai_project for key in ["subscription_id", "resource_group_name", "project_name"]):
+            raise ValueError("azure_ai_project must contain keys: subscription_id, resource_group_name, project_name")
         # check the value of the keys in azure_ai_project is not none
-        if not all(
-            azure_ai_project[key] for key in ["subscription_id", "resource_group_name", "project_name", "credential"]
-        ):
-            raise ValueError("subscription_id, resource_group_name, project_name, and credential must not be None")
+        if not all(azure_ai_project[key] for key in ["subscription_id", "resource_group_name", "project_name"]):
+            raise ValueError("subscription_id, resource_group_name and project_name must not be None")
+        if "credential" not in azure_ai_project and not credential:
+            credential = DefaultAzureCredential()
+        elif "credential" in azure_ai_project:
+            credential = azure_ai_project["credential"]
         self.azure_ai_project = azure_ai_project
         self.token_manager = ManagedIdentityAPITokenManager(
             token_scope=TokenScope.DEFAULT_AZURE_MANAGEMENT,
             logger=logging.getLogger("AdversarialSimulator"),
-            credential=self.azure_ai_project["credential"],
+            credential=credential,
         )
         self.rai_client = RAIClient(azure_ai_project=azure_ai_project, token_manager=self.token_manager)
         self.adversarial_template_handler = AdversarialTemplateHandler(

src/promptflow-evals/tests/evals/e2etests/test_adv_simulator.py

@@ -139,7 +139,6 @@ async def callback(
             )
         )
         assert len(outputs) == 1
-        print(outputs)
         assert len(outputs[0]["messages"]) == 4
 
     @pytest.mark.usefixtures("vcr_recording")
@@ -182,8 +181,6 @@ async def callback(
                 concurrent_async_task=1,
             )
         )
-        print(outputs.to_json_lines())
-        print("*****************************")
         assert len(outputs) == 1
 
     @pytest.mark.usefixtures("vcr_recording")
@@ -227,6 +224,46 @@ async def callback(
                 jailbreak=True,
             )
         )
-        print(outputs.to_json_lines())
-        print("*****************************")
+        assert len(outputs) == 1
+
+    @pytest.mark.usefixtures("vcr_recording")
+    def test_adv_rewrite_sim_responds_with_responses(self, azure_cred, project_scope):
+        os.environ.pop("RAI_SVC_URL", None)
+        from promptflow.evals.synthetic import AdversarialScenario, AdversarialSimulator
+
+        azure_ai_project = {
+            "subscription_id": project_scope["subscription_id"],
+            "resource_group_name": project_scope["resource_group_name"],
+            "project_name": project_scope["project_name"],
+        }
+
+        async def callback(
+            messages: List[Dict], stream: bool = False, session_state: Any = None, context: Dict[str, Any] = None
+        ) -> dict:
+            question = messages["messages"][0]["content"]
+
+            formatted_response = {"content": question, "role": "assistant"}
+            messages["messages"].append(formatted_response)
+            return {
+                "messages": messages["messages"],
+                "stream": stream,
+                "session_state": session_state,
+                "context": context,
+            }
+
+        simulator = AdversarialSimulator(azure_ai_project=azure_ai_project, credential=azure_cred)
+
+        outputs = asyncio.run(
+            simulator(
+                scenario=AdversarialScenario.ADVERSARIAL_REWRITE,
+                max_conversation_turns=1,
+                max_simulation_results=1,
+                target=callback,
+                api_call_retry_limit=3,
+                api_call_retry_sleep_sec=1,
+                api_call_delay_sec=30,
+                concurrent_async_task=1,
+                jailbreak=True,
+            )
+        )
         assert len(outputs) == 1

src/promptflow-evals/tests/evals/unittests/test_simulator.py

@@ -81,3 +81,37 @@ async def callback(x):
                     scenario="unknown-scenario", max_conversation_turns=1, max_simulation_results=3, target=callback
                 )
             )
+
+    @patch("promptflow.evals.synthetic._model_tools._rai_client.RAIClient._get_service_discovery_url")
+    @patch("promptflow.evals.synthetic._model_tools.AdversarialTemplateHandler._get_content_harm_template_collections")
+    @patch("promptflow.evals.synthetic.adversarial_simulator.AdversarialSimulator._simulate_async")
+    @patch("promptflow.evals.synthetic.adversarial_simulator.AdversarialSimulator._ensure_service_dependencies")
+    def test_initialization_parity_with_evals(
+        self,
+        mock_ensure_service_dependencies,
+        mock_get_content_harm_template_collections,
+        mock_simulate_async,
+        mock_get_service_discovery_url,
+    ):
+        mock_get_service_discovery_url.return_value = "http://some.url/discovery/"
+        mock_simulate_async.return_value = MagicMock()
+        mock_get_content_harm_template_collections.return_value = ["t1", "t2", "t3", "t4", "t5", "t6", "t7"]
+        mock_ensure_service_dependencies.return_value = True
+        azure_ai_project = {
+            "subscription_id": "test_subscription",
+            "resource_group_name": "test_resource_group",
+            "project_name": "test_workspace",
+        }
+        available_scenarios = [
+            AdversarialScenario.ADVERSARIAL_CONVERSATION,
+            AdversarialScenario.ADVERSARIAL_QA,
+            AdversarialScenario.ADVERSARIAL_SUMMARIZATION,
+            AdversarialScenario.ADVERSARIAL_SEARCH,
+            AdversarialScenario.ADVERSARIAL_REWRITE,
+            AdversarialScenario.ADVERSARIAL_CONTENT_GEN_UNGROUNDED,
+            AdversarialScenario.ADVERSARIAL_CONTENT_GEN_GROUNDED,
+        ]
+        for scenario in available_scenarios:
+            simulator = AdversarialSimulator(azure_ai_project=azure_ai_project, credential="test_credential")
+            assert callable(simulator)
+            simulator(scenario=scenario, max_conversation_turns=1, max_simulation_results=3, target=async_callback)