feat: Support JSON for consuming orchestration (#346)

* execute orchestration from JSON input * union type in contructor * fix: Changes from lint * update README.md * fix: Changes from lint * changes and added tests * fix: Changes from lint * rename * updated docs * fix: Changes from lint * refactored tests and doc changes * refactor: change model and endpoint * fix: Changes from lint * fix: construct completion post request tests * fix: testcd '/Users/I563080/Dev/ai-sdk-js' * chore: minor formatting --------- Co-authored-by: cloud-sdk-js <[email protected]> Co-authored-by: Junjie Tang <[email protected]> Co-authored-by: Zhongpin Wang <[email protected]>
SAP · Dec 20, 2024 · 17a1eea · 17a1eea
1 parent 4e82eb5
commit 17a1eea
Show file tree

Hide file tree

Showing 11 changed files with 261 additions and 13 deletions.
diff --git a/.changeset/many-badgers-travel.md b/.changeset/many-badgers-travel.md
@@ -0,0 +1,5 @@
+---
+'@sap-ai-sdk/orchestration': minor
+---
+
+[New Functionality] Add support for using a JSON configuration obtained from AI Launchpad to consume orchestration service.
diff --git a/packages/orchestration/README.md b/packages/orchestration/README.md
@@ -15,6 +15,7 @@ This package incorporates generative AI orchestration capabilities into your AI
   - [Content Filtering](#content-filtering)
   - [Data Masking](#data-masking)
   - [Grounding](#grounding)
+  - [Using a JSON Configuration from AI Launchpad](#using-a-json-configuration-from-ai-launchpad)
   - [Using Resource Groups](#using-resource-groups)
   - [Custom Request Configuration](#custom-request-configuration)
   - [Custom Destination](#custom-destination)
@@ -326,6 +327,23 @@ const response = await orchestrationClient.chatCompletion({
 return response.getContent();
 ```
 
+### Using a JSON Configuration from AI Launchpad
+
+If you already have an orchestration workflow created in AI Launchpad, you can either download the configuration as a JSON file or copy the JSON string directly to use it with the orchestration client.
+
+```ts
+const jsonConfig = await fs.promises.readFile(
+  'path/to/orchestration-config.json',
+  'utf-8'
+);
+// Alternatively, you can provide the JSON configuration as a plain string in the code directly.
+// const jsonConfig = 'YOUR_JSON_CONFIG'
+
+const response = await new OrchestrationClient(jsonConfig).chatCompletion();
+
+return response;
+```
+
 ### Using Resource Groups
 
 The resource group can be used as an additional parameter to pick the right orchestration deployment.

diff --git a/packages/orchestration/src/orchestration-client.test.ts b/packages/orchestration/src/orchestration-client.test.ts
@@ -6,6 +6,7 @@ import {
   parseMockResponse
 } from '../../../test-util/mock-http.js';
 import {
+  constructCompletionPostRequestFromJson,
   constructCompletionPostRequest,
   OrchestrationClient
 } from './orchestration-client.js';
@@ -64,6 +65,46 @@ describe('orchestration service client', () => {
     expect(response.getTokenUsage().completion_tokens).toEqual(9);
   });
 
+  it('calls chatCompletion with valid JSON configuration', async () => {
+    const jsonConfig = `{
+      "module_configurations": {
+        "llm_module_config": {
+          "model_name": "gpt-35-turbo-16k",
+          "model_params": {
+            "max_tokens": 50,
+            "temperature": 0.1
+          }
+        },
+        "templating_module_config": {
+          "template": [{ "role": "user", "content": "What is the capital of France?" }]
+        }
+      }
+    }`;
+
+    const mockResponse = await parseMockResponse<CompletionPostResponse>(
+      'orchestration',
+      'orchestration-chat-completion-success-response.json'
+    );
+
+    mockInference(
+      {
+        data: constructCompletionPostRequestFromJson(jsonConfig)
+      },
+      {
+        data: mockResponse,
+        status: 200
+      },
+      {
+        url: 'inference/deployments/1234/completion'
+      }
+    );
+
+    const response = await new OrchestrationClient(jsonConfig).chatCompletion();
+
+    expect(response).toBeInstanceOf(OrchestrationResponse);
+    expect(response.data).toEqual(mockResponse);
+  });
+
   it('calls chatCompletion with filter configuration supplied using convenience function', async () => {
     const config: OrchestrationModuleConfig = {
       llm: {

diff --git a/packages/orchestration/src/orchestration-client.ts b/packages/orchestration/src/orchestration-client.ts
@@ -16,12 +16,12 @@ import type { HttpDestinationOrFetchOptions } from '@sap-cloud-sdk/connectivity'
 export class OrchestrationClient {
   /**
    * Creates an instance of the orchestration client.
-   * @param config - Orchestration module configuration.
+   * @param config - Orchestration module configuration. This can either be an `OrchestrationModuleConfig` object or a JSON string obtained from AI Launchpad.
    * @param deploymentConfig - Deployment configuration.
    * @param destination - The destination to use for the request.
    */
   constructor(
-    private config: OrchestrationModuleConfig,
+    private config: OrchestrationModuleConfig | string,
     private deploymentConfig?: ResourceGroupConfig,
     private destination?: HttpDestinationOrFetchOptions
   ) {}
@@ -36,7 +36,11 @@ export class OrchestrationClient {
     prompt?: Prompt,
     requestConfig?: CustomRequestConfig
   ): Promise<OrchestrationResponse> {
-    const body = constructCompletionPostRequest(this.config, prompt);
+    const body =
+      typeof this.config === 'string'
+        ? constructCompletionPostRequestFromJson(this.config, prompt)
+        : constructCompletionPostRequest(this.config, prompt);
+
     const deploymentId = await resolveDeploymentId({
       scenarioId: 'orchestration',
       resourceGroup: this.deploymentConfig?.resourceGroup
@@ -56,6 +60,24 @@ export class OrchestrationClient {
   }
 }
 
+/**
+ * @internal
+ */
+export function constructCompletionPostRequestFromJson(
+  config: string,
+  prompt?: Prompt
+): Record<string, any> {
+  try {
+    return {
+      messages_history: prompt?.messagesHistory || [],
+      input_params: prompt?.inputParams || {},
+      orchestration_config: JSON.parse(config)
+    };
+  } catch (error) {
+    throw new Error(`Could not parse JSON: ${error}`);
+  }
+}
+
 /**
  * @internal
  */

diff --git a/packages/orchestration/src/orchestration-completion-post-request-from-json.test.ts b/packages/orchestration/src/orchestration-completion-post-request-from-json.test.ts
@@ -0,0 +1,97 @@
+import { constructCompletionPostRequestFromJson } from './orchestration-client.js';
+
+describe('construct completion post request from JSON', () => {
+  it('should throw an error when invalid JSON is provided', () => {
+    const invalidJsonConfig = '{ "module_configurations": {}, ';
+
+    expect(() =>
+      constructCompletionPostRequestFromJson(invalidJsonConfig)
+    ).toThrow('Could not parse JSON');
+  });
+
+  it('should construct completion post request from JSON', () => {
+    const jsonConfig = `{
+      "module_configurations": {
+          "llm_module_config": {
+              "model_name": "gpt-35-turbo-16k",
+              "model_params": {
+                "max_tokens": 50,
+                "temperature": 0.1
+              }
+          },
+          "templating_module_config": {
+              "template": [{ "role": "user", "content": "Hello!" }]
+          }
+          }
+      }`;
+
+    const expectedCompletionPostRequestFromJson: Record<string, any> = {
+      input_params: {},
+      messages_history: [],
+      orchestration_config: JSON.parse(jsonConfig)
+    };
+
+    const completionPostRequestFromJson: Record<string, any> =
+      constructCompletionPostRequestFromJson(jsonConfig);
+
+    expect(expectedCompletionPostRequestFromJson).toEqual(
+      completionPostRequestFromJson
+    );
+  });
+
+  it('should construct completion post request from JSON with input params and message history', () => {
+    const jsonConfig = `{
+            "module_configurations": {
+              "llm_module_config": {
+                "model_name": "gpt-35-turbo-16k",
+                "model_params": {
+                  "max_tokens": 50,
+                  "temperature": 0.1
+                }
+              },
+              "templating_module_config": {
+                "template": [
+                    {
+                        "role": "user",
+                        "content": "Give me {{?number}} words that rhyme with my name."
+                    }
+                ]
+              }
+            }
+        }`;
+    const inputParams = { number: '3' };
+
+    const messagesHistory = [
+      {
+        role: 'system',
+        content:
+          'You are a helpful assistant who remembers all details the user shares with you.'
+      },
+      {
+        role: 'user',
+        content: 'Hi! Im Bob'
+      },
+      {
+        role: 'assistant',
+        content:
+          "Hi Bob, nice to meet you! I'm an AI assistant. I'll remember that your name is Bob as we continue our conversation."
+      }
+    ];
+
+    const expectedCompletionPostRequestFromJson: Record<string, any> = {
+      input_params: inputParams,
+      messages_history: messagesHistory,
+      orchestration_config: JSON.parse(jsonConfig)
+    };
+
+    const completionPostRequestFromJson: Record<string, any> =
+      constructCompletionPostRequestFromJson(jsonConfig, {
+        inputParams,
+        messagesHistory
+      });
+
+    expect(expectedCompletionPostRequestFromJson).toEqual(
+      completionPostRequestFromJson
+    );
+  });
+});
diff --git a/packages/orchestration/src/orchestration-completion-post-request.test.ts b/packages/orchestration/src/orchestration-completion-post-request.test.ts
@@ -3,7 +3,7 @@ import { buildAzureContentFilter } from './orchestration-filter-utility.js';
 import type { CompletionPostRequest } from './client/api/schema';
 import type { OrchestrationModuleConfig } from './orchestration-types.js';
 
-describe('constructCompletionPostRequest()', () => {
+describe('construct completion post request', () => {
   const defaultConfig: OrchestrationModuleConfig = {
     llm: {
       model_name: 'gpt-35-turbo-16k',
@@ -14,7 +14,7 @@ describe('constructCompletionPostRequest()', () => {
     }
   };
 
-  it('with model configuration and prompt template', async () => {
+  it('should construct completion post request with llm and templating module', async () => {
     const expectedCompletionPostRequest: CompletionPostRequest = {
       orchestration_config: {
         module_configurations: {
@@ -29,7 +29,7 @@ describe('constructCompletionPostRequest()', () => {
   });
 
   // TODO: Adapt the test after Cloud SDK fix for: https://github.com/SAP/cloud-sdk-backlog/issues/1234
-  it('with model configuration and empty template', async () => {
+  it('should construct completion post request with llm and empty templating module', async () => {
     const config: OrchestrationModuleConfig = {
       ...defaultConfig,
       templating: { template: [] }
@@ -47,7 +47,7 @@ describe('constructCompletionPostRequest()', () => {
     expect(completionPostRequest).toEqual(expectedCompletionPostRequest);
   });
 
-  it('with model configuration, prompt template and template params', async () => {
+  it('should construct completion post request with llm and templating module with input params', async () => {
     const config: OrchestrationModuleConfig = {
       ...defaultConfig,
       templating: {
@@ -74,7 +74,7 @@ describe('constructCompletionPostRequest()', () => {
     expect(completionPostRequest).toEqual(expectedCompletionPostRequest);
   });
 
-  it('with model configuration, prompt template and empty template params', async () => {
+  it('should construct completion post request with llm and templating module with empty input params', async () => {
     const config: OrchestrationModuleConfig = {
       ...defaultConfig,
       templating: {
@@ -101,7 +101,7 @@ describe('constructCompletionPostRequest()', () => {
     expect(completionPostRequest).toEqual(expectedCompletionPostRequest);
   });
 
-  it('with model name, empty model parameters and prompt template', async () => {
+  it('should construct completion post request with empty model params', async () => {
     const config: OrchestrationModuleConfig = {
       ...defaultConfig,
       llm: {
@@ -123,7 +123,7 @@ describe('constructCompletionPostRequest()', () => {
     expect(completionPostRequest).toEqual(expectedCompletionPostRequest);
   });
 
-  it('with model configuration, prompt template and message history', async () => {
+  it('should construct completion post request with message history', async () => {
     const config: OrchestrationModuleConfig = {
       ...defaultConfig,
       templating: {
@@ -160,7 +160,7 @@ describe('constructCompletionPostRequest()', () => {
     expect(completionPostRequest).toEqual(expectedCompletionPostRequest);
   });
 
-  it('with model configuration, prompt template and filter configuration', async () => {
+  it('should construct completion post request with filtering', async () => {
     const config: OrchestrationModuleConfig = {
       ...defaultConfig,
       filtering: {
@@ -182,7 +182,7 @@ describe('constructCompletionPostRequest()', () => {
   });
 
   // TODO: Adapt the test after Cloud SDK fix for: https://github.com/SAP/cloud-sdk-backlog/issues/1234
-  it('with model configuration, prompt template empty filter configuration', async () => {
+  it('should construct completion post request with empty filtering', async () => {
     const config: OrchestrationModuleConfig = {
       ...defaultConfig,
       filtering: {}

diff --git a/sample-code/src/index.ts b/sample-code/src/index.ts
@@ -12,6 +12,7 @@ export {
   orchestrationOutputFiltering,
   orchestrationRequestConfig,
   orchestrationCompletionMasking,
+  orchestrationFromJSON,
   orchestrationGrounding
 } from './orchestration.js';
 export {

diff --git a/sample-code/src/model-orchestration-config.json b/sample-code/src/model-orchestration-config.json
@@ -0,0 +1,21 @@
+{
+  "module_configurations": {
+    "llm_module_config": {
+      "model_name": "gpt-4o",
+      "model_params": {},
+      "model_version": "latest"
+    },
+    "templating_module_config": {
+      "template": [
+        {
+          "role": "user",
+          "content": "SAP"
+        },
+        {
+          "role": "system",
+          "content": "When given the name of a company, provide the year of it's establishment."
+        }
+      ]
+    }
+  }
+}
diff --git a/sample-code/src/orchestration.ts b/sample-code/src/orchestration.ts
@@ -1,3 +1,4 @@
+import { readFile } from 'node:fs/promises';
 import {
   OrchestrationClient,
   buildAzureContentFilter
@@ -209,6 +210,24 @@ export async function orchestrationRequestConfig(): Promise<OrchestrationRespons
   );
 }
 
+/**
+ * Use the orchestration service with JSON obtained from AI Launchpad.
+ * @returns The orchestration service response.
+ */
+export async function orchestrationFromJSON(): Promise<
+  OrchestrationResponse | undefined
+> {
+  // You can also provide the JSON configuration as a plain string in the code directly instead.
+  const jsonConfig = await readFile(
+    './src/model-orchestration-config.json',
+    'utf-8'
+  );
+  const response = await new OrchestrationClient(jsonConfig).chatCompletion();
+
+  logger.info(response.getContent());
+  return response;
+}
+
 /**
  * Ask about a custom knowledge embedded in document grounding.
  * @returns The orchestration service response.