FriendliAI: Documentation Updates (#7517)

* docs(friendliai.md): update FriendliAI documentation and model details * docs(friendliai.md): remove unused imports for cleaner documentation * feat: add support for parallel function calling, system messages, and response schema in model configuration
BerriAI · Jan 5, 2025 · f7931b6 · f7931b6
1 parent 2b0738e
commit f7931b6
Show file tree

Hide file tree

Showing 5 changed files with 47 additions and 54 deletions.
diff --git a/docs/my-website/docs/providers/friendliai.md b/docs/my-website/docs/providers/friendliai.md
@@ -1,23 +1,32 @@
 # FriendliAI
-https://suite.friendli.ai/
 
+:::info
 **We support ALL FriendliAI models, just set `friendliai/` as a prefix when sending completion requests**
+:::
+
+| Property                   | Details                                                                                         |
+| -------------------------- | ----------------------------------------------------------------------------------------------- |
+| Description                | The fastest and most efficient inference engine to build production-ready, compound AI systems. |
+| Provider Route on LiteLLM  | `friendliai/`                                                                                   |
+| Provider Doc               | [FriendliAI ↗](https://friendli.ai/docs/sdk/integrations/litellm)                               |
+| Supported OpenAI Endpoints | `/chat/completions`, `/completions`                                                             |
 
 ## API Key
+
 ```python
 # env variable
 os.environ['FRIENDLI_TOKEN']
-os.environ['FRIENDLI_API_BASE'] # Optional. Set this when using dedicated endpoint.
 ```
 
 ## Sample Usage
+
 ```python
 from litellm import completion
 import os
 
 os.environ['FRIENDLI_TOKEN'] = ""
 response = completion(
-    model="friendliai/mixtral-8x7b-instruct-v0-1", 
+    model="friendliai/meta-llama-3.1-8b-instruct",
     messages=[
        {"role": "user", "content": "hello from litellm"}
    ],
@@ -26,13 +35,14 @@ print(response)
 ```
 
 ## Sample Usage - Streaming
+
 ```python
 from litellm import completion
 import os
 
 os.environ['FRIENDLI_TOKEN'] = ""
 response = completion(
-    model="friendliai/mixtral-8x7b-instruct-v0-1", 
+    model="friendliai/meta-llama-3.1-8b-instruct",
     messages=[
        {"role": "user", "content": "hello from litellm"}
    ],
@@ -43,18 +53,11 @@ for chunk in response:
     print(chunk)
 ```
 
-
 ## Supported Models
-### Serverless Endpoints
-We support ALL FriendliAI AI models, just set `friendliai/` as a prefix when sending completion requests
 
-| Model Name               | Function Call                                                                                                                                                      |
-|--------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| mixtral-8x7b-instruct | `completion(model="friendliai/mixtral-8x7b-instruct-v0-1", messages)` | 
-| meta-llama-3-8b-instruct | `completion(model="friendliai/meta-llama-3-8b-instruct", messages)` |
-| meta-llama-3-70b-instruct | `completion(model="friendliai/meta-llama-3-70b-instruct", messages)` |  
+We support ALL FriendliAI AI models, just set `friendliai/` as a prefix when sending completion requests
 
-### Dedicated Endpoints
-```
-model="friendliai/$ENDPOINT_ID:$ADAPTER_ROUTE"
-```
+| Model Name                  | Function Call                                                          |
+| --------------------------- | ---------------------------------------------------------------------- |
+| meta-llama-3.1-8b-instruct  | `completion(model="friendliai/meta-llama-3.1-8b-instruct", messages)`  |
+| meta-llama-3.1-70b-instruct | `completion(model="friendliai/meta-llama-3.1-70b-instruct", messages)` |
diff --git a/litellm/__init__.py b/litellm/__init__.py
@@ -609,7 +609,7 @@ def add_known_models():
     "api.deepseek.com/v1",
     "api.together.xyz/v1",
     "app.empower.dev/api/v1",
-    "inference.friendli.ai/v1",
+    "https://api.friendli.ai/serverless/v1",
     "api.sambanova.ai/v1",
     "api.x.ai/v1",
     "api.galadriel.ai/v1",

diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py
@@ -208,11 +208,9 @@ def get_llm_provider(  # noqa: PLR0915
                     elif endpoint == "api.deepseek.com/v1":
                         custom_llm_provider = "deepseek"
                         dynamic_api_key = get_secret_str("DEEPSEEK_API_KEY")
-                    elif endpoint == "inference.friendli.ai/v1":
+                    elif endpoint == "https://api.friendli.ai/serverless/v1":
                         custom_llm_provider = "friendliai"
-                        dynamic_api_key = get_secret_str(
-                            "FRIENDLIAI_API_KEY"
-                        ) or get_secret("FRIENDLI_TOKEN")
+                        dynamic_api_key = get_secret_str("FRIENDLIAI_API_KEY") or get_secret("FRIENDLI_TOKEN")
                     elif endpoint == "api.galadriel.com/v1":
                         custom_llm_provider = "galadriel"
                         dynamic_api_key = get_secret_str("GALADRIEL_API_KEY")
@@ -550,7 +548,7 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
         api_base = (
             api_base
             or get_secret("FRIENDLI_API_BASE")
-            or "https://inference.friendli.ai/v1"
+            or "https://api.friendli.ai/serverless/v1"
         )  # type: ignore
         dynamic_api_key = (
             api_key

diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
@@ -2264,35 +2264,31 @@
         "mode": "chat",
         "supports_function_calling": true
     },
-    "friendliai/mixtral-8x7b-instruct-v0-1": {
-        "max_tokens": 32768,
-        "max_input_tokens": 32768,
-        "max_output_tokens": 32768,
-        "input_cost_per_token": 0.0000004,
-        "output_cost_per_token": 0.0000004,
-        "litellm_provider": "friendliai",
-        "mode": "chat",
-        "supports_function_calling": true
-    },
-    "friendliai/meta-llama-3-8b-instruct": {
+    "friendliai/meta-llama-3.1-8b-instruct": {
         "max_tokens": 8192,
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "input_cost_per_token": 0.0000001,
         "output_cost_per_token": 0.0000001,
         "litellm_provider": "friendliai",
         "mode": "chat",
-        "supports_function_calling": true
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_system_messages": true,
+        "supports_response_schema": true
     },
-    "friendliai/meta-llama-3-70b-instruct": {
+    "friendliai/meta-llama-3.1-70b-instruct": {
         "max_tokens": 8192,
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
-        "input_cost_per_token": 0.0000008,
-        "output_cost_per_token": 0.0000008,
+        "input_cost_per_token": 0.0000006,
+        "output_cost_per_token": 0.0000006,
         "litellm_provider": "friendliai",
         "mode": "chat",
-        "supports_function_calling": true
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_system_messages": true,
+        "supports_response_schema": true
     },
     "claude-instant-1.2": {
         "max_tokens": 8191,

diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
@@ -2264,35 +2264,31 @@
         "mode": "chat",
         "supports_function_calling": true
     },
-    "friendliai/mixtral-8x7b-instruct-v0-1": {
-        "max_tokens": 32768,
-        "max_input_tokens": 32768,
-        "max_output_tokens": 32768,
-        "input_cost_per_token": 0.0000004,
-        "output_cost_per_token": 0.0000004,
-        "litellm_provider": "friendliai",
-        "mode": "chat",
-        "supports_function_calling": true
-    },
-    "friendliai/meta-llama-3-8b-instruct": {
+    "friendliai/meta-llama-3.1-8b-instruct": {
         "max_tokens": 8192,
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
         "input_cost_per_token": 0.0000001,
         "output_cost_per_token": 0.0000001,
         "litellm_provider": "friendliai",
         "mode": "chat",
-        "supports_function_calling": true
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_system_messages": true,
+        "supports_response_schema": true
     },
-    "friendliai/meta-llama-3-70b-instruct": {
+    "friendliai/meta-llama-3.1-70b-instruct": {
         "max_tokens": 8192,
         "max_input_tokens": 8192,
         "max_output_tokens": 8192,
-        "input_cost_per_token": 0.0000008,
-        "output_cost_per_token": 0.0000008,
+        "input_cost_per_token": 0.0000006,
+        "output_cost_per_token": 0.0000006,
         "litellm_provider": "friendliai",
         "mode": "chat",
-        "supports_function_calling": true
+        "supports_function_calling": true,
+        "supports_parallel_function_calling": true,
+        "supports_system_messages": true,
+        "supports_response_schema": true
     },
     "claude-instant-1.2": {
         "max_tokens": 8191,