OPIK-134-evaluate-experiment_name (#318)

* [OPIK-134] update fern/rest_api code * [OPIK-134] parameter experiment_name is optional/can be None now * [OPIK-134] fix fern generated code * [OPIK-134] fix linter * [OPIK-134] add e2e test * [OPIK-134] remove unused files * [OPIK-134] fix linter --------- Co-authored-by: Andres Cruz <[email protected]>
comet-ml · Sep 30, 2024 · 74357d2 · 74357d2
1 parent 2b942b7
commit 74357d2
Show file tree

Hide file tree

Showing 19 changed files with 3,624 additions and 20 deletions.
diff --git a/sdks/python/code_generation/fern/openapi/openapi.yaml b/sdks/python/code_generation/fern/openapi/openapi.yaml
diff --git a/sdks/python/code_generation/fern/readme.md b/sdks/python/code_generation/fern/readme.md
@@ -5,6 +5,8 @@ How to generate new client code for communication with Opik backend
 1. Execute the ./build_and_run.sh script from the root of repository
 2. Go to http://localhost:3003/ (URL for backend API specification)
 3. Download openapi specification file - `openapi.yaml`
-4. Put this file into `code_generation/fern/openapi/openapi.yaml`
+4. Put this file into `code_generation/fern/openapi/openapi.yaml` (overwrite it).<br>
+   Note that this path contains the version of the schema from which the code in `src/opik/rest_api` for the SDK was generated. Therefore, it might not be the latest version of the schema.
 5. Run `fern generate` from inside `code_generation/fern` folder. This will generate a python code inside the directory called `sdks` near the `fern` one.
 7. Replace content of `src/opik/rest_api` with the python package inside `sdks` (there will be few nested directories, navigate until you find python files)
+8. Run `pre-commit run --all-files` to format code
diff --git a/sdks/python/src/opik/api_objects/experiment/experiment.py b/sdks/python/src/opik/api_objects/experiment/experiment.py
@@ -1,5 +1,5 @@
 import logging
-from typing import List
+from typing import List, Optional
 
 from opik.rest_api import client as rest_api_client
 from opik.rest_api.types import experiment_item as rest_experiment_item
@@ -14,7 +14,7 @@ class Experiment:
     def __init__(
         self,
         id: str,
-        name: str,
+        name: Optional[str],
         dataset_name: str,
         rest_client: rest_api_client.OpikApi,
     ) -> None:

diff --git a/sdks/python/src/opik/api_objects/opik_client.py b/sdks/python/src/opik/api_objects/opik_client.py
@@ -364,10 +364,21 @@ def create_dataset(
 
     def create_experiment(
         self,
-        name: str,
         dataset_name: str,
+        name: Optional[str] = None,
         experiment_config: Optional[Dict[str, Any]] = None,
     ) -> experiment.Experiment:
+        """
+        Creates a new experiment using the given dataset name and optional parameters.
+
+        Args:
+            dataset_name (str): The name of the dataset to associate with the experiment.
+            name (Optional[str]): The optional name for the experiment. If None, a generated name will be used.
+            experiment_config (Optional[Dict[str, Any]]): Optional experiment configuration parameters. Must be a dictionary if provided.
+
+        Returns:
+            experiment.Experiment: The newly created experiment object.
+        """
         id = helpers.generate_id()
 
         if isinstance(experiment_config, Mapping):

diff --git a/sdks/python/src/opik/cli.py b/sdks/python/src/opik/cli.py
@@ -29,7 +29,8 @@ def cli() -> None:
 )
 def configure(use_local: bool) -> None:
     """
-    Create a configuration file for the Opik Python SDK, if a configuration file already exists, it will be overwritten. This is also available as a function in the Python SDK.
+    Create a configuration file for the Opik Python SDK, if a configuration file already exists, it will be overwritten.
+    This is also available as a function in the Python SDK.
     """
     if use_local:
         opik_configure.configure(use_local=True, force=True)

diff --git a/sdks/python/src/opik/evaluation/evaluation_result.py b/sdks/python/src/opik/evaluation/evaluation_result.py
@@ -1,4 +1,4 @@
-from typing import List
+from typing import List, Optional
 
 import dataclasses
 
@@ -8,5 +8,5 @@
 @dataclasses.dataclass
 class EvaluationResult:
     experiment_id: str
-    experiment_name: str
+    experiment_name: Optional[str]
     test_results: List[test_result.TestResult]
diff --git a/sdks/python/src/opik/evaluation/evaluator.py b/sdks/python/src/opik/evaluation/evaluator.py
@@ -14,7 +14,7 @@ def evaluate(
     dataset: dataset.Dataset,
     task: LLMTask,
     scoring_metrics: List[base_metric.BaseMetric],
-    experiment_name: str,
+    experiment_name: Optional[str] = None,
     experiment_config: Optional[Dict[str, Any]] = None,
     verbose: int = 1,
     task_threads: int = 16,
@@ -28,7 +28,8 @@ def evaluate(
         task: A callable object that takes DatasetItem as input and returns
             dictionary which will later be used for scoring
 
-        experiment_name: The name of the experiment associated with evaluation run
+        experiment_name: The name of the experiment associated with evaluation run.
+            If None, a generated name will be used.
 
         experiment_config: The dictionary with parameters that describe experiment
 

diff --git a/sdks/python/src/opik/rest_api/__init__.py b/sdks/python/src/opik/rest_api/__init__.py
@@ -86,9 +86,11 @@
     SpanWriteType,
     Trace,
     TraceBatch,
+    TraceCountResponse,
     TracePagePublic,
     TracePublic,
     TraceWrite,
+    WorkspaceTraceCount,
 )
 from .errors import (
     BadRequestError,
@@ -97,7 +99,15 @@
     NotImplementedError,
     UnprocessableEntityError,
 )
-from . import datasets, experiments, feedback_definitions, projects, spans, traces
+from . import (
+    datasets,
+    experiments,
+    feedback_definitions,
+    projects,
+    spans,
+    system_usage,
+    traces,
+)
 from .environment import OpikApiEnvironment
 from .feedback_definitions import FindFeedbackDefinitionsRequestType
 from .spans import GetSpansByProjectRequestType
@@ -195,14 +205,17 @@
     "SpanWriteType",
     "Trace",
     "TraceBatch",
+    "TraceCountResponse",
     "TracePagePublic",
     "TracePublic",
     "TraceWrite",
     "UnprocessableEntityError",
+    "WorkspaceTraceCount",
     "datasets",
     "experiments",
     "feedback_definitions",
     "projects",
     "spans",
+    "system_usage",
     "traces",
 ]
diff --git a/sdks/python/src/opik/rest_api/client.py b/sdks/python/src/opik/rest_api/client.py
@@ -18,6 +18,7 @@
 )
 from .projects.client import AsyncProjectsClient, ProjectsClient
 from .spans.client import AsyncSpansClient, SpansClient
+from .system_usage.client import AsyncSystemUsageClient, SystemUsageClient
 from .traces.client import AsyncTracesClient, TracesClient
 
 
@@ -78,6 +79,7 @@ def __init__(
             else httpx.Client(timeout=_defaulted_timeout),
             timeout=_defaulted_timeout,
         )
+        self.system_usage = SystemUsageClient(client_wrapper=self._client_wrapper)
         self.datasets = DatasetsClient(client_wrapper=self._client_wrapper)
         self.experiments = ExperimentsClient(client_wrapper=self._client_wrapper)
         self.feedback_definitions = FeedbackDefinitionsClient(
@@ -177,6 +179,7 @@ def __init__(
             else httpx.AsyncClient(timeout=_defaulted_timeout),
             timeout=_defaulted_timeout,
         )
+        self.system_usage = AsyncSystemUsageClient(client_wrapper=self._client_wrapper)
         self.datasets = AsyncDatasetsClient(client_wrapper=self._client_wrapper)
         self.experiments = AsyncExperimentsClient(client_wrapper=self._client_wrapper)
         self.feedback_definitions = AsyncFeedbackDefinitionsClient(

diff --git a/sdks/python/src/opik/rest_api/experiments/client.py b/sdks/python/src/opik/rest_api/experiments/client.py
@@ -78,8 +78,8 @@ def create_experiment(
         self,
         *,
         dataset_name: str,
-        name: str,
         id: typing.Optional[str] = OMIT,
+        name: typing.Optional[str] = OMIT,
         metadata: typing.Optional[JsonNodeWrite] = OMIT,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> None:
@@ -90,10 +90,10 @@ def create_experiment(
         ----------
         dataset_name : str
 
-        name : str
-
         id : typing.Optional[str]
 
+        name : typing.Optional[str]
+
         metadata : typing.Optional[JsonNodeWrite]
 
         request_options : typing.Optional[RequestOptions]
@@ -110,7 +110,6 @@ def create_experiment(
         client = OpikApi()
         client.experiments.create_experiment(
             dataset_name="dataset_name",
-            name="name",
         )
         """
         _response = self._client_wrapper.httpx_client.request(
@@ -316,6 +315,66 @@ def get_experiment_item_by_id(
             raise ApiError(status_code=_response.status_code, body=_response.text)
         raise ApiError(status_code=_response.status_code, body=_response_json)
 
+    def stream_experiment_items(
+        self,
+        *,
+        experiment_name: str,
+        limit: typing.Optional[int] = OMIT,
+        last_retrieved_id: typing.Optional[str] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.Iterator[bytes]:
+        """
+        Stream experiment items
+
+        Parameters
+        ----------
+        experiment_name : str
+
+        limit : typing.Optional[int]
+
+        last_retrieved_id : typing.Optional[str]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Yields
+        ------
+        typing.Iterator[bytes]
+            Experiment items stream or error during process
+
+        Examples
+        --------
+        from Opik.client import OpikApi
+
+        client = OpikApi()
+        client.experiments.stream_experiment_items(
+            experiment_name="string",
+            limit=1,
+            last_retrieved_id="string",
+        )
+        """
+        with self._client_wrapper.httpx_client.stream(
+            "v1/private/experiments/items/stream",
+            method="POST",
+            json={
+                "experiment_name": experiment_name,
+                "limit": limit,
+                "last_retrieved_id": last_retrieved_id,
+            },
+            request_options=request_options,
+            omit=OMIT,
+        ) as _response:
+            try:
+                if 200 <= _response.status_code < 300:
+                    for _chunk in _response.iter_bytes():
+                        yield _chunk
+                    return
+                _response.read()
+                _response_json = _response.json()
+            except JSONDecodeError:
+                raise ApiError(status_code=_response.status_code, body=_response.text)
+            raise ApiError(status_code=_response.status_code, body=_response_json)
+
 
 class AsyncExperimentsClient:
     def __init__(self, *, client_wrapper: AsyncClientWrapper):
@@ -384,8 +443,8 @@ async def create_experiment(
         self,
         *,
         dataset_name: str,
-        name: str,
         id: typing.Optional[str] = OMIT,
+        name: typing.Optional[str] = OMIT,
         metadata: typing.Optional[JsonNodeWrite] = OMIT,
         request_options: typing.Optional[RequestOptions] = None,
     ) -> None:
@@ -396,10 +455,10 @@ async def create_experiment(
         ----------
         dataset_name : str
 
-        name : str
-
         id : typing.Optional[str]
 
+        name : typing.Optional[str]
+
         metadata : typing.Optional[JsonNodeWrite]
 
         request_options : typing.Optional[RequestOptions]
@@ -421,7 +480,6 @@ async def create_experiment(
         async def main() -> None:
             await client.experiments.create_experiment(
                 dataset_name="dataset_name",
-                name="name",
             )
 
 
@@ -661,3 +719,71 @@ async def main() -> None:
         except JSONDecodeError:
             raise ApiError(status_code=_response.status_code, body=_response.text)
         raise ApiError(status_code=_response.status_code, body=_response_json)
+
+    async def stream_experiment_items(
+        self,
+        *,
+        experiment_name: str,
+        limit: typing.Optional[int] = OMIT,
+        last_retrieved_id: typing.Optional[str] = OMIT,
+        request_options: typing.Optional[RequestOptions] = None,
+    ) -> typing.AsyncIterator[bytes]:
+        """
+        Stream experiment items
+
+        Parameters
+        ----------
+        experiment_name : str
+
+        limit : typing.Optional[int]
+
+        last_retrieved_id : typing.Optional[str]
+
+        request_options : typing.Optional[RequestOptions]
+            Request-specific configuration.
+
+        Yields
+        ------
+        typing.AsyncIterator[bytes]
+            Experiment items stream or error during process
+
+        Examples
+        --------
+        import asyncio
+
+        from Opik.client import AsyncOpikApi
+
+        client = AsyncOpikApi()
+
+
+        async def main() -> None:
+            await client.experiments.stream_experiment_items(
+                experiment_name="string",
+                limit=1,
+                last_retrieved_id="string",
+            )
+
+
+        asyncio.run(main())
+        """
+        async with self._client_wrapper.httpx_client.stream(
+            "v1/private/experiments/items/stream",
+            method="POST",
+            json={
+                "experiment_name": experiment_name,
+                "limit": limit,
+                "last_retrieved_id": last_retrieved_id,
+            },
+            request_options=request_options,
+            omit=OMIT,
+        ) as _response:
+            try:
+                if 200 <= _response.status_code < 300:
+                    async for _chunk in _response.aiter_bytes():
+                        yield _chunk
+                    return
+                await _response.aread()
+                _response_json = _response.json()
+            except JSONDecodeError:
+                raise ApiError(status_code=_response.status_code, body=_response.text)
+            raise ApiError(status_code=_response.status_code, body=_response_json)
diff --git a/sdks/python/src/opik/rest_api/system_usage/__init__.py b/sdks/python/src/opik/rest_api/system_usage/__init__.py
@@ -0,0 +1 @@
+# This file was auto-generated by Fern from our API Definition.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		# This file was auto-generated by Fern from our API Definition.