From 74357d2e645d916971177d4039009e16ae154e13 Mon Sep 17 00:00:00 2001 From: Alexander Barannikov <32936723+japdubengsub@users.noreply.github.com> Date: Mon, 30 Sep 2024 18:08:13 +0100 Subject: [PATCH] OPIK-134-evaluate-experiment_name (#318) * [OPIK-134] update fern/rest_api code * [OPIK-134] parameter experiment_name is optional/can be None now * [OPIK-134] fix fern generated code * [OPIK-134] fix linter * [OPIK-134] add e2e test * [OPIK-134] remove unused files * [OPIK-134] fix linter --------- Co-authored-by: Andres Cruz --- .../code_generation/fern/openapi/openapi.yaml | 3106 +++++++++++++++++ sdks/python/code_generation/fern/readme.md | 4 +- .../opik/api_objects/experiment/experiment.py | 4 +- .../src/opik/api_objects/opik_client.py | 13 +- sdks/python/src/opik/cli.py | 3 +- .../src/opik/evaluation/evaluation_result.py | 4 +- sdks/python/src/opik/evaluation/evaluator.py | 5 +- sdks/python/src/opik/rest_api/__init__.py | 15 +- sdks/python/src/opik/rest_api/client.py | 3 + .../src/opik/rest_api/experiments/client.py | 142 +- .../opik/rest_api/system_usage/__init__.py | 1 + .../src/opik/rest_api/system_usage/client.py | 100 + .../python/src/opik/rest_api/traces/client.py | 96 + .../src/opik/rest_api/types/__init__.py | 4 + .../src/opik/rest_api/types/experiment.py | 2 +- .../opik/rest_api/types/experiment_public.py | 2 +- .../rest_api/types/trace_count_response.py | 43 + .../rest_api/types/workspace_trace_count.py | 43 + sdks/python/tests/e2e/test_experiment.py | 54 + 19 files changed, 3624 insertions(+), 20 deletions(-) create mode 100644 sdks/python/code_generation/fern/openapi/openapi.yaml create mode 100644 sdks/python/src/opik/rest_api/system_usage/__init__.py create mode 100644 sdks/python/src/opik/rest_api/system_usage/client.py create mode 100644 sdks/python/src/opik/rest_api/types/trace_count_response.py create mode 100644 sdks/python/src/opik/rest_api/types/workspace_trace_count.py diff --git a/sdks/python/code_generation/fern/openapi/openapi.yaml b/sdks/python/code_generation/fern/openapi/openapi.yaml new file mode 100644 index 0000000000..c424c41075 --- /dev/null +++ b/sdks/python/code_generation/fern/openapi/openapi.yaml @@ -0,0 +1,3106 @@ +openapi: 3.1.0 +info: + title: APIs + description: APIs + contact: + name: Support + email: support@comet.com + license: + name: Apache 2.0 + url: http://www.apache.org/licenses/LICENSE-2.0.html + version: 1.0.0 +servers: +- url: "{basePath}/{apiVersion}" + description: Local server + variables: + basePath: + default: http://localhost:8080 + apiVersion: + default: v1 +tags: +- name: System usage + description: System usage related resource +- name: Datasets + description: Dataset resources +- name: Experiments + description: Experiment resources +- name: Feedback-definitions + description: Feedback definitions related resources +- name: Projects + description: Project related resources +- name: Spans + description: Span related resources +- name: Traces + description: Trace related resources +paths: + /v1/internal/usage/workspace-trace-counts: + get: + tags: + - System usage + summary: Get traces count on previous day for all available workspaces + description: Get traces count on previous day for all available workspaces + operationId: getTracesCountForWorkspaces + responses: + "200": + description: TraceCountResponse resource + content: + application/json: + schema: + $ref: '#/components/schemas/TraceCountResponse' + /v1/private/datasets: + get: + tags: + - Datasets + summary: Find datasets + description: Find datasets + operationId: findDatasets + parameters: + - name: page + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 1 + - name: size + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 10 + - name: name + in: query + schema: + type: string + responses: + "200": + description: Dataset resource + content: + application/json: + schema: + $ref: '#/components/schemas/DatasetPage_Public' + post: + tags: + - Datasets + summary: Create dataset + description: Create dataset + operationId: createDataset + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Dataset_Write' + responses: + "201": + description: Created + headers: + Location: + required: true + style: simple + schema: + type: string + example: "${basePath}/api/v1/private/datasets/{id}" + /v1/private/datasets/items: + put: + tags: + - Datasets + summary: Create/update dataset items + description: Create/update dataset items based on dataset item id + operationId: createOrUpdateDatasetItems + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DatasetItemBatch_Write' + responses: + "204": + description: No content + /v1/private/datasets/{id}: + get: + tags: + - Datasets + summary: Get dataset by id + description: Get dataset by id + operationId: getDatasetById + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: Dataset resource + content: + application/json: + schema: + $ref: '#/components/schemas/Dataset_Public' + put: + tags: + - Datasets + summary: Update dataset by id + description: Update dataset by id + operationId: updateDataset + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DatasetUpdate' + responses: + "204": + description: No content + delete: + tags: + - Datasets + summary: Delete dataset by id + description: Delete dataset by id + operationId: deleteDataset + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "204": + description: No content + /v1/private/datasets/delete: + post: + tags: + - Datasets + summary: Delete dataset by name + description: Delete dataset by name + operationId: deleteDatasetByName + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DatasetIdentifier' + responses: + "204": + description: No content + /v1/private/datasets/items/delete: + post: + tags: + - Datasets + summary: Delete dataset items + description: Delete dataset items + operationId: deleteDatasetItems + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DatasetItemsDelete' + responses: + "204": + description: No content + /v1/private/datasets/{id}/items/experiments/items: + get: + tags: + - Datasets + summary: Find dataset items with experiment items + description: Find dataset items with experiment items + operationId: findDatasetItemsWithExperimentItems + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + - name: page + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 1 + - name: size + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 10 + - name: experiment_ids + in: query + required: true + schema: + type: string + - name: filters + in: query + schema: + type: string + responses: + "200": + description: Dataset item resource + content: + application/json: + schema: + $ref: '#/components/schemas/DatasetItemPage_Compare' + /v1/private/datasets/retrieve: + post: + tags: + - Datasets + summary: Get dataset by name + description: Get dataset by name + operationId: getDatasetByIdentifier + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DatasetIdentifier_Public' + responses: + "200": + description: Dataset resource + content: + application/json: + schema: + $ref: '#/components/schemas/Dataset_Public' + /v1/private/datasets/items/{itemId}: + get: + tags: + - Datasets + summary: Get dataset item by id + description: Get dataset item by id + operationId: getDatasetItemById + parameters: + - name: itemId + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: Dataset item resource + content: + application/json: + schema: + $ref: '#/components/schemas/DatasetItem_Public' + /v1/private/datasets/{id}/items: + get: + tags: + - Datasets + summary: Get dataset items + description: Get dataset items + operationId: getDatasetItems + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + - name: page + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 1 + - name: size + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 10 + responses: + "200": + description: Dataset items resource + content: + application/json: + schema: + $ref: '#/components/schemas/DatasetItemPage_Public' + /v1/private/datasets/items/stream: + post: + tags: + - Datasets + summary: Stream dataset items + description: Stream dataset items + operationId: streamDatasetItems + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DatasetItemStreamRequest' + responses: + "200": + description: Dataset items stream or error during process + content: + application/octet-stream: + schema: + maxItems: 2000 + type: array + items: + type: string + anyOf: + - $ref: '#/components/schemas/DatasetItem' + - $ref: '#/components/schemas/ErrorMessage' + /v1/private/experiments: + get: + tags: + - Experiments + summary: Find experiments + description: Find experiments + operationId: findExperiments + parameters: + - name: page + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 1 + - name: size + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 10 + - name: datasetId + in: query + schema: + type: string + format: uuid + - name: name + in: query + schema: + type: string + responses: + "200": + description: Experiments resource + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentPage_Public' + post: + tags: + - Experiments + summary: Create experiment + description: Create experiment + operationId: createExperiment + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Experiment_Write' + responses: + "201": + description: Created + headers: + Location: + required: true + style: simple + schema: + type: string + example: "${basePath}/v1/private/experiments/{id}" + /v1/private/experiments/items: + post: + tags: + - Experiments + summary: Create experiment items + description: Create experiment items + operationId: createExperimentItems + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentItemsBatch' + responses: + "204": + description: No content + /v1/private/experiments/items/delete: + post: + tags: + - Experiments + summary: Delete experiment items + description: Delete experiment items + operationId: deleteExperimentItems + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentItemsDelete' + responses: + "204": + description: No content + /v1/private/experiments/{id}: + get: + tags: + - Experiments + summary: Get experiment by id + description: Get experiment by id + operationId: getExperimentById + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: Experiment resource + content: + application/json: + schema: + $ref: '#/components/schemas/Experiment_Public' + "404": + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage_Public' + /v1/private/experiments/items/{id}: + get: + tags: + - Experiments + summary: Get experiment item by id + description: Get experiment item by id + operationId: getExperimentItemById + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: Experiment item resource + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentItem_Public' + "404": + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage_Public' + /v1/private/experiments/items/stream: + post: + tags: + - Experiments + summary: Stream experiment items + description: Stream experiment items + operationId: streamExperimentItems + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ExperimentItemStreamRequest' + responses: + "200": + description: Experiment items stream or error during process + content: + application/octet-stream: + schema: + maxItems: 2000 + type: array + items: + type: string + anyOf: + - $ref: '#/components/schemas/ExperimentItem' + - $ref: '#/components/schemas/ErrorMessage' + /v1/private/feedback-definitions: + get: + tags: + - Feedback-definitions + summary: Find Feedback definitions + description: Find Feedback definitions + operationId: findFeedbackDefinitions + parameters: + - name: page + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 1 + - name: size + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 10 + - name: name + in: query + schema: + type: string + - name: type + in: query + schema: + type: string + enum: + - numerical + - categorical + responses: + "200": + description: Feedback definitions resource + content: + application/json: + schema: + $ref: '#/components/schemas/FeedbackDefinitionPage_Public' + post: + tags: + - Feedback-definitions + summary: Create feedback definition + description: Get feedback definition + operationId: createFeedbackDefinition + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Feedback_Create' + responses: + "201": + description: Created + headers: + Location: + required: true + style: simple + schema: + type: string + example: "${basePath}/v1/private/feedback-definitions/{feedbackId}" + /v1/private/feedback-definitions/{id}: + get: + tags: + - Feedback-definitions + summary: Get feedback definition by id + description: Get feedback definition by id + operationId: getFeedbackDefinitionById + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: Feedback definition resource + content: + application/json: + schema: + $ref: '#/components/schemas/Feedback_Public' + put: + tags: + - Feedback-definitions + summary: Update feedback definition by id + description: Update feedback definition by id + operationId: updateFeedbackDefinition + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Feedback_Update' + responses: + "204": + description: No Content + delete: + tags: + - Feedback-definitions + summary: Delete feedback definition by id + description: Delete feedback definition by id + operationId: deleteFeedbackDefinitionById + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "204": + description: No Content + /v1/private/projects: + get: + tags: + - Projects + summary: Find projects + description: Find projects + operationId: findProjects + parameters: + - name: page + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 1 + - name: size + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 10 + - name: name + in: query + schema: + type: string + responses: + "200": + description: Project resource + content: + application/json: + schema: + $ref: '#/components/schemas/ProjectPage_Public' + post: + tags: + - Projects + summary: Create project + description: Get project + operationId: createProject + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Project_Write' + responses: + "201": + description: Created + headers: + Location: + required: true + style: simple + schema: + type: string + example: "${basePath}/v1/private/projects/{projectId}" + "422": + description: Unprocessable Content + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage' + "400": + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage' + /v1/private/projects/{id}: + get: + tags: + - Projects + summary: Get project by id + description: Get project by id + operationId: getProjectById + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: Project resource + content: + application/json: + schema: + $ref: '#/components/schemas/Project_Public' + delete: + tags: + - Projects + summary: Delete project by id + description: Delete project by id + operationId: deleteProjectById + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "204": + description: No Content + "409": + description: Conflict + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage' + patch: + tags: + - Projects + summary: Update project by id + description: Update project by id + operationId: updateProject + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/ProjectUpdate' + responses: + "204": + description: No Content + "422": + description: Unprocessable Content + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage' + "400": + description: Bad Request + content: + application/json: + schema: + $ref: '#/components/schemas/ErrorMessage' + /v1/private/spans/{id}/feedback-scores: + put: + tags: + - Spans + summary: Add span feedback score + description: Add span feedback score + operationId: addSpanFeedbackScore + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/FeedbackScore' + responses: + "204": + description: No Content + /v1/private/spans: + get: + tags: + - Spans + summary: Get spans by project_name or project_id and optionally by trace_id + and/or type + description: Get spans by project_name or project_id and optionally by trace_id + and/or type + operationId: getSpansByProject + parameters: + - name: page + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 1 + - name: size + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 10 + - name: project_name + in: query + schema: + type: string + - name: project_id + in: query + schema: + type: string + format: uuid + - name: trace_id + in: query + schema: + type: string + format: uuid + - name: type + in: query + schema: + type: string + enum: + - general + - tool + - llm + - name: filters + in: query + schema: + type: string + responses: + "200": + description: Spans resource + content: + application/json: + schema: + $ref: '#/components/schemas/SpanPage_Public' + post: + tags: + - Spans + summary: Create span + description: Create span + operationId: createSpan + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Span_Write' + responses: + "201": + description: Created + headers: + Location: + required: true + style: simple + schema: + type: string + example: "${basePath}/v1/private/spans/{spanId}" + /v1/private/spans/batch: + post: + tags: + - Spans + summary: Create spans + description: Create spans + operationId: createSpans + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/SpanBatch_Write' + responses: + "204": + description: No Content + /v1/private/spans/{id}: + get: + tags: + - Spans + summary: Get span by id + description: Get span by id + operationId: getSpanById + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: Span resource + content: + application/json: + schema: + $ref: '#/components/schemas/Span_Public' + "404": + description: Not found + content: + application/json: + schema: + $ref: '#/components/schemas/Span_Public' + delete: + tags: + - Spans + summary: Delete span by id + description: Delete span by id + operationId: deleteSpanById + parameters: + - name: id + in: path + required: true + schema: + type: string + responses: + "501": + description: Not implemented + "204": + description: No Content + patch: + tags: + - Spans + summary: Update span by id + description: Update span by id + operationId: updateSpan + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/SpanUpdate' + responses: + "204": + description: No Content + "404": + description: Not found + /v1/private/spans/{id}/feedback-scores/delete: + post: + tags: + - Spans + summary: Delete span feedback score + description: Delete span feedback score + operationId: deleteSpanFeedbackScore + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteFeedbackScore' + responses: + "204": + description: No Content + /v1/private/spans/feedback-scores: + put: + tags: + - Spans + summary: Batch feedback scoring for spans + description: Batch feedback scoring for spans + operationId: scoreBatchOfSpans + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/FeedbackScoreBatch' + responses: + "204": + description: No Content + /v1/private/traces/{id}/feedback-scores: + put: + tags: + - Traces + summary: Add trace feedback score + description: Add trace feedback score + operationId: addTraceFeedbackScore + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/FeedbackScore' + responses: + "204": + description: No Content + /v1/private/traces: + get: + tags: + - Traces + summary: Get traces by project_name or project_id + description: Get traces by project_name or project_id + operationId: getTracesByProject + parameters: + - name: page + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 1 + - name: size + in: query + schema: + minimum: 1 + type: integer + format: int32 + default: 10 + - name: project_name + in: query + schema: + type: string + - name: project_id + in: query + schema: + type: string + format: uuid + - name: filters + in: query + schema: + type: string + responses: + "200": + description: Trace resource + content: + application/json: + schema: + $ref: '#/components/schemas/TracePage_Public' + post: + tags: + - Traces + summary: Create trace + description: Get trace + operationId: createTrace + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/Trace_Write' + responses: + "201": + description: Created + headers: + Location: + required: true + style: simple + schema: + type: string + example: "${basePath}/v1/private/traces/{traceId}" + /v1/private/traces/batch: + post: + tags: + - Traces + summary: Create traces + description: Create traces + operationId: createTraces + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/TraceBatch_Write' + responses: + "204": + description: No Content + /v1/private/traces/{id}: + get: + tags: + - Traces + summary: Get trace by id + description: Get trace by id + operationId: getTraceById + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "200": + description: Trace resource + content: + application/json: + schema: + $ref: '#/components/schemas/Trace_Public' + delete: + tags: + - Traces + summary: Delete trace by id + description: Delete trace by id + operationId: deleteTraceById + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + responses: + "204": + description: No Content + patch: + tags: + - Traces + summary: Update trace by id + description: Update trace by id + operationId: updateTrace + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/TraceUpdate' + responses: + "204": + description: No Content + /v1/private/traces/{id}/feedback-scores/delete: + post: + tags: + - Traces + summary: Delete trace feedback score + description: Delete trace feedback score + operationId: deleteTraceFeedbackScore + parameters: + - name: id + in: path + required: true + schema: + type: string + format: uuid + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/DeleteFeedbackScore' + responses: + "204": + description: No Content + /v1/private/traces/delete: + post: + tags: + - Traces + summary: Delete traces + description: Delete traces + operationId: deleteTraces + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/TracesDelete' + responses: + "204": + description: No Content + /v1/private/traces/feedback-scores: + put: + tags: + - Traces + summary: Batch feedback scoring for traces + description: Batch feedback scoring for traces + operationId: scoreBatchOfTraces + requestBody: + content: + application/json: + schema: + $ref: '#/components/schemas/FeedbackScoreBatch' + responses: + "204": + description: No Content + /is-alive/ping: + get: + operationId: isAlive + responses: + default: + description: default response + content: + application/json: {} +components: + schemas: + TraceCountResponse: + type: object + properties: + workspaces_traces_count: + type: array + items: + $ref: '#/components/schemas/WorkspaceTraceCount' + WorkspaceTraceCount: + type: object + properties: + workspace: + type: string + trace_count: + type: integer + format: int32 + Dataset: + required: + - name + type: object + properties: + id: + type: string + format: uuid + name: + type: string + description: + pattern: ^(?!\s*$).+ + type: string + created_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + last_updated_by: + type: string + readOnly: true + experiment_count: + type: integer + format: int64 + readOnly: true + most_recent_experiment_at: + type: string + format: date-time + readOnly: true + Dataset_Write: + required: + - name + type: object + properties: + id: + type: string + format: uuid + name: + type: string + description: + pattern: ^(?!\s*$).+ + type: string + DatasetItem: + required: + - input + - source + type: object + properties: + id: + type: string + format: uuid + input: + $ref: '#/components/schemas/JsonNode' + expected_output: + $ref: '#/components/schemas/JsonNode' + metadata: + $ref: '#/components/schemas/JsonNode' + trace_id: + type: string + format: uuid + span_id: + type: string + format: uuid + source: + type: string + enum: + - manual + - trace + - span + - sdk + experiment_items: + type: array + readOnly: true + items: + $ref: '#/components/schemas/ExperimentItem' + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + DatasetItemBatch: + required: + - items + type: object + properties: + dataset_name: + pattern: ^(?!\s*$).+ + type: string + description: "If null, dataset_id must be provided" + dataset_id: + type: string + description: "If null, dataset_name must be provided" + format: uuid + items: + maxItems: 1000 + minItems: 1 + type: array + items: + $ref: '#/components/schemas/DatasetItem' + ExperimentItem: + required: + - dataset_item_id + - experiment_id + - trace_id + type: object + properties: + id: + type: string + format: uuid + experiment_id: + type: string + format: uuid + dataset_item_id: + type: string + format: uuid + trace_id: + type: string + format: uuid + input: + $ref: '#/components/schemas/JsonNode' + output: + $ref: '#/components/schemas/JsonNode' + feedback_scores: + type: array + readOnly: true + items: + $ref: '#/components/schemas/FeedbackScore' + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + FeedbackScore: + required: + - name + - source + - value + type: object + properties: + name: + type: string + category_name: + type: string + value: + maximum: 999999999.999999999 + exclusiveMaximum: false + minimum: -999999999.999999999 + exclusiveMinimum: false + type: number + reason: + type: string + source: + type: string + enum: + - ui + - sdk + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + readOnly: true + JsonNode: + type: object + DatasetItemBatch_Write: + required: + - items + type: object + properties: + dataset_name: + pattern: ^(?!\s*$).+ + type: string + description: "If null, dataset_id must be provided" + dataset_id: + type: string + description: "If null, dataset_name must be provided" + format: uuid + items: + maxItems: 1000 + minItems: 1 + type: array + items: + $ref: '#/components/schemas/DatasetItem_Write' + DatasetItem_Write: + required: + - input + - source + type: object + properties: + id: + type: string + format: uuid + input: + $ref: '#/components/schemas/JsonNode_Write' + expected_output: + $ref: '#/components/schemas/JsonNode_Write' + metadata: + $ref: '#/components/schemas/JsonNode_Write' + trace_id: + type: string + format: uuid + span_id: + type: string + format: uuid + source: + type: string + enum: + - manual + - trace + - span + - sdk + JsonNode_Write: + type: object + DatasetIdentifier: + required: + - dataset_name + type: object + properties: + dataset_name: + type: string + DatasetItemsDelete: + required: + - item_ids + type: object + properties: + item_ids: + maxItems: 1000 + minItems: 1 + type: array + items: + type: string + format: uuid + DatasetItemPage_Compare: + type: object + properties: + content: + type: array + items: + $ref: '#/components/schemas/DatasetItem_Compare' + page: + type: integer + format: int32 + size: + type: integer + format: int32 + total: + type: integer + format: int64 + DatasetItem_Compare: + required: + - input + - source + type: object + properties: + id: + type: string + format: uuid + input: + $ref: '#/components/schemas/JsonNode_Compare' + expected_output: + $ref: '#/components/schemas/JsonNode_Compare' + metadata: + $ref: '#/components/schemas/JsonNode_Compare' + trace_id: + type: string + format: uuid + span_id: + type: string + format: uuid + source: + type: string + enum: + - manual + - trace + - span + - sdk + experiment_items: + type: array + readOnly: true + items: + $ref: '#/components/schemas/ExperimentItem_Compare' + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + ExperimentItem_Compare: + required: + - dataset_item_id + - experiment_id + - trace_id + type: object + properties: + id: + type: string + format: uuid + experiment_id: + type: string + format: uuid + dataset_item_id: + type: string + format: uuid + trace_id: + type: string + format: uuid + input: + $ref: '#/components/schemas/JsonNode_Compare' + output: + $ref: '#/components/schemas/JsonNode_Compare' + feedback_scores: + type: array + readOnly: true + items: + $ref: '#/components/schemas/FeedbackScore_Compare' + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + readOnly: true + FeedbackScore_Compare: + required: + - name + - source + - value + type: object + properties: + name: + type: string + category_name: + type: string + value: + maximum: 999999999.999999999 + exclusiveMaximum: false + minimum: -999999999.999999999 + exclusiveMinimum: false + type: number + reason: + type: string + source: + type: string + enum: + - ui + - sdk + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + readOnly: true + JsonNode_Compare: + type: object + readOnly: true + DatasetPage_Public: + type: object + properties: + content: + type: array + items: + $ref: '#/components/schemas/Dataset_Public' + page: + type: integer + format: int32 + size: + type: integer + format: int32 + total: + type: integer + format: int64 + Dataset_Public: + required: + - name + type: object + properties: + id: + type: string + format: uuid + name: + type: string + description: + pattern: ^(?!\s*$).+ + type: string + created_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + last_updated_by: + type: string + readOnly: true + experiment_count: + type: integer + format: int64 + readOnly: true + most_recent_experiment_at: + type: string + format: date-time + readOnly: true + DatasetIdentifier_Public: + required: + - dataset_name + type: object + properties: + dataset_name: + type: string + DatasetItem_Public: + required: + - input + - source + type: object + properties: + id: + type: string + format: uuid + input: + $ref: '#/components/schemas/JsonNode_Public' + expected_output: + $ref: '#/components/schemas/JsonNode_Public' + metadata: + $ref: '#/components/schemas/JsonNode_Public' + trace_id: + type: string + format: uuid + span_id: + type: string + format: uuid + source: + type: string + enum: + - manual + - trace + - span + - sdk + experiment_items: + type: array + readOnly: true + items: + $ref: '#/components/schemas/ExperimentItem_Public' + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + ExperimentItem_Public: + required: + - dataset_item_id + - experiment_id + - trace_id + type: object + properties: + id: + type: string + format: uuid + experiment_id: + type: string + format: uuid + dataset_item_id: + type: string + format: uuid + trace_id: + type: string + format: uuid + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + JsonNode_Public: + type: object + DatasetItemPage_Public: + type: object + properties: + content: + type: array + items: + $ref: '#/components/schemas/DatasetItem_Public' + page: + type: integer + format: int32 + size: + type: integer + format: int32 + total: + type: integer + format: int64 + ErrorMessage: + type: object + properties: + errors: + type: array + items: + type: string + ChunkedOutputJsonNode: + type: object + properties: + type: + type: object + properties: + typeName: + type: string + closed: + type: boolean + DatasetItemStreamRequest: + required: + - dataset_name + type: object + properties: + dataset_name: + type: string + last_retrieved_id: + type: string + format: uuid + steam_limit: + type: integer + format: int32 + DatasetUpdate: + required: + - name + type: object + properties: + name: + type: string + description: + pattern: ^(?!\s*$).+ + type: string + Experiment: + required: + - dataset_name + type: object + properties: + id: + type: string + format: uuid + dataset_name: + type: string + dataset_id: + type: string + format: uuid + readOnly: true + name: + type: string + metadata: + $ref: '#/components/schemas/JsonNode' + feedback_scores: + type: array + readOnly: true + items: + $ref: '#/components/schemas/FeedbackScoreAverage' + trace_count: + type: integer + format: int64 + readOnly: true + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + FeedbackScoreAverage: + required: + - name + - value + type: object + properties: + name: + type: string + value: + type: number + readOnly: true + Experiment_Write: + required: + - dataset_name + type: object + properties: + id: + type: string + format: uuid + dataset_name: + type: string + name: + type: string + metadata: + $ref: '#/components/schemas/JsonNode_Write' + ExperimentItemsBatch: + required: + - experiment_items + type: object + properties: + experiment_items: + maxItems: 1000 + minItems: 1 + uniqueItems: true + type: array + items: + $ref: '#/components/schemas/ExperimentItem' + ExperimentItemsDelete: + required: + - ids + type: object + properties: + ids: + maxItems: 1000 + minItems: 1 + uniqueItems: true + type: array + items: + type: string + format: uuid + ExperimentPage_Public: + type: object + properties: + page: + type: integer + format: int32 + size: + type: integer + format: int32 + total: + type: integer + format: int64 + content: + type: array + items: + $ref: '#/components/schemas/Experiment_Public' + Experiment_Public: + required: + - dataset_name + type: object + properties: + id: + type: string + format: uuid + dataset_name: + type: string + dataset_id: + type: string + format: uuid + readOnly: true + name: + type: string + metadata: + $ref: '#/components/schemas/JsonNode_Public' + feedback_scores: + type: array + readOnly: true + items: + $ref: '#/components/schemas/FeedbackScoreAverage_Public' + trace_count: + type: integer + format: int64 + readOnly: true + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + FeedbackScoreAverage_Public: + required: + - name + - value + type: object + properties: + name: + type: string + value: + type: number + readOnly: true + ErrorMessage_Public: + type: object + properties: + code: + type: integer + format: int32 + message: + type: string + details: + type: string + ExperimentItemStreamRequest: + required: + - experiment_name + type: object + properties: + experiment_name: + type: string + limit: + type: integer + format: int32 + last_retrieved_id: + type: string + format: uuid + CategoricalFeedbackDefinition: + required: + - details + - name + - type + type: object + allOf: + - $ref: '#/components/schemas/Feedback' + - type: object + properties: + details: + $ref: '#/components/schemas/CategoricalFeedbackDetail' + created_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + last_updated_by: + type: string + readOnly: true + CategoricalFeedbackDetail: + required: + - categories + type: object + properties: + categories: + type: object + additionalProperties: + type: number + format: double + Feedback: + required: + - details + - name + - type + type: object + properties: + id: + type: string + format: uuid + readOnly: true + name: + type: string + details: + type: object + createdAt: + type: string + format: date-time + readOnly: true + createdBy: + type: string + readOnly: true + lastUpdatedAt: + type: string + format: date-time + readOnly: true + lastUpdatedBy: + type: string + readOnly: true + type: + type: string + enum: + - numerical + - categorical + discriminator: + propertyName: type + mapping: + numerical: '#/components/schemas/NumericalFeedbackDefinition' + categorical: '#/components/schemas/CategoricalFeedbackDefinition' + NumericalFeedbackDefinition: + required: + - details + - name + - type + type: object + allOf: + - $ref: '#/components/schemas/Feedback' + - type: object + properties: + details: + $ref: '#/components/schemas/NumericalFeedbackDetail' + created_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + last_updated_by: + type: string + readOnly: true + NumericalFeedbackDetail: + required: + - max + - min + type: object + properties: + max: + type: number + min: + type: number + CategoricalFeedbackDefinition_Create: + required: + - details + - name + - type + type: object + allOf: + - $ref: '#/components/schemas/Feedback_Create' + - type: object + properties: + details: + $ref: '#/components/schemas/CategoricalFeedbackDetail_Create' + CategoricalFeedbackDetail_Create: + required: + - categories + type: object + properties: + categories: + type: object + additionalProperties: + type: number + format: double + Feedback_Create: + required: + - details + - name + - type + type: object + properties: + id: + type: string + format: uuid + readOnly: true + name: + type: string + details: + type: object + type: + type: string + enum: + - numerical + - categorical + discriminator: + propertyName: type + mapping: + numerical: '#/components/schemas/NumericalFeedbackDefinition_Create' + categorical: '#/components/schemas/CategoricalFeedbackDefinition_Create' + NumericalFeedbackDefinition_Create: + required: + - details + - name + - type + type: object + allOf: + - $ref: '#/components/schemas/Feedback_Create' + - type: object + properties: + details: + $ref: '#/components/schemas/NumericalFeedbackDetail_Create' + NumericalFeedbackDetail_Create: + required: + - max + - min + type: object + properties: + max: + type: number + min: + type: number + CategoricalFeedbackDefinition_Public: + required: + - details + - name + - type + type: object + allOf: + - $ref: '#/components/schemas/Feedback_Public' + - type: object + properties: + details: + $ref: '#/components/schemas/CategoricalFeedbackDetail_Public' + created_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + last_updated_by: + type: string + readOnly: true + CategoricalFeedbackDetail_Public: + required: + - categories + type: object + properties: + categories: + type: object + additionalProperties: + type: number + format: double + FeedbackDefinitionPage_Public: + type: object + properties: + page: + type: integer + format: int32 + size: + type: integer + format: int32 + total: + type: integer + format: int64 + content: + type: array + items: + $ref: '#/components/schemas/FeedbackObject_Public' + FeedbackObject_Public: + required: + - details + - name + - type + type: object + properties: + id: + type: string + format: uuid + readOnly: true + name: + type: string + details: + type: object + createdAt: + type: string + format: date-time + readOnly: true + createdBy: + type: string + readOnly: true + lastUpdatedAt: + type: string + format: date-time + readOnly: true + lastUpdatedBy: + type: string + readOnly: true + type: + type: string + enum: + - numerical + - categorical + discriminator: + propertyName: type + mapping: + numerical: '#/components/schemas/NumericalFeedbackDefinition_Public' + categorical: '#/components/schemas/CategoricalFeedbackDefinition_Public' + NumericalFeedbackDefinition_Public: + required: + - details + - name + - type + type: object + allOf: + - $ref: '#/components/schemas/Feedback_Public' + - type: object + properties: + details: + $ref: '#/components/schemas/NumericalFeedbackDetail_Public' + created_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + last_updated_by: + type: string + readOnly: true + NumericalFeedbackDetail_Public: + required: + - max + - min + type: object + properties: + max: + type: number + min: + type: number + Feedback_Public: + required: + - details + - name + - type + type: object + properties: + id: + type: string + format: uuid + readOnly: true + name: + type: string + details: + type: object + createdAt: + type: string + format: date-time + readOnly: true + createdBy: + type: string + readOnly: true + lastUpdatedAt: + type: string + format: date-time + readOnly: true + lastUpdatedBy: + type: string + readOnly: true + type: + type: string + enum: + - numerical + - categorical + discriminator: + propertyName: type + mapping: + numerical: '#/components/schemas/NumericalFeedbackDefinition_Public' + categorical: '#/components/schemas/CategoricalFeedbackDefinition_Public' + CategoricalFeedbackDefinition_Update: + required: + - details + - name + - type + type: object + allOf: + - $ref: '#/components/schemas/Feedback_Update' + - type: object + properties: + details: + $ref: '#/components/schemas/CategoricalFeedbackDetail_Update' + CategoricalFeedbackDetail_Update: + required: + - categories + type: object + properties: + categories: + type: object + additionalProperties: + type: number + format: double + Feedback_Update: + required: + - details + - name + - type + type: object + properties: + id: + type: string + format: uuid + readOnly: true + name: + type: string + details: + type: object + type: + type: string + enum: + - numerical + - categorical + discriminator: + propertyName: type + mapping: + numerical: '#/components/schemas/NumericalFeedbackDefinition_Update' + categorical: '#/components/schemas/CategoricalFeedbackDefinition_Update' + NumericalFeedbackDefinition_Update: + required: + - details + - name + - type + type: object + allOf: + - $ref: '#/components/schemas/Feedback_Update' + - type: object + properties: + details: + $ref: '#/components/schemas/NumericalFeedbackDetail_Update' + NumericalFeedbackDetail_Update: + required: + - max + - min + type: object + properties: + max: + type: number + min: + type: number + Project: + required: + - name + type: object + properties: + id: + type: string + format: uuid + readOnly: true + name: + type: string + description: + pattern: ^(?!\s*$).+ + type: string + created_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + last_updated_by: + type: string + readOnly: true + Project_Write: + required: + - name + type: object + properties: + name: + type: string + description: + pattern: ^(?!\s*$).+ + type: string + ProjectPage_Public: + type: object + properties: + page: + type: integer + format: int32 + size: + type: integer + format: int32 + total: + type: integer + format: int64 + content: + type: array + items: + $ref: '#/components/schemas/Project_Public' + Project_Public: + required: + - name + type: object + properties: + id: + type: string + format: uuid + readOnly: true + name: + type: string + description: + pattern: ^(?!\s*$).+ + type: string + created_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + last_updated_by: + type: string + readOnly: true + ProjectUpdate: + type: object + properties: + name: + pattern: ^(?!\s*$).+ + type: string + description: + pattern: ^(?!\s*$).+ + type: string + Span: + required: + - name + - start_time + - trace_id + - type + type: object + properties: + id: + type: string + format: uuid + project_name: + pattern: ^(?!\s*$).+ + type: string + description: "If null, the default project is used" + project_id: + type: string + format: uuid + readOnly: true + trace_id: + type: string + format: uuid + parent_span_id: + type: string + format: uuid + name: + type: string + type: + type: string + enum: + - general + - tool + - llm + start_time: + type: string + format: date-time + end_time: + type: string + format: date-time + input: + $ref: '#/components/schemas/JsonNode' + output: + $ref: '#/components/schemas/JsonNode' + metadata: + $ref: '#/components/schemas/JsonNode' + tags: + uniqueItems: true + type: array + items: + type: string + usage: + type: object + additionalProperties: + type: integer + format: int32 + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + feedback_scores: + type: array + readOnly: true + items: + $ref: '#/components/schemas/FeedbackScore' + Span_Write: + required: + - name + - start_time + - trace_id + - type + type: object + properties: + id: + type: string + format: uuid + project_name: + pattern: ^(?!\s*$).+ + type: string + description: "If null, the default project is used" + trace_id: + type: string + format: uuid + parent_span_id: + type: string + format: uuid + name: + type: string + type: + type: string + enum: + - general + - tool + - llm + start_time: + type: string + format: date-time + end_time: + type: string + format: date-time + input: + $ref: '#/components/schemas/JsonNode_Write' + output: + $ref: '#/components/schemas/JsonNode_Write' + metadata: + $ref: '#/components/schemas/JsonNode_Write' + tags: + uniqueItems: true + type: array + items: + type: string + usage: + type: object + additionalProperties: + type: integer + format: int32 + SpanBatch: + required: + - spans + type: object + properties: + spans: + maxItems: 1000 + minItems: 1 + type: array + items: + $ref: '#/components/schemas/Span' + SpanBatch_Write: + required: + - spans + type: object + properties: + spans: + maxItems: 1000 + minItems: 1 + type: array + items: + $ref: '#/components/schemas/Span_Write' + DeleteFeedbackScore: + required: + - name + type: object + properties: + name: + type: string + FeedbackScore_Public: + required: + - name + - source + - value + type: object + properties: + name: + type: string + category_name: + type: string + value: + maximum: 999999999.999999999 + exclusiveMaximum: false + minimum: -999999999.999999999 + exclusiveMinimum: false + type: number + reason: + type: string + source: + type: string + enum: + - ui + - sdk + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + readOnly: true + Span_Public: + required: + - name + - start_time + - trace_id + - type + type: object + properties: + id: + type: string + format: uuid + project_id: + type: string + format: uuid + readOnly: true + trace_id: + type: string + format: uuid + parent_span_id: + type: string + format: uuid + name: + type: string + type: + type: string + enum: + - general + - tool + - llm + start_time: + type: string + format: date-time + end_time: + type: string + format: date-time + input: + $ref: '#/components/schemas/JsonNode_Public' + output: + $ref: '#/components/schemas/JsonNode_Public' + metadata: + $ref: '#/components/schemas/JsonNode_Public' + tags: + uniqueItems: true + type: array + items: + type: string + usage: + type: object + additionalProperties: + type: integer + format: int32 + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + feedback_scores: + type: array + readOnly: true + items: + $ref: '#/components/schemas/FeedbackScore_Public' + SpanPage_Public: + type: object + properties: + page: + type: integer + format: int32 + size: + type: integer + format: int32 + total: + type: integer + format: int64 + content: + type: array + items: + $ref: '#/components/schemas/Span_Public' + FeedbackScoreBatch: + required: + - scores + type: object + properties: + scores: + maxItems: 1000 + minItems: 1 + type: array + items: + $ref: '#/components/schemas/FeedbackScoreBatchItem' + FeedbackScoreBatchItem: + required: + - id + - name + - source + - value + type: object + properties: + id: + type: string + format: uuid + project_name: + pattern: ^(?!\s*$).+ + type: string + description: "If null, the default project is used" + name: + type: string + category_name: + type: string + value: + maximum: 999999999.999999999 + exclusiveMaximum: false + minimum: -999999999.999999999 + exclusiveMinimum: false + type: number + reason: + type: string + source: + type: string + enum: + - ui + - sdk + SpanUpdate: + required: + - trace_id + type: object + properties: + project_name: + pattern: ^(?!\s*$).+ + type: string + description: "If null and project_id not specified, Default Project is assumed" + project_id: + type: string + description: "If null and project_name not specified, Default Project is\ + \ assumed" + format: uuid + trace_id: + type: string + format: uuid + parent_span_id: + type: string + format: uuid + end_time: + type: string + format: date-time + input: + $ref: '#/components/schemas/JsonNode' + output: + $ref: '#/components/schemas/JsonNode' + metadata: + $ref: '#/components/schemas/JsonNode' + tags: + uniqueItems: true + type: array + items: + type: string + usage: + type: object + additionalProperties: + type: integer + format: int32 + Trace: + required: + - name + - start_time + type: object + properties: + id: + type: string + format: uuid + project_name: + pattern: ^(?!\s*$).+ + type: string + description: "If null, the default project is used" + project_id: + type: string + format: uuid + readOnly: true + name: + type: string + start_time: + type: string + format: date-time + end_time: + type: string + format: date-time + input: + $ref: '#/components/schemas/JsonNode' + output: + $ref: '#/components/schemas/JsonNode' + metadata: + $ref: '#/components/schemas/JsonNode' + tags: + uniqueItems: true + type: array + items: + type: string + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + feedback_scores: + type: array + readOnly: true + items: + $ref: '#/components/schemas/FeedbackScore' + Trace_Write: + required: + - name + - start_time + type: object + properties: + id: + type: string + format: uuid + project_name: + pattern: ^(?!\s*$).+ + type: string + description: "If null, the default project is used" + name: + type: string + start_time: + type: string + format: date-time + end_time: + type: string + format: date-time + input: + $ref: '#/components/schemas/JsonNode_Write' + output: + $ref: '#/components/schemas/JsonNode_Write' + metadata: + $ref: '#/components/schemas/JsonNode_Write' + tags: + uniqueItems: true + type: array + items: + type: string + TraceBatch: + required: + - traces + type: object + properties: + traces: + maxItems: 1000 + minItems: 1 + type: array + items: + $ref: '#/components/schemas/Trace' + TraceBatch_Write: + required: + - traces + type: object + properties: + traces: + maxItems: 1000 + minItems: 1 + type: array + items: + $ref: '#/components/schemas/Trace_Write' + TracesDelete: + required: + - ids + type: object + properties: + ids: + maxItems: 1000 + minItems: 1 + uniqueItems: true + type: array + items: + type: string + format: uuid + Trace_Public: + required: + - name + - start_time + type: object + properties: + id: + type: string + format: uuid + project_id: + type: string + format: uuid + readOnly: true + name: + type: string + start_time: + type: string + format: date-time + end_time: + type: string + format: date-time + input: + $ref: '#/components/schemas/JsonNode_Public' + output: + $ref: '#/components/schemas/JsonNode_Public' + metadata: + $ref: '#/components/schemas/JsonNode_Public' + tags: + uniqueItems: true + type: array + items: + type: string + created_at: + type: string + format: date-time + readOnly: true + last_updated_at: + type: string + format: date-time + readOnly: true + created_by: + type: string + readOnly: true + last_updated_by: + type: string + readOnly: true + feedback_scores: + type: array + readOnly: true + items: + $ref: '#/components/schemas/FeedbackScore_Public' + TracePage_Public: + type: object + properties: + page: + type: integer + format: int32 + size: + type: integer + format: int32 + total: + type: integer + format: int64 + content: + type: array + items: + $ref: '#/components/schemas/Trace_Public' + TraceUpdate: + type: object + properties: + project_name: + pattern: ^(?!\s*$).+ + type: string + description: "If null and project_id not specified, Default Project is assumed" + project_id: + type: string + description: "If null and project_name not specified, Default Project is\ + \ assumed" + format: uuid + end_time: + type: string + format: date-time + input: + $ref: '#/components/schemas/JsonNode' + output: + $ref: '#/components/schemas/JsonNode' + metadata: + $ref: '#/components/schemas/JsonNode' + tags: + uniqueItems: true + type: array + items: + type: string diff --git a/sdks/python/code_generation/fern/readme.md b/sdks/python/code_generation/fern/readme.md index 36e73d89c9..1ec9addc06 100644 --- a/sdks/python/code_generation/fern/readme.md +++ b/sdks/python/code_generation/fern/readme.md @@ -5,6 +5,8 @@ How to generate new client code for communication with Opik backend 1. Execute the ./build_and_run.sh script from the root of repository 2. Go to http://localhost:3003/ (URL for backend API specification) 3. Download openapi specification file - `openapi.yaml` -4. Put this file into `code_generation/fern/openapi/openapi.yaml` +4. Put this file into `code_generation/fern/openapi/openapi.yaml` (overwrite it).
+ Note that this path contains the version of the schema from which the code in `src/opik/rest_api` for the SDK was generated. Therefore, it might not be the latest version of the schema. 5. Run `fern generate` from inside `code_generation/fern` folder. This will generate a python code inside the directory called `sdks` near the `fern` one. 7. Replace content of `src/opik/rest_api` with the python package inside `sdks` (there will be few nested directories, navigate until you find python files) +8. Run `pre-commit run --all-files` to format code diff --git a/sdks/python/src/opik/api_objects/experiment/experiment.py b/sdks/python/src/opik/api_objects/experiment/experiment.py index c27b0b9dae..0c60834f81 100644 --- a/sdks/python/src/opik/api_objects/experiment/experiment.py +++ b/sdks/python/src/opik/api_objects/experiment/experiment.py @@ -1,5 +1,5 @@ import logging -from typing import List +from typing import List, Optional from opik.rest_api import client as rest_api_client from opik.rest_api.types import experiment_item as rest_experiment_item @@ -14,7 +14,7 @@ class Experiment: def __init__( self, id: str, - name: str, + name: Optional[str], dataset_name: str, rest_client: rest_api_client.OpikApi, ) -> None: diff --git a/sdks/python/src/opik/api_objects/opik_client.py b/sdks/python/src/opik/api_objects/opik_client.py index 80fe2e4e85..f3e8215e9c 100644 --- a/sdks/python/src/opik/api_objects/opik_client.py +++ b/sdks/python/src/opik/api_objects/opik_client.py @@ -364,10 +364,21 @@ def create_dataset( def create_experiment( self, - name: str, dataset_name: str, + name: Optional[str] = None, experiment_config: Optional[Dict[str, Any]] = None, ) -> experiment.Experiment: + """ + Creates a new experiment using the given dataset name and optional parameters. + + Args: + dataset_name (str): The name of the dataset to associate with the experiment. + name (Optional[str]): The optional name for the experiment. If None, a generated name will be used. + experiment_config (Optional[Dict[str, Any]]): Optional experiment configuration parameters. Must be a dictionary if provided. + + Returns: + experiment.Experiment: The newly created experiment object. + """ id = helpers.generate_id() if isinstance(experiment_config, Mapping): diff --git a/sdks/python/src/opik/cli.py b/sdks/python/src/opik/cli.py index 6340b6f6b4..96fd79d47d 100644 --- a/sdks/python/src/opik/cli.py +++ b/sdks/python/src/opik/cli.py @@ -29,7 +29,8 @@ def cli() -> None: ) def configure(use_local: bool) -> None: """ - Create a configuration file for the Opik Python SDK, if a configuration file already exists, it will be overwritten. This is also available as a function in the Python SDK. + Create a configuration file for the Opik Python SDK, if a configuration file already exists, it will be overwritten. + This is also available as a function in the Python SDK. """ if use_local: opik_configure.configure(use_local=True, force=True) diff --git a/sdks/python/src/opik/evaluation/evaluation_result.py b/sdks/python/src/opik/evaluation/evaluation_result.py index 09a8108645..5d61041177 100644 --- a/sdks/python/src/opik/evaluation/evaluation_result.py +++ b/sdks/python/src/opik/evaluation/evaluation_result.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Optional import dataclasses @@ -8,5 +8,5 @@ @dataclasses.dataclass class EvaluationResult: experiment_id: str - experiment_name: str + experiment_name: Optional[str] test_results: List[test_result.TestResult] diff --git a/sdks/python/src/opik/evaluation/evaluator.py b/sdks/python/src/opik/evaluation/evaluator.py index 2c4636a955..0f2b1fa738 100644 --- a/sdks/python/src/opik/evaluation/evaluator.py +++ b/sdks/python/src/opik/evaluation/evaluator.py @@ -14,7 +14,7 @@ def evaluate( dataset: dataset.Dataset, task: LLMTask, scoring_metrics: List[base_metric.BaseMetric], - experiment_name: str, + experiment_name: Optional[str] = None, experiment_config: Optional[Dict[str, Any]] = None, verbose: int = 1, task_threads: int = 16, @@ -28,7 +28,8 @@ def evaluate( task: A callable object that takes DatasetItem as input and returns dictionary which will later be used for scoring - experiment_name: The name of the experiment associated with evaluation run + experiment_name: The name of the experiment associated with evaluation run. + If None, a generated name will be used. experiment_config: The dictionary with parameters that describe experiment diff --git a/sdks/python/src/opik/rest_api/__init__.py b/sdks/python/src/opik/rest_api/__init__.py index bbb10002dc..10855a8dc7 100644 --- a/sdks/python/src/opik/rest_api/__init__.py +++ b/sdks/python/src/opik/rest_api/__init__.py @@ -86,9 +86,11 @@ SpanWriteType, Trace, TraceBatch, + TraceCountResponse, TracePagePublic, TracePublic, TraceWrite, + WorkspaceTraceCount, ) from .errors import ( BadRequestError, @@ -97,7 +99,15 @@ NotImplementedError, UnprocessableEntityError, ) -from . import datasets, experiments, feedback_definitions, projects, spans, traces +from . import ( + datasets, + experiments, + feedback_definitions, + projects, + spans, + system_usage, + traces, +) from .environment import OpikApiEnvironment from .feedback_definitions import FindFeedbackDefinitionsRequestType from .spans import GetSpansByProjectRequestType @@ -195,14 +205,17 @@ "SpanWriteType", "Trace", "TraceBatch", + "TraceCountResponse", "TracePagePublic", "TracePublic", "TraceWrite", "UnprocessableEntityError", + "WorkspaceTraceCount", "datasets", "experiments", "feedback_definitions", "projects", "spans", + "system_usage", "traces", ] diff --git a/sdks/python/src/opik/rest_api/client.py b/sdks/python/src/opik/rest_api/client.py index c371dbcd86..d6332dd500 100644 --- a/sdks/python/src/opik/rest_api/client.py +++ b/sdks/python/src/opik/rest_api/client.py @@ -18,6 +18,7 @@ ) from .projects.client import AsyncProjectsClient, ProjectsClient from .spans.client import AsyncSpansClient, SpansClient +from .system_usage.client import AsyncSystemUsageClient, SystemUsageClient from .traces.client import AsyncTracesClient, TracesClient @@ -78,6 +79,7 @@ def __init__( else httpx.Client(timeout=_defaulted_timeout), timeout=_defaulted_timeout, ) + self.system_usage = SystemUsageClient(client_wrapper=self._client_wrapper) self.datasets = DatasetsClient(client_wrapper=self._client_wrapper) self.experiments = ExperimentsClient(client_wrapper=self._client_wrapper) self.feedback_definitions = FeedbackDefinitionsClient( @@ -177,6 +179,7 @@ def __init__( else httpx.AsyncClient(timeout=_defaulted_timeout), timeout=_defaulted_timeout, ) + self.system_usage = AsyncSystemUsageClient(client_wrapper=self._client_wrapper) self.datasets = AsyncDatasetsClient(client_wrapper=self._client_wrapper) self.experiments = AsyncExperimentsClient(client_wrapper=self._client_wrapper) self.feedback_definitions = AsyncFeedbackDefinitionsClient( diff --git a/sdks/python/src/opik/rest_api/experiments/client.py b/sdks/python/src/opik/rest_api/experiments/client.py index 2414e45592..2d0cdb4d04 100644 --- a/sdks/python/src/opik/rest_api/experiments/client.py +++ b/sdks/python/src/opik/rest_api/experiments/client.py @@ -78,8 +78,8 @@ def create_experiment( self, *, dataset_name: str, - name: str, id: typing.Optional[str] = OMIT, + name: typing.Optional[str] = OMIT, metadata: typing.Optional[JsonNodeWrite] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> None: @@ -90,10 +90,10 @@ def create_experiment( ---------- dataset_name : str - name : str - id : typing.Optional[str] + name : typing.Optional[str] + metadata : typing.Optional[JsonNodeWrite] request_options : typing.Optional[RequestOptions] @@ -110,7 +110,6 @@ def create_experiment( client = OpikApi() client.experiments.create_experiment( dataset_name="dataset_name", - name="name", ) """ _response = self._client_wrapper.httpx_client.request( @@ -316,6 +315,66 @@ def get_experiment_item_by_id( raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) + def stream_experiment_items( + self, + *, + experiment_name: str, + limit: typing.Optional[int] = OMIT, + last_retrieved_id: typing.Optional[str] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> typing.Iterator[bytes]: + """ + Stream experiment items + + Parameters + ---------- + experiment_name : str + + limit : typing.Optional[int] + + last_retrieved_id : typing.Optional[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Yields + ------ + typing.Iterator[bytes] + Experiment items stream or error during process + + Examples + -------- + from Opik.client import OpikApi + + client = OpikApi() + client.experiments.stream_experiment_items( + experiment_name="string", + limit=1, + last_retrieved_id="string", + ) + """ + with self._client_wrapper.httpx_client.stream( + "v1/private/experiments/items/stream", + method="POST", + json={ + "experiment_name": experiment_name, + "limit": limit, + "last_retrieved_id": last_retrieved_id, + }, + request_options=request_options, + omit=OMIT, + ) as _response: + try: + if 200 <= _response.status_code < 300: + for _chunk in _response.iter_bytes(): + yield _chunk + return + _response.read() + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + class AsyncExperimentsClient: def __init__(self, *, client_wrapper: AsyncClientWrapper): @@ -384,8 +443,8 @@ async def create_experiment( self, *, dataset_name: str, - name: str, id: typing.Optional[str] = OMIT, + name: typing.Optional[str] = OMIT, metadata: typing.Optional[JsonNodeWrite] = OMIT, request_options: typing.Optional[RequestOptions] = None, ) -> None: @@ -396,10 +455,10 @@ async def create_experiment( ---------- dataset_name : str - name : str - id : typing.Optional[str] + name : typing.Optional[str] + metadata : typing.Optional[JsonNodeWrite] request_options : typing.Optional[RequestOptions] @@ -421,7 +480,6 @@ async def create_experiment( async def main() -> None: await client.experiments.create_experiment( dataset_name="dataset_name", - name="name", ) @@ -661,3 +719,71 @@ async def main() -> None: except JSONDecodeError: raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) + + async def stream_experiment_items( + self, + *, + experiment_name: str, + limit: typing.Optional[int] = OMIT, + last_retrieved_id: typing.Optional[str] = OMIT, + request_options: typing.Optional[RequestOptions] = None, + ) -> typing.AsyncIterator[bytes]: + """ + Stream experiment items + + Parameters + ---------- + experiment_name : str + + limit : typing.Optional[int] + + last_retrieved_id : typing.Optional[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Yields + ------ + typing.AsyncIterator[bytes] + Experiment items stream or error during process + + Examples + -------- + import asyncio + + from Opik.client import AsyncOpikApi + + client = AsyncOpikApi() + + + async def main() -> None: + await client.experiments.stream_experiment_items( + experiment_name="string", + limit=1, + last_retrieved_id="string", + ) + + + asyncio.run(main()) + """ + async with self._client_wrapper.httpx_client.stream( + "v1/private/experiments/items/stream", + method="POST", + json={ + "experiment_name": experiment_name, + "limit": limit, + "last_retrieved_id": last_retrieved_id, + }, + request_options=request_options, + omit=OMIT, + ) as _response: + try: + if 200 <= _response.status_code < 300: + async for _chunk in _response.aiter_bytes(): + yield _chunk + return + await _response.aread() + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) diff --git a/sdks/python/src/opik/rest_api/system_usage/__init__.py b/sdks/python/src/opik/rest_api/system_usage/__init__.py new file mode 100644 index 0000000000..67a41b2742 --- /dev/null +++ b/sdks/python/src/opik/rest_api/system_usage/__init__.py @@ -0,0 +1 @@ +# This file was auto-generated by Fern from our API Definition. diff --git a/sdks/python/src/opik/rest_api/system_usage/client.py b/sdks/python/src/opik/rest_api/system_usage/client.py new file mode 100644 index 0000000000..aac45ee0ed --- /dev/null +++ b/sdks/python/src/opik/rest_api/system_usage/client.py @@ -0,0 +1,100 @@ +# This file was auto-generated by Fern from our API Definition. + +import typing +from json.decoder import JSONDecodeError + +from ..core.api_error import ApiError +from ..core.client_wrapper import AsyncClientWrapper, SyncClientWrapper +from ..core.pydantic_utilities import pydantic_v1 +from ..core.request_options import RequestOptions +from ..types.trace_count_response import TraceCountResponse + + +class SystemUsageClient: + def __init__(self, *, client_wrapper: SyncClientWrapper): + self._client_wrapper = client_wrapper + + def get_traces_count_for_workspaces( + self, *, request_options: typing.Optional[RequestOptions] = None + ) -> TraceCountResponse: + """ + Get traces count on previous day for all available workspaces + + Parameters + ---------- + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + TraceCountResponse + TraceCountResponse resource + + Examples + -------- + from Opik.client import OpikApi + + client = OpikApi() + client.system_usage.get_traces_count_for_workspaces() + """ + _response = self._client_wrapper.httpx_client.request( + "v1/internal/usage/workspace-trace-counts", + method="GET", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return pydantic_v1.parse_obj_as(TraceCountResponse, _response.json()) # type: ignore + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + + +class AsyncSystemUsageClient: + def __init__(self, *, client_wrapper: AsyncClientWrapper): + self._client_wrapper = client_wrapper + + async def get_traces_count_for_workspaces( + self, *, request_options: typing.Optional[RequestOptions] = None + ) -> TraceCountResponse: + """ + Get traces count on previous day for all available workspaces + + Parameters + ---------- + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + TraceCountResponse + TraceCountResponse resource + + Examples + -------- + import asyncio + + from Opik.client import AsyncOpikApi + + client = AsyncOpikApi() + + + async def main() -> None: + await client.system_usage.get_traces_count_for_workspaces() + + + asyncio.run(main()) + """ + _response = await self._client_wrapper.httpx_client.request( + "v1/internal/usage/workspace-trace-counts", + method="GET", + request_options=request_options, + ) + try: + if 200 <= _response.status_code < 300: + return pydantic_v1.parse_obj_as(TraceCountResponse, _response.json()) # type: ignore + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) diff --git a/sdks/python/src/opik/rest_api/traces/client.py b/sdks/python/src/opik/rest_api/traces/client.py index 70bc18144c..20f16b0a37 100644 --- a/sdks/python/src/opik/rest_api/traces/client.py +++ b/sdks/python/src/opik/rest_api/traces/client.py @@ -509,6 +509,50 @@ def delete_trace_feedback_score( raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) + def delete_traces( + self, + *, + ids: typing.Sequence[str], + request_options: typing.Optional[RequestOptions] = None, + ) -> None: + """ + Delete traces + + Parameters + ---------- + ids : typing.Sequence[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + None + + Examples + -------- + from Opik.client import OpikApi + + client = OpikApi() + client.traces.delete_traces( + ids=["ids"], + ) + """ + _response = self._client_wrapper.httpx_client.request( + "v1/private/traces/delete", + method="POST", + json={"ids": ids}, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + return + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + def score_batch_of_traces( self, *, @@ -1112,6 +1156,58 @@ async def main() -> None: raise ApiError(status_code=_response.status_code, body=_response.text) raise ApiError(status_code=_response.status_code, body=_response_json) + async def delete_traces( + self, + *, + ids: typing.Sequence[str], + request_options: typing.Optional[RequestOptions] = None, + ) -> None: + """ + Delete traces + + Parameters + ---------- + ids : typing.Sequence[str] + + request_options : typing.Optional[RequestOptions] + Request-specific configuration. + + Returns + ------- + None + + Examples + -------- + import asyncio + + from Opik.client import AsyncOpikApi + + client = AsyncOpikApi() + + + async def main() -> None: + await client.traces.delete_traces( + ids=["ids"], + ) + + + asyncio.run(main()) + """ + _response = await self._client_wrapper.httpx_client.request( + "v1/private/traces/delete", + method="POST", + json={"ids": ids}, + request_options=request_options, + omit=OMIT, + ) + try: + if 200 <= _response.status_code < 300: + return + _response_json = _response.json() + except JSONDecodeError: + raise ApiError(status_code=_response.status_code, body=_response.text) + raise ApiError(status_code=_response.status_code, body=_response_json) + async def score_batch_of_traces( self, *, diff --git a/sdks/python/src/opik/rest_api/types/__init__.py b/sdks/python/src/opik/rest_api/types/__init__.py index 3ca6fd5529..daa2acc3df 100644 --- a/sdks/python/src/opik/rest_api/types/__init__.py +++ b/sdks/python/src/opik/rest_api/types/__init__.py @@ -91,9 +91,11 @@ from .span_write_type import SpanWriteType from .trace import Trace from .trace_batch import TraceBatch +from .trace_count_response import TraceCountResponse from .trace_page_public import TracePagePublic from .trace_public import TracePublic from .trace_write import TraceWrite +from .workspace_trace_count import WorkspaceTraceCount __all__ = [ "CategoricalFeedbackDefinition", @@ -181,7 +183,9 @@ "SpanWriteType", "Trace", "TraceBatch", + "TraceCountResponse", "TracePagePublic", "TracePublic", "TraceWrite", + "WorkspaceTraceCount", ] diff --git a/sdks/python/src/opik/rest_api/types/experiment.py b/sdks/python/src/opik/rest_api/types/experiment.py index 8d1af9d446..00d14a6821 100644 --- a/sdks/python/src/opik/rest_api/types/experiment.py +++ b/sdks/python/src/opik/rest_api/types/experiment.py @@ -13,7 +13,7 @@ class Experiment(pydantic_v1.BaseModel): id: typing.Optional[str] = None dataset_name: str dataset_id: typing.Optional[str] = None - name: str + name: typing.Optional[str] = None metadata: typing.Optional[JsonNode] = None feedback_scores: typing.Optional[typing.List[FeedbackScoreAverage]] = None trace_count: typing.Optional[int] = None diff --git a/sdks/python/src/opik/rest_api/types/experiment_public.py b/sdks/python/src/opik/rest_api/types/experiment_public.py index ccee386288..747044e95b 100644 --- a/sdks/python/src/opik/rest_api/types/experiment_public.py +++ b/sdks/python/src/opik/rest_api/types/experiment_public.py @@ -13,7 +13,7 @@ class ExperimentPublic(pydantic_v1.BaseModel): id: typing.Optional[str] = None dataset_name: str dataset_id: typing.Optional[str] = None - name: str + name: typing.Optional[str] = None metadata: typing.Optional[JsonNodePublic] = None feedback_scores: typing.Optional[typing.List[FeedbackScoreAveragePublic]] = None trace_count: typing.Optional[int] = None diff --git a/sdks/python/src/opik/rest_api/types/trace_count_response.py b/sdks/python/src/opik/rest_api/types/trace_count_response.py new file mode 100644 index 0000000000..0bb8d1f0b9 --- /dev/null +++ b/sdks/python/src/opik/rest_api/types/trace_count_response.py @@ -0,0 +1,43 @@ +# This file was auto-generated by Fern from our API Definition. + +import datetime as dt +import typing + +from ..core.datetime_utils import serialize_datetime +from ..core.pydantic_utilities import deep_union_pydantic_dicts, pydantic_v1 +from .workspace_trace_count import WorkspaceTraceCount + + +class TraceCountResponse(pydantic_v1.BaseModel): + workspaces_traces_count: typing.Optional[typing.List[WorkspaceTraceCount]] = None + + def json(self, **kwargs: typing.Any) -> str: + kwargs_with_defaults: typing.Any = { + "by_alias": True, + "exclude_unset": True, + **kwargs, + } + return super().json(**kwargs_with_defaults) + + def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]: + kwargs_with_defaults_exclude_unset: typing.Any = { + "by_alias": True, + "exclude_unset": True, + **kwargs, + } + kwargs_with_defaults_exclude_none: typing.Any = { + "by_alias": True, + "exclude_none": True, + **kwargs, + } + + return deep_union_pydantic_dicts( + super().dict(**kwargs_with_defaults_exclude_unset), + super().dict(**kwargs_with_defaults_exclude_none), + ) + + class Config: + frozen = True + smart_union = True + extra = pydantic_v1.Extra.allow + json_encoders = {dt.datetime: serialize_datetime} diff --git a/sdks/python/src/opik/rest_api/types/workspace_trace_count.py b/sdks/python/src/opik/rest_api/types/workspace_trace_count.py new file mode 100644 index 0000000000..bbd8957d21 --- /dev/null +++ b/sdks/python/src/opik/rest_api/types/workspace_trace_count.py @@ -0,0 +1,43 @@ +# This file was auto-generated by Fern from our API Definition. + +import datetime as dt +import typing + +from ..core.datetime_utils import serialize_datetime +from ..core.pydantic_utilities import deep_union_pydantic_dicts, pydantic_v1 + + +class WorkspaceTraceCount(pydantic_v1.BaseModel): + workspace: typing.Optional[str] = None + trace_count: typing.Optional[int] = None + + def json(self, **kwargs: typing.Any) -> str: + kwargs_with_defaults: typing.Any = { + "by_alias": True, + "exclude_unset": True, + **kwargs, + } + return super().json(**kwargs_with_defaults) + + def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]: + kwargs_with_defaults_exclude_unset: typing.Any = { + "by_alias": True, + "exclude_unset": True, + **kwargs, + } + kwargs_with_defaults_exclude_none: typing.Any = { + "by_alias": True, + "exclude_none": True, + **kwargs, + } + + return deep_union_pydantic_dicts( + super().dict(**kwargs_with_defaults_exclude_unset), + super().dict(**kwargs_with_defaults_exclude_none), + ) + + class Config: + frozen = True + smart_union = True + extra = pydantic_v1.Extra.allow + json_encoders = {dt.datetime: serialize_datetime} diff --git a/sdks/python/tests/e2e/test_experiment.py b/sdks/python/tests/e2e/test_experiment.py index d167fb023f..7172f02969 100644 --- a/sdks/python/tests/e2e/test_experiment.py +++ b/sdks/python/tests/e2e/test_experiment.py @@ -1,5 +1,6 @@ import opik +from opik import synchronization from opik.api_objects.dataset import dataset_item from opik.evaluation import metrics from . import verifiers @@ -117,3 +118,56 @@ def task(item: dataset_item.DatasetItem): traces_amount=1, # one trace per dataset item feedback_scores_amount=1, # an average value of all Equals metric scores ) + + +def test_experiment_creation__name_can_be_omitted( + opik_client: opik.Opik, dataset_name: str, experiment_name: str +): + """ + We can send "None" as experiment_name and the backend will set it for us + """ + dataset = opik_client.create_dataset(dataset_name) + + dataset.insert( + [ + { + "input": {"question": "What is the of capital of France?"}, + "expected_output": {"output": "Paris"}, + }, + ] + ) + + def task(item: dataset_item.DatasetItem): + if item.input == {"question": "What is the of capital of France?"}: + return {"output": "Paris", "reference": item.expected_output["output"]} + + raise AssertionError( + f"Task received dataset item with an unexpected input: {item.input}" + ) + + equals_metric = metrics.Equals() + evaluation_result = opik.evaluate( + dataset=dataset, + task=task, + scoring_metrics=[equals_metric], + experiment_name=None, + ) + + opik.flush_tracker() + + experiment_id = evaluation_result.experiment_id + + if not synchronization.until( + lambda: ( + opik_client._rest_client.experiments.get_experiment_by_id(experiment_id) + is not None + ), + allow_errors=True, + ): + raise AssertionError(f"Failed to get experiment with id {experiment_id}.") + + experiment_content = opik_client._rest_client.experiments.get_experiment_by_id( + experiment_id + ) + + assert experiment_content.name is not None