Merge remote-tracking branch 'origin/main' into dspy-integration-docs

.github/workflows/backend_formatting_check.yml

@@ -0,0 +1,36 @@
+name: "Backend Formatting Check"
+run-name: "Backend Formatting Check on ${{ github.ref_name }} by @${{ github.actor }}"
+
+on:
+  pull_request:
+    paths:
+      - "apps/opik-backend/**/*.java"
+  push:
+    branches:
+      - "main"
+    paths:
+      - "apps/opik-backend/**/*.java"
+
+  workflow_dispatch:
+
+jobs:
+  run-backend-formatting-check:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: apps/opik-backend/
+    steps:
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 1
+
+      - name: Set up JDK 21
+        uses: actions/setup-java@v4
+        with:
+          java-version: "21"
+          distribution: "corretto"
+          cache: maven
+
+      - name: Run Formatting Check for backend
+        run: mvn clean spotless:check

.github/workflows/documentation_codeblock_tests.yml

@@ -0,0 +1,82 @@
+name: Documentation - Test codeblocks
+on:
+  workflow_dispatch:
+    inputs:
+      install_opik:
+        description: 'Enable opik installation from source files'
+        required: false
+        default: 'false'
+        type: choice
+        options:
+          - 'false'
+          - 'true'
+  pull_request:
+    paths:
+      - 'apps/opik-documentation/documentation/docs/*.md'
+      - 'apps/opik-documentation/documentation/docs/*.mdx'
+      - 'apps/opik-documentation/documentation/docs/**/*.md'
+      - 'apps/opik-documentation/documentation/docs/**/*.mdx'
+
+jobs:
+  collect_test_paths:
+    runs-on: ubuntu-latest
+    outputs:
+      test_paths: ${{ steps.paths.outputs.paths }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # Fetch all history for git diff
+
+      - id: paths
+        working-directory: apps/opik-documentation/documentation
+        run: |
+          # Get list of changed files in docs directory
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+            # For pull requests, compare with base branch
+            echo "paths=$(
+              git diff --name-only origin/${{ github.base_ref }} |
+              grep -E '^apps/opik-documentation/documentation/docs/.*\.(md|mdx)$' |
+              sed 's|apps/opik-documentation/documentation/||' |
+              jq -R -s -c 'split("\n")[:-1]'
+            )" >> $GITHUB_OUTPUT
+          else
+            # For manual runs and scheduled runs, check all files
+            echo "paths=$(
+              (
+                ls -d docs/*/ 2>/dev/null;
+                find docs -maxdepth 1 -type f -name "*.md" -o -name "*.mdx"
+              ) | jq -R -s -c 'split("\n")[:-1]'
+            )" >> $GITHUB_OUTPUT
+          fi
+
+  test:
+    needs: collect_test_paths
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        path: ${{ fromJson(needs.collect_test_paths.outputs.test_paths) }}
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Install dependencies
+        working-directory: apps/opik-documentation/documentation
+        run: |
+          python -m pip install --upgrade pip
+          pip install pytest
+          pip install -r requirements.txt
+          if [ "${{ github.event.inputs.install_opik }}" = "true" ]; then
+            pip install -e .
+          fi
+          
+      - name: Run tests
+        working-directory: apps/opik-documentation/documentation
+        run: |
+          if [ -n "${{ matrix.path }}" ]; then
+            pytest ${{ matrix.path }} -v --suppress-no-test-exit-code
+          fi

.gitignore

@@ -9,6 +9,9 @@
 target/
 **/dependency-reduced-pom.xml
 
+# BE related
+/apps/opik-backend/redoc/openapi.yaml
+
 # FE related
 /apps/opik-frontend/dist
 /apps/opik-frontend/build

CONTRIBUTING.md

@@ -283,6 +283,23 @@ Replace `{project.pom.version}` with the version of the project in the pom file.
 
 Once the backend is running, you can access the Opik API at `http://localhost:8080`.
 
+#### Formatting the code
+
+Before submitting a PR, please ensure that your code is formatted correctly.
+Run the following command to automatically format your code:
+
+```bash
+mvn spotless:apply
+```
+
+Our CI will check that the code is formatted correctly and will fail if it is not by running the following command:
+
+```bash
+mvn spotless:check
+```
+
+#### Testing the backend
+
 Before submitting a PR, please ensure that your code passes the test suite:
 
 ```bash

apps/opik-backend/config.yml

@@ -224,8 +224,22 @@ llmProviderClient:
   # Default: 60s
   # Description: Write timeout for LLM providers
   writeTimeout: ${LLM_PROVIDER_CLIENT_WRITE_TIMEOUT:-60s}
+  # Default: false
+  # Description: Whether or not to log requests
+  logRequests: ${LLM_PROVIDER_CLIENT_LOG_REQUESTS:-false}
+  # Default: false
+  # Description: Whether or not to log responses
+  logResponses: ${LLM_PROVIDER_CLIENT_LOG_RESPONSES:-false}
   # Configuration for OpenAI client
   openAiClient:
     # Default:
     # Description: OpenAI API URL
     url: ${LLM_PROVIDER_OPENAI_URL:-}
+  # Configuration for Anthropic client
+  anthropicClient:
+    # Default: https://api.anthropic.com/v1/
+    # Description: Anthropic API URL
+    url: ${LLM_PROVIDER_ANTHROPIC_URL:-https://api.anthropic.com/v1/}
+    # Default: 2023-06-01
+    # Description: Anthropic API version https://docs.anthropic.com/en/api/versioning
+    version: ${LLM_PROVIDER_ANTHROPIC_VERSION:-'2023-06-01'}

apps/opik-backend/pom.xml

@@ -211,6 +211,10 @@
             <groupId>dev.langchain4j</groupId>
             <artifactId>langchain4j-open-ai</artifactId>
         </dependency>
+        <dependency>
+            <groupId>dev.langchain4j</groupId>
+            <artifactId>langchain4j-anthropic</artifactId>
+        </dependency>
 
         <!-- Test -->
 

apps/opik-backend/src/main/java/com/comet/opik/OpikApplication.java

@@ -82,6 +82,7 @@ public void initialize(Bootstrap<OpikConfiguration> bootstrap) {
     @Override
     public void run(OpikConfiguration configuration, Environment environment) {
         EncryptionUtils.setConfig(configuration);
+
         // Resources
         var jersey = environment.jersey();
 

apps/opik-backend/src/main/java/com/comet/opik/api/AutomationRuleEvaluator.java

@@ -42,9 +42,11 @@ public static final class AutomationRuleEvaluatorLlmAsJudge extends AutomationRu
         @Schema(accessMode = Schema.AccessMode.READ_WRITE)
         JsonNode code;
 
-        @ConstructorProperties({"id", "projectId", "name", "samplingRate", "code", "createdAt", "createdBy", "lastUpdatedAt", "lastUpdatedBy"})
-        public AutomationRuleEvaluatorLlmAsJudge(UUID id, UUID projectId, @NotBlank String name, float samplingRate, @NotNull JsonNode code,
-                                           Instant createdAt, String createdBy, Instant lastUpdatedAt, String lastUpdatedBy) {
+        @ConstructorProperties({"id", "projectId", "name", "samplingRate", "code", "createdAt", "createdBy",
+                "lastUpdatedAt", "lastUpdatedBy"})
+        public AutomationRuleEvaluatorLlmAsJudge(UUID id, UUID projectId, @NotBlank String name, float samplingRate,
+                @NotNull JsonNode code,
+                Instant createdAt, String createdBy, Instant lastUpdatedAt, String lastUpdatedBy) {
             super(id, projectId, name, samplingRate, createdAt, createdBy, lastUpdatedAt, lastUpdatedBy);
             this.code = code;
         }
@@ -60,8 +62,7 @@ public AutomationRuleEvaluatorType type() {
     UUID id;
 
     @JsonView({View.Public.class, View.Write.class})
-    @NotNull
-    UUID projectId;
+    @NotNull UUID projectId;
 
     @JsonView({View.Public.class, View.Write.class})
     @Schema(accessMode = Schema.AccessMode.READ_WRITE)
@@ -100,17 +101,21 @@ public AutomationRuleAction getAction() {
     }
 
     public static class View {
-        public static class Write {}
-        public static class Public {}
+        public static class Write {
+        }
+        public static class Public {
+        }
     }
 
     @Builder(toBuilder = true)
     public record AutomationRuleEvaluatorPage(
-            @JsonView({View.Public.class}) int page,
+            @JsonView( {
+                    View.Public.class}) int page,
             @JsonView({View.Public.class}) int size,
             @JsonView({View.Public.class}) long total,
             @JsonView({View.Public.class}) List<AutomationRuleEvaluatorLlmAsJudge> content)
-            implements Page<AutomationRuleEvaluatorLlmAsJudge>{
+            implements
+                Page<AutomationRuleEvaluatorLlmAsJudge>{
 
         public static AutomationRuleEvaluator.AutomationRuleEvaluatorPage empty(int page) {
             return new AutomationRuleEvaluator.AutomationRuleEvaluatorPage(page, 0, 0, List.of());

apps/opik-backend/src/main/java/com/comet/opik/api/ExperimentItemSearchCriteria.java

@@ -0,0 +1,13 @@
+package com.comet.opik.api;
+
+import lombok.Builder;
+
+import java.util.UUID;
+
+@Builder(toBuilder = true)
+public record ExperimentItemSearchCriteria(
+        String experimentName,
+        Integer limit,
+        UUID lastRetrievedId,
+        boolean truncate) {
+}

apps/opik-backend/src/main/java/com/comet/opik/api/ExperimentItemStreamRequest.java

@@ -3,9 +3,11 @@
 import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.databind.PropertyNamingStrategies;
 import com.fasterxml.jackson.databind.annotation.JsonNaming;
+import io.swagger.v3.oas.annotations.media.Schema;
 import jakarta.validation.constraints.Max;
 import jakarta.validation.constraints.Min;
 import jakarta.validation.constraints.NotBlank;
+import jakarta.ws.rs.DefaultValue;
 import lombok.Builder;
 
 import java.util.UUID;
@@ -16,7 +18,8 @@
 public record ExperimentItemStreamRequest(
         @NotBlank String experimentName,
         @Min(1) @Max(2000) Integer limit,
-        UUID lastRetrievedId) {
+        UUID lastRetrievedId,
+        @Schema(description = "Truncate image included in either input, output or metadata", defaultValue = "true") @DefaultValue("true") boolean truncate) {
 
     @Override
     public Integer limit() {

apps/opik-backend/src/main/java/com/comet/opik/api/LlmProvider.java

@@ -10,8 +10,8 @@
 @Getter
 @RequiredArgsConstructor
 public enum LlmProvider {
-
-    OPEN_AI("openai");
+    OPEN_AI("openai"),
+    ANTHROPIC("anthropic");
 
     @JsonValue
     private final String value;

apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/AutomationRuleEvaluatorsResource.java

@@ -59,14 +59,15 @@ public class AutomationRuleEvaluatorsResource {
     })
     @JsonView(AutomationRuleEvaluator.View.Public.class)
     public Response find(@PathParam("projectId") UUID projectId,
-                         @QueryParam("name") String name,
-                         @QueryParam("page") @Min(1) @DefaultValue("1") int page,
-                         @QueryParam("size") @Min(1) @DefaultValue("10") int size) {
+            @QueryParam("name") String name,
+            @QueryParam("page") @Min(1) @DefaultValue("1") int page,
+            @QueryParam("size") @Min(1) @DefaultValue("10") int size) {
 
         String workspaceId = requestContext.get().getWorkspaceId();
         log.info("Looking for automated evaluators for project id '{}' on workspaceId '{}' (page {})", projectId,
                 workspaceId, page);
-        Page<AutomationRuleEvaluator.AutomationRuleEvaluatorLlmAsJudge> definitionPage = service.find(projectId, workspaceId, name, page, size);
+        Page<AutomationRuleEvaluator.AutomationRuleEvaluatorLlmAsJudge> definitionPage = service.find(projectId,
+                workspaceId, name, page, size);
         log.info("Found {} automated evaluators for project id '{}' on workspaceId '{}' (page {}, total {})",
                 definitionPage.size(), projectId, workspaceId, page, definitionPage.total());
 
@@ -99,8 +100,7 @@ public Response getEvaluator(@PathParam("projectId") UUID projectId, @PathParam(
     })
     @RateLimited
     public Response createEvaluator(
-            @RequestBody(content = @Content(schema = @Schema(implementation = AutomationRuleEvaluator.class)))
-            @JsonView(AutomationRuleEvaluator.View.Write.class) @NotNull @Valid AutomationRuleEvaluator<?> evaluator,
+            @RequestBody(content = @Content(schema = @Schema(implementation = AutomationRuleEvaluator.class))) @JsonView(AutomationRuleEvaluator.View.Write.class) @NotNull @Valid AutomationRuleEvaluator<?> evaluator,
             @Context UriInfo uriInfo) {
 
         String workspaceId = requestContext.get().getWorkspaceId();
@@ -148,12 +148,15 @@ public Response updateEvaluator(@PathParam("id") UUID id,
             @ApiResponse(responseCode = "204", description = "No Content"),
     })
     public Response deleteEvaluators(
-            @NotNull @RequestBody(content = @Content(schema = @Schema(implementation = BatchDelete.class))) @Valid BatchDelete batchDelete, @PathParam("projectId") UUID projectId) {
+            @NotNull @RequestBody(content = @Content(schema = @Schema(implementation = BatchDelete.class))) @Valid BatchDelete batchDelete,
+            @PathParam("projectId") UUID projectId) {
         String workspaceId = requestContext.get().getWorkspaceId();
-        log.info("Deleting automation rule evaluators by ids, count '{}', on workspace_id '{}'", batchDelete.ids().size(),
+        log.info("Deleting automation rule evaluators by ids, count '{}', on workspace_id '{}'",
+                batchDelete.ids().size(),
                 workspaceId);
         service.delete(batchDelete.ids(), projectId, workspaceId);
-        log.info("Deleted automation rule evaluators by ids, count '{}', on workspace_id '{}'", batchDelete.ids().size(),
+        log.info("Deleted automation rule evaluators by ids, count '{}', on workspace_id '{}'",
+                batchDelete.ids().size(),
                 workspaceId);
         return Response.noContent().build();
     }

apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/ChatCompletionsResource.java

@@ -3,6 +3,7 @@
 import com.codahale.metrics.annotation.Timed;
 import com.comet.opik.domain.ChatCompletionService;
 import com.comet.opik.infrastructure.auth.RequestContext;
+import com.comet.opik.utils.ChunkedOutputHandlers;
 import dev.ai4j.openai4j.chat.ChatCompletionRequest;
 import dev.ai4j.openai4j.chat.ChatCompletionResponse;
 import io.dropwizard.jersey.errors.ErrorMessage;
@@ -26,6 +27,7 @@
 import lombok.NonNull;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
+import org.glassfish.jersey.server.ChunkedOutput;
 
 @Path("/v1/private/chat/completions")
 @Produces(MediaType.APPLICATION_JSON)
@@ -57,7 +59,10 @@ public Response create(
             log.info("Creating and streaming chat completions, workspaceId '{}', model '{}'",
                     workspaceId, request.model());
             type = MediaType.SERVER_SENT_EVENTS;
-            entity = chatCompletionService.createAndStreamResponse(request, workspaceId);
+            var chunkedOutput = new ChunkedOutput<String>(String.class, "\r\n");
+            chatCompletionService.createAndStreamResponse(request, workspaceId,
+                    new ChunkedOutputHandlers(chunkedOutput));
+            entity = chunkedOutput;
         } else {
             log.info("Creating chat completions, workspaceId '{}', model '{}'", workspaceId, request.model());
             type = MediaType.APPLICATION_JSON;

apps/opik-backend/src/main/java/com/comet/opik/api/resources/v1/priv/DatasetsResource.java

@@ -272,7 +272,7 @@ public Response getDatasetItems(
             @PathParam("id") UUID id,
             @QueryParam("page") @Min(1) @DefaultValue("1") int page,
             @QueryParam("size") @Min(1) @DefaultValue("10") int size,
-            @QueryParam("truncate") boolean truncate) {
+            @QueryParam("truncate") @Schema(description = "Truncate image included in either input, output or metadata") boolean truncate) {
 
         String workspaceId = requestContext.get().getWorkspaceId();
         log.info("Finding dataset items by id '{}', page '{}', size '{} on workspace_id '{}''", id, page, size,
@@ -373,7 +373,7 @@ public Response findDatasetItemsWithExperimentItems(
             @QueryParam("size") @Min(1) @DefaultValue("10") int size,
             @QueryParam("experiment_ids") @NotNull @NotBlank String experimentIdsQueryParam,
             @QueryParam("filters") String filters,
-            @QueryParam("truncate") boolean truncate) {
+            @QueryParam("truncate") @Schema(description = "Truncate image included in either input, output or metadata") boolean truncate) {
 
         var experimentIds = IdParamsValidator.getIds(experimentIdsQueryParam);