Integrate ShellSentry w/ AI

.github/workflows/ci.yml

@@ -43,7 +43,7 @@ jobs:
           touch ~/.gradle/gradle.properties
 
       - name: Bootstrap
-        run: ./gradlew bootstrap --stacktrace
+        run: ./gradlew bootstrap
 
       - name: Get changed files
         if: always() && github.event_name == 'pull_request'
@@ -89,7 +89,11 @@ jobs:
           path: build/skippy/**
 
       - name: Build and run tests
-        run: ./gradlew check ${{ env.APP_TARGET }} globalCiLint globalCiUnitTest --continue --quiet --stacktrace -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt
+        timeout-minutes: 30
+        env:
+          OPEN_AI_KEY: ${{ secrets.OPEN_AI_KEY }}
+        # TODO the check task undoes what Skippy provides, better to split it
+        run: ./scripts/shell_sentry.sh ./gradlew ${{ env.APP_TARGET }} checkSortDependencies spotlessCheck globalCiLint globalCiUnitTest -Pslack.avoidance.affectedProjectsFile=build/skippy/affected_projects.txt
 
       - name: Filter sarif
         if: always() && github.event_name == 'pull_request'

config/shell-sentry/config.json

@@ -0,0 +1,4 @@
+{
+  "version": 2,
+  "gradle_enterprise_server": "https://gradle.com"
+}

scripts/shell-sentry.main.kts

@@ -0,0 +1,213 @@
+#!/usr/bin/env kotlin
+@file:DependsOn("com.github.ajalt.clikt:clikt-jvm:4.2.0")
+@file:DependsOn("com.github.ajalt.mordant:mordant:2.1.0")
+@file:DependsOn("com.slack.cli:kotlin-cli-util:2.3.0-alpha01")
+@file:DependsOn("com.squareup.retrofit2:retrofit:2.9.0")
+@file:DependsOn("com.squareup.retrofit2:converter-moshi:2.9.0")
+@file:DependsOn("com.squareup.okhttp3:okhttp:5.0.0-alpha.11")
+@file:DependsOn("com.squareup.moshi:moshi:1.15.0")
+@file:DependsOn("com.squareup.moshi:moshi-kotlin:1.15.0")
+// // To silence this stupid log https://www.slf4j.org/codes.html#StaticLoggerBinder
+@file:DependsOn("org.slf4j:slf4j-nop:2.0.7")
+
+import com.github.ajalt.mordant.rendering.Whitespace
+import com.github.ajalt.mordant.table.table
+import com.github.ajalt.mordant.terminal.Terminal
+import com.squareup.moshi.Json
+import com.squareup.moshi.JsonClass
+import com.squareup.moshi.Moshi
+import com.squareup.moshi.kotlin.reflect.KotlinJsonAdapterFactory
+import java.nio.file.Path
+import kotlin.io.path.readText
+import kotlinx.coroutines.runBlocking
+import okhttp3.OkHttpClient
+import retrofit2.HttpException
+import retrofit2.Retrofit
+import retrofit2.converter.moshi.MoshiConverterFactory
+import retrofit2.create
+import retrofit2.http.Body
+import retrofit2.http.POST
+import slack.cli.shellsentry.AnalysisResult
+import slack.cli.shellsentry.NoStacktraceThrowable
+import slack.cli.shellsentry.RetrySignal
+import slack.cli.shellsentry.ShellSentry
+import slack.cli.shellsentry.ShellSentryExtension
+import kotlin.io.path.createTempFile
+import kotlin.io.path.writeText
+
+class GuessedIssue(message: String) : NoStacktraceThrowable(message)
+
+class AiClient(private val accessToken: String) {
+
+  private val moshi = Moshi.Builder().add(KotlinJsonAdapterFactory()).build()
+  private val client =
+    OkHttpClient.Builder()
+      .addInterceptor { chain ->
+        val request =
+          chain.request().newBuilder().addHeader("Authorization", "Bearer $accessToken").build()
+        chain.proceed(request)
+      }
+      .build()
+  private val api =
+    Retrofit.Builder()
+      .baseUrl("https://api.openai.com")
+      .client(client)
+      .addConverterFactory(MoshiConverterFactory.create(moshi))
+      .build()
+      .create<ChatGptApi>()
+
+  suspend fun analyze(content: String): AnalysisResult? {
+    return try {
+      val approxMaxChars = ChatGptApi.remainingTokens * ChatGptApi.AVG_TOKEN
+      val start = if (content.length > approxMaxChars) {
+        println("-- Truncating content due to token limit (max: ${ChatGptApi.remainingTokens}, content: ~${content.length / ChatGptApi.AVG_TOKEN}))")
+        println("-- Truncating to last $approxMaxChars chars (~${(approxMaxChars / content.length.toFloat()) * 100}%)}")
+        content.length - approxMaxChars
+      } else {
+        println("-- Not truncating content")
+        0
+      }
+      val analyzableContent = content.substring(start)
+      createTempFile().apply {
+        writeText(analyzableContent)
+        println("Analyzable content written to file://${toAbsolutePath()}")
+      }
+      val prompt = "${ChatGptApi.ANALYSIS_PROMPT}\n\n$analyzableContent"
+      val response = api.analyze(prompt)
+      val rawJson = response.choices.first().message.content.trim()
+      val parsed =
+        moshi.adapter(ChoiceAnalysis::class.java).fromJson(rawJson)
+          ?: error("Could not parse: $rawJson")
+      AnalysisResult(
+        parsed.message,
+        parsed.explanation,
+        if (parsed.retry) RetrySignal.RetryImmediately else RetrySignal.Ack,
+        parsed.confidence
+      ) { message ->
+        GuessedIssue(message)
+      }
+    } catch (t: Throwable) {
+      if (t is HttpException) {
+        System.err.println(
+          """
+          HTTP Error: ${t.code()}
+          Message: ${t.message()}
+          ${t.response()?.errorBody()?.string()}
+        """.trimIndent()
+        )
+      } else {
+        System.err.println(t.stackTraceToString())
+      }
+      null
+    }
+  }
+
+  interface ChatGptApi {
+    @POST("/v1/chat/completions")
+    suspend fun completion(@Body request: CompletionRequest): CompletionResponse
+
+    companion object {
+      private const val MAX_TOKENS = 4096
+      const val AVG_TOKEN = 4
+
+      val ANALYSIS_PROMPT =
+        """
+          Given the following console output, please provide a diagnosis in a raw JSON object format:
+
+          - "message": A broad single-line description of the error without specifying exact details, suitable for crash reporter grouping.
+          - "explanation": A detailed, multi-line message explaining the error and suggesting a solution.
+          - "retry": A boolean value (true/false) indicating whether a retry could potentially resolve the CI issue.
+          - "confidence": An integer value between 1-100 representing your confidence about the accuracy of your error identification.
+        """
+          .trimIndent()
+
+      private val promptTokens = ANALYSIS_PROMPT.length / AVG_TOKEN
+
+      // Because these are all sorta fuzzy guesses, we want to leave some buffer
+      private const val TOKEN_BUFFER = 700
+      val remainingTokens = MAX_TOKENS - promptTokens - TOKEN_BUFFER
+    }
+  }
+
+
+  private suspend fun ChatGptApi.analyze(content: String) =
+    completion(CompletionRequest(messages = listOf(Message(content = content))))
+
+  @JsonClass(generateAdapter = false)
+  data class CompletionRequest(val model: String = "gpt-3.5-turbo", val messages: List<Message>)
+
+  @JsonClass(generateAdapter = false)
+  data class Message(val role: String = "user", val content: String)
+
+  @JsonClass(generateAdapter = false)
+  data class CompletionResponse(
+    val id: String,
+    @Json(name = "object") val objectType: String,
+    val created: Long,
+    val model: String,
+    val choices: List<Choice>,
+    val usage: Usage
+  ) {
+
+    @JsonClass(generateAdapter = false)
+    data class Choice(
+      val index: Int,
+      @Json(name = "finish_reason") val finishReason: String,
+      val message: Message,
+    )
+
+    @JsonClass(generateAdapter = false)
+    data class Usage(
+      @Json(name = "prompt_tokens") val promptTokens: Int,
+      @Json(name = "completion_tokens") val completionTokens: Int,
+      @Json(name = "total_tokens") val totalTokens: Int,
+    )
+  }
+
+  @JsonClass(generateAdapter = false)
+  data class ChoiceAnalysis(
+    val message: String,
+    val explanation: String,
+    val retry: Boolean,
+    val confidence: Int,
+  )
+}
+
+class AiExtension(private val aiClient: AiClient) : ShellSentryExtension {
+  override fun check(
+    command: String,
+    exitCode: Int,
+    isAfterRetry: Boolean,
+    consoleOutput: Path
+  ): AnalysisResult? {
+    println("-- AIExtension: Checking")
+    val result = runBlocking { aiClient.analyze(consoleOutput.readText()) } ?: return null
+
+    println("\n")
+    val t = Terminal()
+    t.println(table {
+      captionTop("Analysis Result")
+      body { row("Message", result.message) }
+      body {
+        row("Explanation", result.explanation) {
+          this.whitespace = Whitespace.PRE_WRAP
+        }
+      }
+      body {
+        row(
+          "Retry?",
+          "${result.retrySignal != RetrySignal.Ack && result.retrySignal != RetrySignal.Unknown}"
+        )
+      }
+      body { row("Confidence", "${result.confidence}%") }
+    })
+    println("\n")
+
+    return result
+  }
+}
+
+val openAiKey: String? = System.getenv("OPEN_AI_KEY")
+val extensions = if (openAiKey != null) listOf(AiExtension(AiClient(openAiKey))) else emptyList()
+
+ShellSentry.create(args, ::println).copy(extensions = extensions).exec()

scripts/shell_sentry.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+./scripts/shell-sentry.main.kts --debug --verbose --config config/shell-sentry/config.json -- "$*"