Render uncompilable test suite if it was successfully parsed (#370)

* feat: render parsable but uncompilable test cases & remove `TestGenerationData.compilableTestCases` field

`TestGenerationData.compilableTestCases` was only used to store test cases.
These stored tests were never used afterward due to `Report`'s `testCaseList`,
from which all tests were retrieved for render.

Authored-By: Vladislav Artiukhov

* feat: generate Javadoc comments for `Report` via AI

* fix: apply ktlint

* fix: `PromptManager`'s javadoc minor change

* feat: check that test suite is present for `NO_COMPILABLE_TEST_CASES_GENERATED` result code

* publish: core version `3.0.1`

* fix: apply ktlint

---------

Co-authored-by: Vladislav Artiukhov <[email protected]>

core/build.gradle.kts

@@ -42,7 +42,7 @@ publishing {
         create<MavenPublication>("maven") {
             groupId = group as String
             artifactId = "testspark-core"
-            version = "3.0.0"
+            version = "3.0.1"
             from(components["java"])
 
             artifact(tasks["sourcesJar"])

core/src/main/kotlin/org/jetbrains/research/testspark/core/data/Report.kt

@@ -1,17 +1,26 @@
 package org.jetbrains.research.testspark.core.data
 
 /**
- * Storage of generated tests. Implemented on the basis of org.evosuite.utils.CompactReport structure.
+ * Stores generated test cases and their coverage.
+ * Implemented on the basis of `org.evosuite.utils.CompactReport` structure.
+ *
+ * `Report`'s member fields were created based on the fields in
+ * `org.evosuite.utils.CompactReport` for easier transformation.
  */
 open class Report {
     // Fields were created based on the fields in org.evosuite.utils.CompactReport for easier transformation
-    var UUT: String = "" // Unit Under Test
+    /**
+     * Unit Under Test. This variable stores the name of the class or component that is being tested.
+     */
+    var UUT: String = ""
     var allCoveredLines: Set<Int> = setOf()
     var allUncoveredLines: Set<Int> = setOf()
     var testCaseList: HashMap<Int, TestCase> = hashMapOf()
 
     /**
-     * AllCoveredLines update
+     * Calculates the normalized report by updating the set of all covered lines.
+     *
+     * @return The normalized report.
      */
     fun normalized(): Report {
         allCoveredLines = testCaseList.values.map { it.coveredLines }.flatten().toSet()

core/src/main/kotlin/org/jetbrains/research/testspark/core/data/TestGenerationData.kt

@@ -1,7 +1,5 @@
 package org.jetbrains.research.testspark.core.data
 
-import org.jetbrains.research.testspark.core.test.data.TestCaseGeneratedByLLM
-
 data class TestGenerationData(
     // Result processing
     // Report object for each test case
@@ -23,14 +21,10 @@ data class TestGenerationData(
     // changing parameters with a large prompt
     var polyDepthReducing: Int = 0,
     var inputParamsDepthReducing: Int = 0,
-
-    // list of correct test cases during the incorrect compilation
-    val compilableTestCases: MutableSet<TestCaseGeneratedByLLM> = mutableSetOf(),
-
 ) {
 
     /**
-     * Cleaning all old data before new test generation.
+     * Cleaning all old data before a new test generation.
      */
     fun clear() {
         testGenerationResultList.clear()
@@ -42,6 +36,5 @@ data class TestGenerationData(
         otherInfo = ""
         polyDepthReducing = 0
         inputParamsDepthReducing = 0
-        compilableTestCases.clear()
     }
 }

core/src/main/kotlin/org/jetbrains/research/testspark/core/generation/llm/LLMWithFeedbackCycle.kt

@@ -27,28 +27,55 @@ enum class FeedbackCycleExecutionResult {
     SAVING_TEST_FILES_ISSUE,
 }
 
+/**
+ * Represents a response (result) of a feedback cycle.
+ *
+ * @param executionResult The result of executing the feedback cycle.
+ * @param generatedTestSuite The test suite generated by LLM.
+ * @param compilableTestCases The set of compilable test cases generated by LLM.
+ *
+ * @throws IllegalArgumentException if `executionResult` is [FeedbackCycleExecutionResult.OK] and `generatedTestSuite` is null.
+ */
 data class FeedbackResponse(
     val executionResult: FeedbackCycleExecutionResult,
     val generatedTestSuite: TestSuiteGeneratedByLLM?,
     val compilableTestCases: MutableSet<TestCaseGeneratedByLLM>,
 ) {
     init {
-        if (executionResult == FeedbackCycleExecutionResult.OK && generatedTestSuite == null) {
+        if ((executionResult == FeedbackCycleExecutionResult.OK || executionResult == FeedbackCycleExecutionResult.NO_COMPILABLE_TEST_CASES_GENERATED) &&
+            (generatedTestSuite == null)
+        ) {
             throw IllegalArgumentException("Test suite must be provided when FeedbackCycleExecutionResult is OK, got null")
-        } else if (executionResult != FeedbackCycleExecutionResult.OK && generatedTestSuite != null) {
-            throw IllegalArgumentException(
-                "Test suite must not be provided when FeedbackCycleExecutionResult is not OK, got $generatedTestSuite",
-            )
         }
     }
 }
 
+/**
+ * LLMWithFeedbackCycle class represents a feedback cycle for an LLM.
+ *
+ * @property report The `Report` instance used for storing generated tests.
+ * @property language The `SupportedLanguage` enum value representing the programming language used.
+ * @property initialPromptMessage The initial prompt message to start the feedback cycle.
+ * @property promptSizeReductionStrategy The `PromptSizeReductionStrategy` instance used for reducing the prompt size.
+ * @property testSuiteFilename The name of the file in which the test suite is saved in the result path.
+ * @property packageName The package name for the generated tests.
+ * @property resultPath The temporary path where all the generated tests and their Jacoco report are saved.
+ * @property buildPath All the directories where the compiled code of the project under test is saved.
+ * @property requestManager The `RequestManager` instance used for making LLM requests.
+ * @property testsAssembler The `TestsAssembler` instance used for assembling generated tests.
+ * @property testCompiler The `TestCompiler` instance used for compiling tests.
+ * @property testStorage The `TestsPersistentStorage` instance used for storing generated tests.
+ * @property testsPresenter The `TestsPresenter` instance used for presenting generated tests.
+ * @property indicator The `CustomProgressIndicator` instance used for tracking progress.
+ * @property requestsCountThreshold The threshold for the maximum number of requests in the feedback cycle.
+ * @property errorMonitor The `ErrorMonitor` instance used for monitoring errors.
+ */
 class LLMWithFeedbackCycle(
     private val report: Report,
     private val language: SupportedLanguage,
     private val initialPromptMessage: String,
     private val promptSizeReductionStrategy: PromptSizeReductionStrategy,
-    // filename in which the test suite is saved in result path
+    // filename in which the test suite is saved in the result path
     private val testSuiteFilename: String,
     private val packageName: String,
     // temp path where all the generated tests and their jacoco report are saved
@@ -98,6 +125,11 @@ class LLMWithFeedbackCycle(
 
             if (isLastIteration(requestsCount) && compilableTestCases.isEmpty()) {
                 executionResult = FeedbackCycleExecutionResult.NO_COMPILABLE_TEST_CASES_GENERATED
+                // record a report with parsable yet potentially
+                // non-compilable test cases stored in
+                // the generated test suite
+                // TODO: ensure generatedTestSuite is always non-null here
+                generatedTestSuite?.let { recordReport(report, it.testCases) }
                 break
             }
 
@@ -130,7 +162,9 @@ class LLMWithFeedbackCycle(
                     if (promptSizeReductionStrategy.isReductionPossible()) {
                         nextPromptMessage = promptSizeReductionStrategy.reduceSizeAndGeneratePrompt()
                         /**
-                         * Current attempt does not count as a failure since it was rejected due to the prompt size exceeding the threshold
+                         * The current attempt does not count as a failure
+                         * since it was rejected due to the prompt size
+                         * exceeding the threshold
                          */
                         requestsCount--
                         continue
@@ -245,14 +279,13 @@ class LLMWithFeedbackCycle(
 
             generatedTestsArePassing = true
 
-            for (index in testCases.indices) {
-                report.testCaseList[index] =
-                    TestCase(index, testCases[index].name, testCases[index].toString(), setOf())
-            }
+            recordReport(report, testCases)
         }
 
         // test suite must not be provided upon failed execution
-        if (executionResult != FeedbackCycleExecutionResult.OK) {
+        if (executionResult != FeedbackCycleExecutionResult.OK &&
+            executionResult != FeedbackCycleExecutionResult.NO_COMPILABLE_TEST_CASES_GENERATED
+        ) {
             generatedTestSuite = null
         }
 
@@ -263,5 +296,17 @@ class LLMWithFeedbackCycle(
         )
     }
 
+    /**
+     * Records the generated test cases in the given report.
+     *
+     * @param report The report object to store the test cases in.
+     * @param testCases The list of test cases generated by LLM.
+     */
+    private fun recordReport(report: Report, testCases: MutableList<TestCaseGeneratedByLLM>) {
+        for ((index, test) in testCases.withIndex()) {
+            report.testCaseList[index] = TestCase(index, test.name, test.toString(), setOf())
+        }
+    }
+
     private fun isLastIteration(requestsCount: Int): Boolean = requestsCount > requestsCountThreshold
 }

src/main/kotlin/org/jetbrains/research/testspark/tools/llm/generation/LLMProcessManager.kt

@@ -199,12 +199,14 @@ class LLMProcessManager(
         when (feedbackResponse.executionResult) {
             FeedbackCycleExecutionResult.OK -> {
                 log.info("Add ${feedbackResponse.compilableTestCases.size} compilable test cases into generatedTestsData")
-                // store compilable test cases
-                generatedTestsData.compilableTestCases.addAll(feedbackResponse.compilableTestCases)
             }
 
             FeedbackCycleExecutionResult.NO_COMPILABLE_TEST_CASES_GENERATED -> {
-                llmErrorManager.errorProcess(LLMMessagesBundle.get("invalidLLMResult"), project, errorMonitor)
+                if (feedbackResponse.generatedTestSuite != null) {
+                    llmErrorManager.warningProcess(LLMMessagesBundle.get("noCompilableTestCases"), project)
+                } else {
+                    llmErrorManager.errorProcess(LLMMessagesBundle.get("invalidLLMResult"), project, errorMonitor)
+                }
             }
 
             FeedbackCycleExecutionResult.CANCELED -> {

src/main/kotlin/org/jetbrains/research/testspark/tools/llm/generation/PromptManager.kt

@@ -282,7 +282,7 @@ class PromptManager(
      *
      * @param psiClass the PsiClassWrapper containing the method
      * @param lineNumber the line number within the file where the method is located
-     * @return the method descriptor as a String, or an empty string if no method is found
+     * @return the method descriptor as `String`, or an empty string if no method is found
      */
     private fun getMethodDescriptor(
         psiClass: PsiClassWrapper?,

src/main/resources/properties/llm/LLMMessages.properties

@@ -3,6 +3,7 @@ wrongToken=The provided token for Large Language Model is not correct. Please up
 emptyResponse=Large Language Model could not generate any tests for this class. Asking the AI assistance to fix its mistake.
 emptyBuildPath=Build path is Empty!\nPlease make sure that IDEA recognizes all of your module or enter proper build path in settings
 invalidLLMResult=The result is invalid or uses unknown commands due to randomness and lack of guarantees from Large Language Model.\nPlease try again
+noCompilableTestCases=LLM did not manage to make the test suite compilable. Manual effort may help to resolve the issues.
 compilationError=The test generated by Large Language Model is not compilable. Asking the AI assistance to fix its mistake.
 modifyWithLLMError=Large Language Model was unable to generate a new test. Please, check your request and try again.
 serverProblems=Large Language Model server is not responding