Better performance and feedback

src/main/kotlin/astminer/common/model/ParsingModel.kt

@@ -11,12 +11,11 @@ abstract class Node(val originalToken: String?) {
     abstract val children: List<Node>
     abstract val parent: Node?
 
-    val normalizedToken: String = run {
+    val normalizedToken: String =
         originalToken?.let {
             val subtokens = splitToSubtokens(it)
             if (subtokens.isEmpty()) EMPTY_TOKEN else subtokens.joinToString(TOKEN_DELIMITER)
         } ?: EMPTY_TOKEN
-    }
 
     var technicalToken: String? = null
 

src/main/kotlin/astminer/common/model/ParsingResultModel.kt

@@ -7,14 +7,12 @@ import java.io.File
 import kotlin.concurrent.thread
 
 private val logger = KotlinLogging.logger("HandlerFactory")
-private const val NUM_OF_THREADS = 16
 
 interface ParsingResultFactory {
     fun parse(file: File): ParsingResult<out Node>
 
     fun <T> parseFiles(
         files: List<File>,
-        progressBar: ProgressBar? = null,
         action: (ParsingResult<out Node>) -> T
     ): List<T?> {
         val results = mutableListOf<T?>()
@@ -25,24 +23,25 @@ interface ParsingResultFactory {
                 logger.error(parsingException) { "Failed to parse file ${file.path}" }
                 results.add(null)
             }
-            progressBar?.step()
         }
         return results
     }
 
-    fun <T> parseFilesAsync(files: List<File>, action: (ParsingResult<out Node>) -> T): List<T?> {
+    fun <T> parseFilesInThreads(
+        files: List<File>,
+        numOfThreads: Int,
+        action: (ParsingResult<out Node>) -> T
+    ): List<T?> {
         val results = mutableListOf<T?>()
         val threads = mutableListOf<Thread>()
-        val progressBar = ProgressBar("Parsing progress:", files.size.toLong())
 
         synchronized(results) {
-            files.chunked(files.size / NUM_OF_THREADS + 1).filter { it.isNotEmpty() }
+            files.chunked(files.size / numOfThreads + 1).filter { it.isNotEmpty() }
                 .map { chunk ->
-                    threads.add(thread { results.addAll(parseFiles(chunk, progressBar, action)) })
+                    threads.add(thread { results.addAll(parseFiles(chunk, action)) })
                 }
         }
         threads.map { it.join() }
-        progressBar.close()
         return results
     }
 }
@@ -57,11 +56,9 @@ interface PreprocessingParsingResultFactory : ParsingResultFactory {
      */
     override fun <T> parseFiles(
         files: List<File>,
-        progressBar: ProgressBar?,
         action: (ParsingResult<out Node>) -> T
     ) =
         files.map { file ->
-            progressBar?.step()
             try {
                 val preprocessedFile = preprocess(file)
                 val result = action(parse(preprocessedFile))

src/main/kotlin/astminer/config/PipelineConfig.kt

@@ -14,5 +14,6 @@ data class PipelineConfig(
     val parser: ParserConfig,
     val filters: List<FilterConfig> = emptyList(),
     @SerialName("label") val labelExtractor: LabelExtractorConfig,
-    val storage: StorageConfig
+    val storage: StorageConfig,
+    val performance: PerformanceConfig = defaultPerformanceConfig
 )

src/main/kotlin/astminer/pipeline/Pipeline.kt

@@ -10,6 +10,7 @@ import astminer.parse.getParsingResultFactory
 import astminer.pipeline.branch.FilePipelineBranch
 import astminer.pipeline.branch.FunctionPipelineBranch
 import astminer.pipeline.branch.IllegalLabelExtractorException
+import me.tongfei.progressbar.ProgressBar
 import java.io.File
 
 /**
@@ -44,6 +45,7 @@ class Pipeline(private val config: PipelineConfig) {
      * Runs the pipeline that is defined in the [config].
      */
     fun run() {
+        println("Working in ${config.performance.numOfThreads}")
         for (language in config.parser.languages) {
             println("Parsing $language")
             val parsingResultFactory = getParsingResultFactory(language, config.parser.name)
@@ -52,15 +54,19 @@ class Pipeline(private val config: PipelineConfig) {
             val files = getProjectFilesWithExtension(inputDirectory, language.fileExtension)
             println("${files.size} files retrieved")
 
+            val progressBar = ProgressBar("", files.size.toLong())
+
             createStorage(language).use { storage ->
                 synchronized(storage) {
-                    parsingResultFactory.parseFilesAsync(files) { parseResult ->
+                    parsingResultFactory.parseFilesInThreads(files, config.performance.numOfThreads) { parseResult ->
                         for (labeledResult in branch.process(parseResult)) {
                             storage.store(labeledResult)
                         }
+                        progressBar.step()
                     }
                 }
             }
+            progressBar.close()
         }
         println("Done!")
     }