Node range

detekt.yaml

@@ -26,6 +26,8 @@ style:
     max: 5
   WildcardImport:
     active: false
+  UseDataClass:
+    allowVars: true
 
 formatting:
   autoCorrect: true

src/main/kotlin/astminer/common/SimpleNode.kt

@@ -0,0 +1,23 @@
+package astminer.common
+
+import astminer.common.model.Node
+import astminer.common.model.NodeRange
+
+/** Node simplest implementation **/
+class SimpleNode(
+    override val typeLabel: String,
+    override val children: MutableList<SimpleNode>,
+    override val parent: Node? = null,
+    override val range: NodeRange? = null,
+    token: String?
+) : Node(token) {
+    override fun removeChildrenOfType(typeLabel: String) {
+        children.removeIf { it.typeLabel == typeLabel }
+    }
+
+    override fun getChildrenOfType(typeLabel: String) = super.getChildrenOfType(typeLabel).map { it as SimpleNode }
+    override fun getChildOfType(typeLabel: String) = super.getChildOfType(typeLabel) as? SimpleNode
+
+    override fun preOrder() = super.preOrder().map { it as SimpleNode }
+    override fun postOrder() = super.postOrder().map { it as SimpleNode }
+}

src/main/kotlin/astminer/common/TokenNormalization.kt

@@ -0,0 +1,52 @@
+package astminer.common
+
+object TokenNormalization {
+    const val EMPTY_TOKEN = "<E>"
+    const val TOKEN_DELIMITER = "|"
+
+    private val newLineReg = "\\\\n".toRegex()
+    private val whitespaceReg = "//s+".toRegex()
+    private val quotesApostrophesCommasReg = "[\"',]".toRegex()
+    private val unicodeWeirdCharReg = "\\P{Print}".toRegex()
+    private val notALetterReg = "[^A-Za-z]".toRegex()
+
+    private val splitRegex = "(?<=[a-z])(?=[A-Z])|_|[0-9]|(?<=[A-Z])(?=[A-Z][a-z])|\\s+".toRegex()
+
+    fun normalizeToken(token: String?): String {
+        if (token == null) return EMPTY_TOKEN
+        val subTokens = splitToSubtokens(token)
+        return if (subTokens.isEmpty()) EMPTY_TOKEN else subTokens.joinToString(TOKEN_DELIMITER)
+    }
+
+    /**
+     * The function was adopted from the original code2vec implementation in order to match their behavior:
+     * https://github.com/tech-srl/code2vec/blob/master/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/Common.java
+     */
+    fun splitToSubtokens(token: String) = token
+        .trim()
+        .split(splitRegex)
+        .map { s -> normalizeSubToken(s, "") }
+        .filter { it.isNotEmpty() }
+        .toList()
+
+    /**
+     * The function was adopted from the original code2vec implementation in order to match their behavior:
+     * https://github.com/tech-srl/code2vec/blob/master/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/Common.java
+     */
+    fun normalizeSubToken(token: String, defaultToken: String): String {
+        val cleanToken = token.lowercase()
+            .replace(newLineReg, "") // escaped new line
+            .replace(whitespaceReg, "") // whitespaces
+            .replace(quotesApostrophesCommasReg, "") // quotes, apostrophes, commas
+            .replace(unicodeWeirdCharReg, "") // unicode weird characters
+
+        val stripped = cleanToken.replace(notALetterReg, "")
+
+        return stripped.ifEmpty {
+            val carefulStripped = cleanToken.replace(" ", "_")
+            carefulStripped.ifEmpty {
+                defaultToken
+            }
+        }
+    }
+}

src/main/kotlin/astminer/common/model/FunctionInfoModel.kt

@@ -20,7 +20,7 @@ interface NamedTree<T : Node> {
     val nameNode: T?
         get() = notImplemented("nameNode")
     val name: String?
-        get() = nameNode?.originalToken
+        get() = nameNode?.token?.original
     val root: T
         get() = notImplemented("root")
     val body: T?

src/main/kotlin/astminer/common/model/ParsingModel.kt

@@ -1,31 +1,22 @@
 package astminer.common.model
 
-import astminer.common.EMPTY_TOKEN
-import astminer.common.splitToSubtokens
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
 import java.io.File
 import java.io.InputStream
-import java.util.*
 
-abstract class Node(val originalToken: String?) {
+abstract class Node(originalToken: String?) {
     abstract val typeLabel: String
     abstract val children: List<Node>
     abstract val parent: Node?
-
-    val normalizedToken: String =
-        originalToken?.let {
-            val subtokens = splitToSubtokens(it)
-            if (subtokens.isEmpty()) EMPTY_TOKEN else subtokens.joinToString(TOKEN_DELIMITER)
-        } ?: EMPTY_TOKEN
-
-    var technicalToken: String? = null
-
-    val token: String
-        get() = technicalToken ?: normalizedToken
-
+    abstract val range: NodeRange?
     val metadata: MutableMap<String, Any> = HashMap()
+    val token = Token(originalToken)
+
     fun isLeaf() = children.isEmpty()
 
     override fun toString(): String = "$typeLabel : $token"
+
     fun prettyPrint(indent: Int = 0, indentSymbol: String = "--") {
         repeat(indent) { print(indentSymbol) }
         println(this)
@@ -52,30 +43,16 @@ abstract class Node(val originalToken: String?) {
 
     fun postOrderIterator(): Iterator<Node> = postOrder().listIterator()
     open fun postOrder(): List<Node> = mutableListOf<Node>().also { doTraversePostOrder(it) }
-
-    companion object {
-        const val TOKEN_DELIMITER = "|"
-    }
 }
 
-/** Node simplest implementation **/
-class SimpleNode(
-    override val typeLabel: String,
-    override val children: MutableList<SimpleNode>,
-    override val parent: Node?,
-    token: String?
-) : Node(token) {
-    override fun removeChildrenOfType(typeLabel: String) {
-        children.removeIf { it.typeLabel == typeLabel }
-    }
-
-    override fun getChildrenOfType(typeLabel: String) = super.getChildrenOfType(typeLabel).map { it as SimpleNode }
-    override fun getChildOfType(typeLabel: String) = super.getChildOfType(typeLabel) as? SimpleNode
-
-    override fun preOrder() = super.preOrder().map { it as SimpleNode }
-    override fun postOrder() = super.postOrder().map { it as SimpleNode }
+@Serializable
+data class NodeRange(val start: Position, val end: Position) {
+    override fun toString(): String = "[${start.line}, ${start.column}] - [${end.line}, ${end.column}]"
 }
 
+@Serializable
+data class Position(@SerialName("l") val line: Int, @SerialName("c") val column: Int)
+
 interface Parser<T : Node> {
     /**
      * Parse input stream into an AST.

src/main/kotlin/astminer/common/model/Token.kt

@@ -0,0 +1,14 @@
+package astminer.common.model
+
+import astminer.common.TokenNormalization
+
+class Token(val original: String?) {
+    val final: String
+        get() = technical ?: normalized
+
+    var technical: String? = null
+
+    val normalized = TokenNormalization.normalizeToken(original)
+
+    override fun toString(): String = final
+}

src/main/kotlin/astminer/config/StorageConfigs.kt

@@ -42,8 +42,16 @@ class DotAstStorageConfig : StorageConfig() {
  */
 @Serializable
 @SerialName("json AST")
-class JsonAstStorageConfig(private val withPaths: Boolean = false) : StorageConfig() {
-    override fun createStorage(outputDirectoryPath: String) = JsonAstStorage(outputDirectoryPath, withPaths)
+class JsonAstStorageConfig(
+    private val withPaths: Boolean = false,
+    private val withRanges: Boolean = false
+) : StorageConfig() {
+    override fun createStorage(outputDirectoryPath: String) =
+        JsonAstStorage(
+            outputDirectoryPath,
+            withPaths,
+            withRanges
+        )
 }
 
 /**

src/main/kotlin/astminer/featureextraction/TreeFeature.kt

@@ -57,7 +57,7 @@ object Tokens : TreeFeature<List<String>> {
 
     private fun findTokens(node: Node, tokensList: MutableList<String>): List<String> {
         node.children.forEach { findTokens(it, tokensList) }
-        tokensList.add(node.token)
+        tokensList.add(node.token.final)
         return tokensList
     }
 }

src/main/kotlin/astminer/filters/CommonFilters.kt

@@ -1,5 +1,6 @@
 package astminer.filters
 
+import astminer.common.TokenNormalization
 import astminer.common.model.*
 import astminer.featureextraction.NumberOfNodes
 
@@ -23,8 +24,8 @@ class TreeSizeFilter(private val minSize: Int = 0, private val maxSize: Int? = n
  * Filter that excludes trees that have more words than [maxWordsNumber] in any token of their node.
  */
 class WordsNumberFilter(private val maxWordsNumber: Int) : FunctionFilter, FileFilter {
-    private fun validateTree(root: Node) =
-        !root.preOrder().any { node -> node.token.split(Node.TOKEN_DELIMITER).size > maxWordsNumber }
+    private fun validateTree(root: Node) = root.preOrder()
+        .none { node -> node.token.final.split(TokenNormalization.TOKEN_DELIMITER).size > maxWordsNumber }
 
     override fun validate(functionInfo: FunctionInfo<out Node>) = validateTree(functionInfo.root)
 

src/main/kotlin/astminer/filters/FunctionFilters.kt

@@ -1,9 +1,9 @@
 package astminer.filters
 
+import astminer.common.TokenNormalization
 import astminer.common.model.FunctionFilter
 import astminer.common.model.FunctionInfo
 import astminer.common.model.Node
-import astminer.common.splitToSubtokens
 
 /**
  * Filter that excludes functions that have at least one of modifiers from the [excludeModifiers] list.
@@ -38,7 +38,7 @@ object ConstructorFilter : FunctionFilter {
 class FunctionNameWordsNumberFilter(private val maxWordsNumber: Int) : FunctionFilter {
     override fun validate(functionInfo: FunctionInfo<out Node>): Boolean {
         val name = functionInfo.name
-        return name != null && splitToSubtokens(name).size <= maxWordsNumber
+        return name != null && TokenNormalization.splitToSubtokens(name).size <= maxWordsNumber
     }
 }
 

src/main/kotlin/astminer/labelextractor/FunctionNameLabelExtractor.kt

@@ -14,13 +14,13 @@ object FunctionNameLabelExtractor : FunctionLabelExtractor {
     private const val RECURSIVE_CALL_TOKEN = "SELF"
 
     override fun process(functionInfo: FunctionInfo<out Node>): LabeledResult<out Node>? {
-        val normalizedName = functionInfo.nameNode?.normalizedToken ?: return null
+        val normalizedName = functionInfo.nameNode?.token?.normalized ?: return null
         functionInfo.root.preOrder().forEach { node ->
-            if (node.originalToken == functionInfo.nameNode?.originalToken) {
-                node.technicalToken = RECURSIVE_CALL_TOKEN
+            if (node.token.original == functionInfo.nameNode?.token?.original) {
+                node.token.technical = RECURSIVE_CALL_TOKEN
             }
         }
-        functionInfo.nameNode?.technicalToken = HIDDEN_METHOD_NAME_TOKEN
+        functionInfo.nameNode?.token?.technical = HIDDEN_METHOD_NAME_TOKEN
         return LabeledResult(functionInfo.root, normalizedName, functionInfo.qualifiedPath)
     }
 }

src/main/kotlin/astminer/parse/ForeignParser.kt

@@ -1,7 +1,8 @@
 package astminer.parse
 
+import astminer.common.SimpleNode
+import astminer.common.model.NodeRange
 import astminer.common.model.Parser
-import astminer.common.model.SimpleNode
 import astminer.config.FileExtension
 import astminer.config.ParserType
 import kotlinx.serialization.Serializable
@@ -23,17 +24,29 @@ import kotlin.io.path.createTempDirectory
  *      {
  *        "token": null,
  *        "nodeType": "i_am_root",
- *        "children": [1,2]
+ *        "children": [1,2],
+ *        "range" : {
+ *          "start" : { "l" : 0, "c" : 0 },
+ *          "end" : { "l" 1, "c" : 4 }
+ *        }
  *      },
  *      {
  *        "token": "Hello",
  *        "nodeType": "left_child",
  *        "children": []
+ *        "range" : {
+ *          "start" : { "l" : 0, "c": 0 },
+ *          "end" : { "l": 0, "c": 5 }
+ *        }
  *      },
  *      {
  *        "token": "World!",
  *        "nodeType": "right_child",
- *        "children": []
+ *        "children": [],
+ *        "range" : {
+ *          "start" : { "l" : 1, "c" : 0 },
+ *          "end" : { "l" : 1, "c" : 6 }
+ *        }
  *      }
  *    ]
  *  }
@@ -57,7 +70,14 @@ private fun launchScript(args: List<String>): String {
 
 private fun convertFromForeignTree(context: ForeignTree, rootId: Int = 0, parent: SimpleNode? = null): SimpleNode {
     val foreignNode = context.tree[rootId]
-    val node = SimpleNode(foreignNode.nodeType, mutableListOf(), parent, foreignNode.token)
+
+    val node = SimpleNode(
+        children = mutableListOf(),
+        parent = parent,
+        typeLabel = foreignNode.nodeType,
+        token = foreignNode.token,
+        range = foreignNode.range
+    )
     val children = foreignNode.children.map { convertFromForeignTree(context, it, node) }
     node.children.addAll(children)
     return node
@@ -67,7 +87,12 @@ private fun convertFromForeignTree(context: ForeignTree, rootId: Int = 0, parent
 private data class ForeignTree(val tree: List<ForeignNode>)
 
 @Serializable
-private data class ForeignNode(val token: String?, val nodeType: String, val children: List<Int>)
+private data class ForeignNode(
+    val token: String?,
+    val nodeType: String,
+    val range: NodeRange? = null,
+    val children: List<Int>
+)
 
 /** Use this parser to get a tree from external script.
  *  It uses `getTreeFromScript` and `getArguments` functions to generate