Node range

detekt.yaml

@@ -26,6 +26,8 @@ style:
     max: 5
   WildcardImport:
     active: false
+  UseDataClass:
+    allowVars: true
 
 formatting:
   autoCorrect: true

src/main/kotlin/astminer/common/SimpleNode.kt

@@ -0,0 +1,23 @@
+package astminer.common
+
+import astminer.common.model.Node
+import astminer.common.model.NodeRange
+
+/** Node simplest implementation **/
+class SimpleNode(
+    override val typeLabel: String,
+    override val children: MutableList<SimpleNode>,
+    override val parent: Node? = null,
+    override val range: NodeRange? = null,
+    token: String?
+) : Node(token) {
+    override fun removeChildrenOfType(typeLabel: String) {
+        children.removeIf { it.typeLabel == typeLabel }
+    }
+
+    override fun getChildrenOfType(typeLabel: String) = super.getChildrenOfType(typeLabel).map { it as SimpleNode }
+    override fun getChildOfType(typeLabel: String) = super.getChildOfType(typeLabel) as? SimpleNode
+
+    override fun preOrder() = super.preOrder().map { it as SimpleNode }
+    override fun postOrder() = super.postOrder().map { it as SimpleNode }
+}

src/main/kotlin/astminer/common/TreeUtil.kt → src/main/kotlin/astminer/common/TokenNormalization.kt

@@ -1,19 +1,35 @@
 package astminer.common
 
-const val EMPTY_TOKEN = "EMPTY"
+const val EMPTY_TOKEN = "<E>"
+const val TOKEN_DELIMITER = "|"
+
+/** Splits tokens in sub-tokens and normalizes them by removing new lines, whitespaces, quotes etc
+ * @see splitToSubtokens
+ * @see normalizeSubToken**/
+fun normalizeToken(token: String?): String {
+    if (token == null) return EMPTY_TOKEN
+    val subTokens = splitToSubtokens(token)
+    return if (subTokens.isEmpty()) EMPTY_TOKEN else subTokens.joinToString(TOKEN_DELIMITER)
+}
 
 /**
  * The function was adopted from the original code2vec implementation in order to match their behavior:
  * https://github.com/tech-srl/code2vec/blob/master/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/Common.java
  */
+fun splitToSubtokens(token: String) = token
+    .trim()
+    .split(splitRegex)
+    .map { s -> normalizeSubToken(s, "") }
+    .filter { it.isNotEmpty() }
+    .toList()
 
-val newLineReg = "\\\\n".toRegex()
-val whitespaceReg = "//s+".toRegex()
-val quotesApostrophesCommasReg = "[\"',]".toRegex()
-val unicodeWeirdCharReg = "\\P{Print}".toRegex()
-val notALetterReg = "[^A-Za-z]".toRegex()
+private val splitRegex = "(?<=[a-z])(?=[A-Z])|_|[0-9]|(?<=[A-Z])(?=[A-Z][a-z])|\\s+".toRegex()
 
-fun normalizeToken(token: String, defaultToken: String): String {
+/**
+ * The function was adopted from the original code2vec implementation in order to match their behavior:
+ * https://github.com/tech-srl/code2vec/blob/master/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/Common.java
+ */
+fun normalizeSubToken(token: String, defaultToken: String): String {
     val cleanToken = token.lowercase()
         .replace(newLineReg, "") // escaped new line
         .replace(whitespaceReg, "") // whitespaces
@@ -30,16 +46,8 @@ fun normalizeToken(token: String, defaultToken: String): String {
     }
 }
 
-/**
- * The function was adopted from the original code2vec implementation in order to match their behavior:
- * https://github.com/tech-srl/code2vec/blob/master/JavaExtractor/JPredict/src/main/java/JavaExtractor/Common/Common.java
- */
-
-val splitRegex = "(?<=[a-z])(?=[A-Z])|_|[0-9]|(?<=[A-Z])(?=[A-Z][a-z])|\\s+".toRegex()
-
-fun splitToSubtokens(token: String) = token
-    .trim()
-    .split(splitRegex)
-    .map { s -> normalizeToken(s, "") }
-    .filter { it.isNotEmpty() }
-    .toList()
+private val newLineReg = "\\\\n".toRegex()
+private val whitespaceReg = "//s+".toRegex()
+private val quotesApostrophesCommasReg = "[\"',]".toRegex()
+private val unicodeWeirdCharReg = "\\P{Print}".toRegex()
+private val notALetterReg = "[^A-Za-z]".toRegex()

src/main/kotlin/astminer/common/model/FunctionInfoModel.kt

@@ -20,7 +20,7 @@ interface NamedTree<T : Node> {
     val nameNode: T?
         get() = notImplemented("nameNode")
     val name: String?
-        get() = nameNode?.originalToken
+        get() = nameNode?.token?.original
     val root: T
         get() = notImplemented("root")
     val body: T?

src/main/kotlin/astminer/common/model/ParsingModel.kt

@@ -1,31 +1,22 @@
 package astminer.common.model
 
-import astminer.common.EMPTY_TOKEN
-import astminer.common.splitToSubtokens
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
 import java.io.File
 import java.io.InputStream
-import java.util.*
 
-abstract class Node(val originalToken: String?) {
+abstract class Node(originalToken: String?) {
     abstract val typeLabel: String
     abstract val children: List<Node>
     abstract val parent: Node?
-
-    val normalizedToken: String =
-        originalToken?.let {
-            val subtokens = splitToSubtokens(it)
-            if (subtokens.isEmpty()) EMPTY_TOKEN else subtokens.joinToString(TOKEN_DELIMITER)
-        } ?: EMPTY_TOKEN
-
-    var technicalToken: String? = null
-
-    val token: String
-        get() = technicalToken ?: normalizedToken
-
+    abstract val range: NodeRange?
     val metadata: MutableMap<String, Any> = HashMap()
+    val token = Token(originalToken)
+
     fun isLeaf() = children.isEmpty()
 
     override fun toString(): String = "$typeLabel : $token"
+
     fun prettyPrint(indent: Int = 0, indentSymbol: String = "--") {
         repeat(indent) { print(indentSymbol) }
         println(this)
@@ -52,30 +43,16 @@ abstract class Node(val originalToken: String?) {
 
     fun postOrderIterator(): Iterator<Node> = postOrder().listIterator()
     open fun postOrder(): List<Node> = mutableListOf<Node>().also { doTraversePostOrder(it) }
-
-    companion object {
-        const val TOKEN_DELIMITER = "|"
-    }
 }
 
-/** Node simplest implementation **/
-class SimpleNode(
-    override val typeLabel: String,
-    override val children: MutableList<SimpleNode>,
-    override val parent: Node?,
-    token: String?
-) : Node(token) {
-    override fun removeChildrenOfType(typeLabel: String) {
-        children.removeIf { it.typeLabel == typeLabel }
-    }
-
-    override fun getChildrenOfType(typeLabel: String) = super.getChildrenOfType(typeLabel).map { it as SimpleNode }
-    override fun getChildOfType(typeLabel: String) = super.getChildOfType(typeLabel) as? SimpleNode
-
-    override fun preOrder() = super.preOrder().map { it as SimpleNode }
-    override fun postOrder() = super.postOrder().map { it as SimpleNode }
+@Serializable
+data class NodeRange(val start: Position, val end: Position) {
+    override fun toString(): String = "[${start.line}, ${start.column}] - [${end.line}, ${end.column}]"
 }
 
+@Serializable
+data class Position(@SerialName("l") val line: Int, @SerialName("c") val column: Int)
+
 interface Parser<T : Node> {
     /**
      * Parse input stream into an AST.

src/main/kotlin/astminer/common/model/Token.kt

@@ -0,0 +1,23 @@
+package astminer.common.model
+
+import astminer.common.normalizeToken
+
+class Token(val original: String?) {
+    /** Final token after all normalizations and shadowing
+     * @see technical
+     * @see normalized **/
+    val final: String
+        get() = technical ?: normalized
+
+    /** Token that shadows any original or normalized token
+     * and have the most priority in calculating final token
+     * that will be saved. It can be useful when it's necessary to hide something
+     * (for example method name in method name prediction problem) **/
+    var technical: String? = null
+
+    /** Original token after string normalization
+     * @see normalizeToken **/
+    val normalized = normalizeToken(original)
+
+    override fun toString(): String = final
+}

src/main/kotlin/astminer/config/StorageConfigs.kt

@@ -42,8 +42,16 @@ class DotAstStorageConfig : StorageConfig() {
  */
 @Serializable
 @SerialName("json AST")
-class JsonAstStorageConfig(private val withPaths: Boolean = false) : StorageConfig() {
-    override fun createStorage(outputDirectoryPath: String) = JsonAstStorage(outputDirectoryPath, withPaths)
+class JsonAstStorageConfig(
+    private val withPaths: Boolean = false,
+    private val withRanges: Boolean = false
+) : StorageConfig() {
+    override fun createStorage(outputDirectoryPath: String) =
+        JsonAstStorage(
+            outputDirectoryPath,
+            withPaths,
+            withRanges
+        )
 }
 
 /**

src/main/kotlin/astminer/featureextraction/TreeFeature.kt

@@ -57,7 +57,7 @@ object Tokens : TreeFeature<List<String>> {
 
     private fun findTokens(node: Node, tokensList: MutableList<String>): List<String> {
         node.children.forEach { findTokens(it, tokensList) }
-        tokensList.add(node.token)
+        tokensList.add(node.token.final)
         return tokensList
     }
 }

src/main/kotlin/astminer/filters/CommonFilters.kt

@@ -1,5 +1,6 @@
 package astminer.filters
 
+import astminer.common.TOKEN_DELIMITER
 import astminer.common.model.*
 import astminer.featureextraction.NumberOfNodes
 
@@ -23,8 +24,8 @@ class TreeSizeFilter(private val minSize: Int = 0, private val maxSize: Int? = n
  * Filter that excludes trees that have more words than [maxWordsNumber] in any token of their node.
  */
 class WordsNumberFilter(private val maxWordsNumber: Int) : FunctionFilter, FileFilter {
-    private fun validateTree(root: Node) =
-        !root.preOrder().any { node -> node.token.split(Node.TOKEN_DELIMITER).size > maxWordsNumber }
+    private fun validateTree(root: Node) = root.preOrder()
+        .none { node -> node.token.final.split(TOKEN_DELIMITER).size > maxWordsNumber }
 
     override fun validate(functionInfo: FunctionInfo<out Node>) = validateTree(functionInfo.root)
 

src/main/kotlin/astminer/labelextractor/FunctionNameLabelExtractor.kt

@@ -14,13 +14,13 @@ object FunctionNameLabelExtractor : FunctionLabelExtractor {
     private const val RECURSIVE_CALL_TOKEN = "SELF"
 
     override fun process(functionInfo: FunctionInfo<out Node>): LabeledResult<out Node>? {
-        val normalizedName = functionInfo.nameNode?.normalizedToken ?: return null
+        val normalizedName = functionInfo.nameNode?.token?.normalized ?: return null
         functionInfo.root.preOrder().forEach { node ->
-            if (node.originalToken == functionInfo.nameNode?.originalToken) {
-                node.technicalToken = RECURSIVE_CALL_TOKEN
+            if (node.token.original == functionInfo.nameNode?.token?.original) {
+                node.token.technical = RECURSIVE_CALL_TOKEN
             }
         }
-        functionInfo.nameNode?.technicalToken = HIDDEN_METHOD_NAME_TOKEN
+        functionInfo.nameNode?.token?.technical = HIDDEN_METHOD_NAME_TOKEN
         return LabeledResult(functionInfo.root, normalizedName, functionInfo.qualifiedPath)
     }
 }

src/main/kotlin/astminer/parse/ForeignParser.kt

@@ -1,7 +1,8 @@
 package astminer.parse
 
+import astminer.common.SimpleNode
+import astminer.common.model.NodeRange
 import astminer.common.model.Parser
-import astminer.common.model.SimpleNode
 import astminer.config.FileExtension
 import astminer.config.ParserType
 import kotlinx.serialization.Serializable
@@ -23,17 +24,29 @@ import kotlin.io.path.createTempDirectory
  *      {
  *        "token": null,
  *        "nodeType": "i_am_root",
- *        "children": [1,2]
+ *        "children": [1,2],
+ *        "range" : {
+ *          "start" : { "l" : 0, "c" : 0 },
+ *          "end" : { "l" 1, "c" : 4 }
+ *        }
  *      },
  *      {
  *        "token": "Hello",
  *        "nodeType": "left_child",
  *        "children": []
+ *        "range" : {
+ *          "start" : { "l" : 0, "c": 0 },
+ *          "end" : { "l": 0, "c": 5 }
+ *        }
  *      },
  *      {
  *        "token": "World!",
  *        "nodeType": "right_child",
- *        "children": []
+ *        "children": [],
+ *        "range" : {
+ *          "start" : { "l" : 1, "c" : 0 },
+ *          "end" : { "l" : 1, "c" : 6 }
+ *        }
  *      }
  *    ]
  *  }
@@ -57,7 +70,14 @@ private fun launchScript(args: List<String>): String {
 
 private fun convertFromForeignTree(context: ForeignTree, rootId: Int = 0, parent: SimpleNode? = null): SimpleNode {
     val foreignNode = context.tree[rootId]
-    val node = SimpleNode(foreignNode.nodeType, mutableListOf(), parent, foreignNode.token)
+
+    val node = SimpleNode(
+        children = mutableListOf(),
+        parent = parent,
+        typeLabel = foreignNode.nodeType,
+        token = foreignNode.token,
+        range = foreignNode.range
+    )
     val children = foreignNode.children.map { convertFromForeignTree(context, it, node) }
     node.children.addAll(children)
     return node
@@ -67,7 +87,12 @@ private fun convertFromForeignTree(context: ForeignTree, rootId: Int = 0, parent
 private data class ForeignTree(val tree: List<ForeignNode>)
 
 @Serializable
-private data class ForeignNode(val token: String?, val nodeType: String, val children: List<Int>)
+private data class ForeignNode(
+    val token: String?,
+    val nodeType: String,
+    val range: NodeRange? = null,
+    val children: List<Int>
+)
 
 /** Use this parser to get a tree from external script.
  *  It uses `getTreeFromScript` and `getArguments` functions to generate

src/main/kotlin/astminer/parse/antlr/AntlrNode.kt

@@ -1,11 +1,13 @@
 package astminer.parse.antlr
 
 import astminer.common.model.Node
+import astminer.common.model.NodeRange
 
 class AntlrNode(
     override val typeLabel: String,
     override var parent: AntlrNode?,
-    originalToken: String?
+    originalToken: String?,
+    override val range: NodeRange? = null
 ) : Node(originalToken) {
 
     override val children: MutableList<AntlrNode> = mutableListOf()

src/main/kotlin/astminer/parse/antlr/compressedTreesUtil.kt

@@ -0,0 +1,23 @@
+package astminer.parse.antlr
+
+import astminer.common.model.Node
+
+fun decompressTypeLabel(typeLabel: String) = typeLabel.split("|")
+
+fun AntlrNode.lastLabel() = decompressTypeLabel(typeLabel).last()
+
+fun AntlrNode.firstLabel() = decompressTypeLabel(typeLabel).first()
+
+fun AntlrNode.hasLastLabel(label: String): Boolean = lastLabel() == label
+
+fun AntlrNode.lastLabelIn(labels: List<String>): Boolean = labels.contains(lastLabel())
+
+fun AntlrNode.hasFirstLabel(label: String): Boolean = firstLabel() == label
+
+fun AntlrNode.firstLabelIn(labels: List<String>): Boolean = labels.contains(firstLabel())
+
+fun Node.getTokensFromSubtree(): String =
+    if (isLeaf()) token.original ?: "" else children.joinToString(separator = "") { it.getTokensFromSubtree() }
+
+fun AntlrNode.getItOrChildrenOfType(typeLabel: String): List<AntlrNode> =
+    if (hasLastLabel(typeLabel)) listOf(this) else this.getChildrenOfType(typeLabel)