Skip to content

Commit

Permalink
Reimplement yourkit to cflamegraph converter
Browse files Browse the repository at this point in the history
  • Loading branch information
kornilova203 committed Jul 21, 2018
1 parent f537cbd commit 0b0eadd
Show file tree
Hide file tree
Showing 7 changed files with 103 additions and 279 deletions.
6 changes: 4 additions & 2 deletions src/main/flatbuffers/cflamegraph_schema.fbs
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
namespace com.github.kornilova_l.flamegraph.cflamegraph;

// Contains the same information as flamegraph format

table Names {
class_names:[string];
method_names:[string];
Expand All @@ -10,8 +12,8 @@ struct Node {
class_name_id:int; // -1 if class name is not set
method_name_id:int;
description_id:int; // -1 if description is not set
width:int;
depth:int;
width:int; // cannot be 0
depth:int; // depth starts with 1
}

table Tree {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,17 +13,17 @@ import java.io.File
* It uses FlatBuffers for data serialization and does not duplicate
* class names, method names and descriptions.
*
* See scheme in /src/main/flatbuffers/cflamegraph_schema.fbs
* See schema in /src/main/flatbuffers/cflamegraph_schema.fbs
*
* Example:
* ._ _
* |c|d|___ _
* |b()____|e|_______ _
* |Class.a__________|f|
* |c|d|____ _
* |void b()|e|_______ _
* |Class.a___________|f|
*
* In original flamegraph format this example would look like this:
* a;b();c 5
* a;b();d 5
* a;void b();d 5
* a;b() 10
* a;e 5
* a 50
Expand All @@ -33,7 +33,7 @@ import java.io.File
* Names: {
* classNames: ["Class"],
* methodNames: ["a", "b", "c", "d", "e", "f"],
* descriptions: ["()"]
* descriptions: ["()void"]
* }
* Tree: {
* nodes: [
Expand Down
Original file line number Diff line number Diff line change
@@ -1,131 +1,78 @@
package com.github.kornilova_l.flamegraph.plugin.server.converters.file_to_call_traces.cflamegraph

import com.github.kornilova_l.flamegraph.cflamegraph.Node
import com.github.kornilova_l.flamegraph.cflamegraph.Tree
import com.github.kornilova_l.flamegraph.plugin.pleaseReportIssue
import com.github.kornilova_l.flamegraph.plugin.server.trees.util.TreesUtil
import com.github.kornilova_l.flamegraph.plugin.server.trees.util.TreesUtil.parsePositiveInt
import com.github.kornilova_l.flamegraph.plugin.server.trees.util.TreesUtil.parsePositiveLong
import com.github.kornilova_l.flamegraph.proto.TreeProtos
import java.io.BufferedReader
import java.io.File
import java.io.FileReader
import java.nio.ByteBuffer
import java.util.*
import java.util.regex.Pattern


internal class Converter(file: File) {
val tree: TreeProtos.Tree
private val classNames = HashMap<Int, String>()
private val methodNames = HashMap<Int, String>()
private val descriptions = HashMap<Int, String>()
private val headerPattern = Pattern.compile("--[CMD]-- \\d+")
private val classNames: Map<Int, String>
private val methodNames: Map<Int, String>
private val descriptions: Map<Int, String>
private var maxDepth = 0

init {
val tree = createEmptyTree()
val cflamegraphTree = Tree.getRootAsTree(ByteBuffer.wrap(file.readBytes()))

val names = cflamegraphTree.names()
classNames = convertNamesToMap(names.classNamesLength()) { names.classNames(it) }
methodNames = convertNamesToMap(names.methodNamesLength()) { names.methodNames(it) }
descriptions = convertNamesToMap(names.descriptionsLength()) { names.descriptions(it) }

val currentStack = ArrayList<TreeProtos.Tree.Node.Builder>()
currentStack.add(tree.baseNodeBuilder)
BufferedReader(FileReader(file), 1000 * 8192).use { reader ->
var line: String? = initMaps(reader)
while (line != null) {
if (line.isNotBlank()) {
processLine(line, currentStack)
}
line = reader.readLine()
}

for (i in 0 until cflamegraphTree.nodesLength()) {
processNode(cflamegraphTree.nodes(i), currentStack)
}

tree.depth = maxDepth
TreesUtil.setNodesOffsetRecursively(tree.baseNodeBuilder, 0)
TreesUtil.setTreeWidth(tree)
TreesUtil.setNodesCount(tree)
this.tree = tree.build()
}

private fun initMaps(reader: BufferedReader): String {
var line = reader.readLine()
while (line.isEmpty() || headerPattern.matcher(line).matches()) {
if (line.isEmpty()) {
line = reader.readLine()
continue
}
val linesCount = Integer.parseInt(line.substring(6))
val value = line[2]
val currentMap = when (value) {
'C' -> classNames
'M' -> methodNames
'D' -> descriptions
else ->
/* this should never happen because we checked line with regexp */
throw IllegalArgumentException("$pleaseReportIssue: Cannot read header $line")
}
for (i in 0 until linesCount) {
val mapLine = reader.readLine()
val lastSpacePos = mapLine.lastIndexOf(' ')
val id = parsePositiveInt(mapLine, lastSpacePos + 1, mapLine.length)
val name = mapLine.substring(0, lastSpacePos)
currentMap[id] = name
}
line = reader.readLine()
private fun convertNamesToMap(itemsCount: Int, getter: (Int) -> String): Map<Int, String> {
val map = HashMap<Int, String>()
for (i in 0 until itemsCount) {
map[i] = getter(i)
}
return line
return map
}

private fun processLine(line: String,
private fun processNode(node: Node,
currentStack: ArrayList<TreeProtos.Tree.Node.Builder>) {
var className: String? = null
var methodName: String? = null
var desc: String? = null
var width = -1L
var depth = -1
var i = 0
while (i < line.length - 1) {
val c = line[i]
val endOfNumPos = getNextEndOfNum(line, i + 2)
when (c) {
'C' -> className = classNames[getParamIntValue(line, i, endOfNumPos)]!!
'M' -> methodName = methodNames[getParamIntValue(line, i, endOfNumPos)]!!
'D' -> desc = descriptions[getParamIntValue(line, i, endOfNumPos)]!!
'd' -> depth = getParamIntValue(line, i, endOfNumPos)
'w' -> width = getParamLongValue(line, i, endOfNumPos)
}
i = endOfNumPos
}
if (depth == -1 || width == -1L) {
throw IllegalArgumentException("$pleaseReportIssue: Cannot find depth or width value in line: $line")
}
if (methodName == null) {
throw IllegalArgumentException("$pleaseReportIssue: Line must contain method name: $line")
}
while (depth < currentStack.size) { // if some calls are finished
validateNode(node)

while (node.depth() < currentStack.size) { // if some calls are finished
currentStack.removeAt(currentStack.size - 1)
}

val newNode = TreesUtil.updateNodeList(
currentStack[currentStack.size - 1],
className ?: "",
methodName,
desc ?: "",
width
if (node.classNameId() >= 0) classNames[node.classNameId()]!! else "",
methodNames[node.methodNameId()]!!,
if (node.descriptionId() >= 0) descriptions[node.descriptionId()]!! else "",
node.width().toLong()
)
currentStack.add(newNode)
if (currentStack.size - 1 > maxDepth) {
maxDepth = currentStack.size - 1
}
}

private fun getParamIntValue(line: String, paramPos: Int, endOfNumPos: Int): Int {
return parsePositiveInt(line, paramPos + 1, endOfNumPos)
}

private fun getParamLongValue(line: String, paramPos: Int, endOfNumPos: Int): Long {
return parsePositiveLong(line, paramPos + 1, endOfNumPos)
}

private fun getNextEndOfNum(line: String, startIndex: Int): Int {
for (i in startIndex until line.length) {
if (line[i] !in '0'..'9') {
return i
}
}
return line.length
private fun validateNode(node: Node) {
require(node.depth() > 0) { "$pleaseReportIssue: node depth must be bigger than 0." }
require(node.width() > 0) { "$pleaseReportIssue: node width must be bigger than 0." }
require(node.methodNameId() >= 0) { "$pleaseReportIssue: node method name id must be set." }
}

private fun createEmptyTree(): TreeProtos.Tree.Builder {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,14 @@ package com.github.kornilova_l.flamegraph.plugin.server.converters.file_to_call_

import com.github.kornilova_l.flamegraph.plugin.server.converters.file_to_call_traces.FileToCallTracesConverter
import com.github.kornilova_l.flamegraph.plugin.server.converters.file_to_file.ProfilerToFlamegraphConverter
import com.github.kornilova_l.flamegraph.plugin.server.converters.file_to_file.yourkit_csv.YourkitCsvToCFlamegraphConverter
import com.github.kornilova_l.flamegraph.proto.TreeProtos.Tree
import java.io.BufferedReader
import java.io.File
import java.io.FileReader


/**
* This converter is no longer supported.
* When new csv file is added it is converted with [YourkitCsvToCFlamegraphConverter]
*/
@Deprecated("When a new csv file is added it's converted with YourkitCsvToCFlamegraphConverter. " +
"This converter is to support already uploaded csv files.")
class YourkitCsvToCallTracesConverter : FileToCallTracesConverter() {
override fun getId(): String {
return "yourkit"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,65 +1,67 @@
package com.github.kornilova_l.flamegraph.plugin.server.converters.file_to_file

import com.github.kornilova_l.flamegraph.cflamegraph.Names
import com.github.kornilova_l.flamegraph.cflamegraph.Node
import com.github.kornilova_l.flamegraph.cflamegraph.Tree
import com.github.kornilova_l.flamegraph.plugin.server.FileToFileConverterFileSaver
import com.google.flatbuffers.FlatBufferBuilder
import com.intellij.openapi.extensions.ExtensionPointName
import java.io.BufferedWriter
import java.io.File
import java.io.FileWriter
import java.io.FileOutputStream


class CompressedFlamegraphFileSaver : FileToFileConverterFileSaver() {
override val extension = ProfilerToCompressedFlamegraphConverter.cFlamegraphExtension

override fun tryToConvert(file: File): Boolean {
val cFlamegraph = ProfilerToCompressedFlamegraphConverter.convert(file)
if (cFlamegraph != null) {
BufferedWriter(FileWriter(file)).use { writer ->
writeMap(cFlamegraph.classNames, writer, 'C')
writeMap(cFlamegraph.methodNames, writer, 'M')
writeMap(cFlamegraph.descriptions, writer, 'D')
for (line in cFlamegraph.lines) {
if (line.classNameId != null) {
writer.write("C")
writer.write(line.classNameId.toString())
}
writer.write("M")
writer.write(line.methodNameId.toString())
if (line.descId != null) {
writer.write("D")
writer.write(line.descId.toString())
}
writer.write("d")
writer.write(line.depth.toString())
writer.write("w")
writer.write(line.width.toString())
writer.write("\n")
}
}
return true
val cFlamegraph = ProfilerToCompressedFlamegraphConverter.convert(file) ?: return false
val builder = FlatBufferBuilder(1024)

val classNamesOffsets = IntArray(cFlamegraph.classNames.size)
for (i in 0 until cFlamegraph.classNames.size) {
classNamesOffsets[i] = builder.createString(cFlamegraph.classNames[i])
}
return false
}
val classNames = Names.createClassNamesVector(builder, classNamesOffsets)

val methodNamesOffsets = IntArray(cFlamegraph.methodNames.size)
for (i in 0 until cFlamegraph.methodNames.size) {
methodNamesOffsets[i] = builder.createString(cFlamegraph.methodNames[i])
}
val methodNames = Names.createMethodNamesVector(builder, methodNamesOffsets)

val descriptionsOffsets = IntArray(cFlamegraph.descriptions.size)
for (i in 0 until cFlamegraph.descriptions.size) {
descriptionsOffsets[i] = builder.createString(cFlamegraph.descriptions[i])
}
val descriptions = Names.createDescriptionsVector(builder, descriptionsOffsets)

val names = Names.createNames(builder, classNames, methodNames, descriptions)

private fun writeMap(map: Map<String, Int>, writer: BufferedWriter, letter: Char) {
if (map.isNotEmpty()) {
writer.write("--$letter-- ${map.size}\n")
for (entry in map.entries) {
writer.write(entry.key)
writer.write(" ")
writer.write(entry.value.toString())
writer.write("\n")
}
Tree.startNodesVector(builder, cFlamegraph.lines.size)
for (i in cFlamegraph.lines.size - 1 downTo 0) {
val line = cFlamegraph.lines[i]
Node.createNode(builder, line.classNameId ?: -1, line.methodNameId,
line.descId ?: -1, line.width, line.depth)
}
val nodes = builder.endVector()

val tree = Tree.createTree(builder, names, nodes)

builder.finish(tree)
FileOutputStream(file).write(builder.sizedByteArray())

return true
}
}

@Suppress("ArrayInDataClass") // Instances of the class will not be compared
data class CFlamegraph(val lines: List<CFlamegraphLine>,
val classNames: Map<String, Int>,
val methodNames: Map<String, Int>,
val descriptions: Map<String, Int>)
val classNames: Array<String>, // a "map" from id to class name
val methodNames: Array<String>, // a "map" from id to method name
val descriptions: Array<String>) // a "map" from id to description


data class CFlamegraphLine(val classNameId: Int?, val methodNameId: Int, val descId: Int?, val width: Long, val depth: Int)
data class CFlamegraphLine(val classNameId: Int?, val methodNameId: Int, val descId: Int?, val width: Int, val depth: Int)

abstract class ProfilerToCompressedFlamegraphConverter {
companion object {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import com.github.kornilova_l.flamegraph.plugin.server.converters.file_to_call_t
import com.github.kornilova_l.flamegraph.plugin.server.converters.file_to_file.CFlamegraph
import com.github.kornilova_l.flamegraph.plugin.server.converters.file_to_file.CFlamegraphLine
import com.github.kornilova_l.flamegraph.plugin.server.trees.util.TreesUtil.parsePositiveInt
import com.github.kornilova_l.flamegraph.plugin.server.trees.util.TreesUtil.parsePositiveLong
import java.io.BufferedReader
import java.io.File
import java.io.FileReader
Expand All @@ -26,7 +25,15 @@ class Converter(file: File) {
line = reader.readLine()
}
}
cFlamegraph = CFlamegraph(cFlamegraphLines, classNames, methodNames, descriptions)
cFlamegraph = CFlamegraph(cFlamegraphLines, toArray(classNames), toArray(methodNames), toArray(descriptions))
}

private fun toArray(names: HashMap<String, Int>): Array<String> {
val array = Array(names.size) { "" }
for (entry in names) {
array[entry.value] = entry.key
}
return array
}

private fun processLine(line: String) {
Expand All @@ -38,21 +45,21 @@ class Converter(file: File) {
if (!name.contains('(')) {
return
}
var width = -1L
var width = -1
var depth = -1
try {
/* find next delimiter */
for (i in delimPos + 1 until line.length - 2) {
if (line[i] == '"' && line[i + 1] == ',' && line[i + 2] == '"') {
width = parsePositiveLong(line, delimPos + 3, i)
width = parsePositiveInt(line, delimPos + 3, i)
depth = parsePositiveInt(line, i + 3, line.length - 1)
break
}
}
} catch (e: NumberFormatException) {
e.printStackTrace()
}
if (width == -1L || depth == -1) {
if (width == -1 || depth == -1) {
return
}
depth -= 1 // after this depth of first call is 1
Expand Down
Loading

0 comments on commit 0b0eadd

Please sign in to comment.