Graph Visualization: Added the General Chunk algorithm, multiple stacks

to address debug markers, colors for nodes and unit testing.

General Chunk Algorithm: This algorithm creates super-nodes to avoid
display more than a fix number of nodes in every level. For every
level, the nodes are grouped in super-nodes based on command names,
then if the numbers of nodes exceed the maximum limit, chunks are
created, and if in the new chunks the numbers of nodes exceed the maximum limit,
then new chunks are created and so on.

Results: Applying the General Chunk Algorithm is possible to visualize graphs
containing millions of nodes and edges. The time is reduced from ~20 seconds
to ~2 seconds to expand super-nodes in Tensorboard.

Note1: All "op" attribute for nodes in pbtxt format must be different, since
the find_similar_subgraphs algorithm from Tensorboard computes the similarity
between nodes with same "op".

Note2: Adding colors to the nodes increase the time for expand super-nodes.

gapis/api/BUILD.bazel

@@ -93,6 +93,7 @@ go_test(
     srcs = [
         "cmd_id_group_test.go",
         "cmd_service_test.go",
+        "graph_visualization_test.go",
         "subcmd_idx_test.go",
         "subcmd_idx_trie_test.go",
     ],

gapis/api/graph_visualization.go

@@ -86,6 +86,24 @@ func (label *Label) GetCommandId() int {
 	return 0
 }
 
+// GetTopLevelName returns the name of the first level
+// corresponding to the top level in hierarchy.
+func (label *Label) GetTopLevelName() string {
+	if len(label.LevelsName) > 0 {
+		return label.LevelsName[0]
+	}
+	return ""
+}
+
+// GetTopLevelID returns the ID of the first level
+// corresponding to the top level in hierarchy.
+func (label *Label) GetTopLevelID() int {
+	if len(label.LevelsID) > 0 {
+		return label.LevelsID[0]
+	}
+	return 0
+}
+
 // GetLabelAsAString returns the Label as a string concatenating
 // names and ID for each level delimited by '/'.
 func (label *Label) GetLabelAsAString() string {

gapis/api/graph_visualization_test.go

@@ -0,0 +1,88 @@
+// Copyright (C) 2018 Google Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package api
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestLabel(t *testing.T) {
+
+	label := &Label{}
+	label.PushBack("name1", 1)
+	label.PushBack("name2", 2)
+	label.PushBack("name3", 3)
+	label.PushBack("name4", 4)
+	label.PushFront("name5", 5)
+	label.PushFront("name6", 6)
+
+	obtainedLabel := label
+
+	wantedLabel := &Label{
+		LevelsName: []string{"name6", "name5", "name1", "name2", "name3", "name4"},
+		LevelsID:   []int{6, 5, 1, 2, 3, 4},
+	}
+
+	if !reflect.DeepEqual(wantedLabel, obtainedLabel) {
+		t.Errorf("The Label is different\n")
+		t.Errorf("Wanted %v, obtained %v\n", wantedLabel, obtainedLabel)
+	}
+
+	label.Insert(0, "name10", 10)
+	label.Insert(5, "name11", 11)
+	label.Insert(5, "name12", 12)
+
+	wantedLabel = &Label{
+		LevelsName: []string{"name10", "name6", "name5", "name1", "name2", "name12", "name11", "name3", "name4"},
+		LevelsID:   []int{10, 6, 5, 1, 2, 12, 11, 3, 4},
+	}
+
+	if !reflect.DeepEqual(wantedLabel, obtainedLabel) {
+		t.Errorf("The Label is different\n")
+		t.Errorf("Wanted %v, obtained %v\n", wantedLabel, obtainedLabel)
+	}
+
+	obtainedLabel.PushBackLabel(obtainedLabel)
+	wantedLabel = &Label{
+		LevelsName: []string{"name10", "name6", "name5", "name1", "name2", "name12", "name11", "name3", "name4",
+			"name10", "name6", "name5", "name1", "name2", "name12", "name11", "name3", "name4"},
+		LevelsID: []int{10, 6, 5, 1, 2, 12, 11, 3, 4,
+			10, 6, 5, 1, 2, 12, 11, 3, 4},
+	}
+
+	if !reflect.DeepEqual(wantedLabel, obtainedLabel) {
+		t.Errorf("The Label is different\n")
+		t.Errorf("Wanted %v, obtained %v\n", wantedLabel, obtainedLabel)
+	}
+
+	if obtainedLabel.GetCommandName() != "name4" {
+		t.Errorf("The command name is different")
+	}
+	if obtainedLabel.GetCommandId() != 4 {
+		t.Errorf("The command ID is different")
+	}
+	if obtainedLabel.GetTopLevelName() != "name10" {
+		t.Errorf("The top name is different")
+	}
+	if obtainedLabel.GetTopLevelID() != 10 {
+		t.Errorf("The top ID is different")
+	}
+	wantedLabelAsAString := "name1010/name66/name55/name11/name22/name1212/name1111/name33/name44/" +
+		"name1010/name66/name55/name11/name22/name1212/name1111/name33/name44"
+	if obtainedLabel.GetLabelAsAString() != wantedLabelAsAString {
+		t.Errorf("The LabelAsAString is different")
+		t.Errorf("Wanted %v, obtained %v\n", wantedLabelAsAString, obtainedLabel.GetLabelAsAString())
+	}
+}

gapis/api/vulkan/BUILD.bazel

@@ -151,6 +151,7 @@ go_test(
     name = "go_default_test",
     srcs = [
         "externs_test.go",
+        "graph_visualization_test.go",
         "image_primer_shaders_test.go",
         "image_primer_test.go",
     ],

gapis/api/vulkan/graph_visualization.go

@@ -51,9 +51,75 @@ type labelForVulkanCommands struct {
 	subCommandIndexNameToHierarchyLabel map[string]*api.Label
 	commandBufferIdToHierarchy          map[VkCommandBuffer]*api.Hierarchy
 	commandBufferIdToOrderNumber        map[VkCommandBuffer]int
-	labelsInsideDebugMarkers            []*api.Label
-	positionOfDebugMarkersBegin         []int
-	numberOfDebugMarker                 int
+	nameAndIdToDebugMarkerStack         map[string]*debugMarkerStack
+}
+
+type debugMarkerStack struct {
+	// positionsOfDebugMarkersBegin is used like a stack, representing the position
+	// of debug_marker_begin commands in labels to access fast to its Label.
+	positionsOfDebugMarkersBegin []int
+
+	// labels are all labels from commands considered.
+	labels []*api.Label
+
+	// debugMarkerID is the unique ID assigned to debug_markers created.
+	debugMarkerID int
+}
+
+func (dm *debugMarkerStack) getSize() int {
+	return len(dm.positionsOfDebugMarkersBegin)
+}
+
+// top returns the Label of the last debug_marker_begin command if exists.
+func (dm *debugMarkerStack) top() *api.Label {
+	if dm.getSize() > 0 {
+		position := dm.positionsOfDebugMarkersBegin[dm.getSize()-1]
+		if position < len(dm.labels) {
+			return dm.labels[position]
+		}
+	}
+	return &api.Label{}
+}
+
+func (dm *debugMarkerStack) pop() {
+	if dm.getSize() > 0 {
+		dm.positionsOfDebugMarkersBegin = dm.positionsOfDebugMarkersBegin[:dm.getSize()-1]
+	}
+}
+
+func (dm *debugMarkerStack) push(label *api.Label) {
+	dm.labels = append(dm.labels, label)
+	if label.GetCommandName() == VK_CMD_DEBUG_MARKER_BEGIN {
+		dm.positionsOfDebugMarkersBegin = append(dm.positionsOfDebugMarkersBegin, len(dm.labels)-1)
+	}
+}
+
+// addDebugMarker adds a new debug marker from the last debug_marker_begin Label to the current Label.
+func (dm *debugMarkerStack) addDebugMarker() {
+	position := dm.positionsOfDebugMarkersBegin[dm.getSize()-1]
+	size := dm.labels[position].GetSize()
+	dm.debugMarkerID++
+	for i := position; i < len(dm.labels); i++ {
+		dm.labels[i].Insert(size-1, VK_CMD_DEBUG_MARKER, dm.debugMarkerID)
+	}
+}
+
+func (dm *debugMarkerStack) checkDebugMarkers(label *api.Label) {
+	// It preserves Labels with no-decreasing sizes in the stack of debug_markers_begin avoiding
+	// insert debug markers in Labels with smaller sizes that would break the levels of hierarchy.
+	for dm.getSize() > 0 && label.GetSize() < dm.top().GetSize() {
+		dm.pop()
+	}
+	dm.push(label)
+
+	if label.GetCommandName() == VK_CMD_DEBUG_MARKER_END {
+		if dm.getSize() > 0 {
+			if label.GetSize() == dm.top().GetSize() && getMaxCommonPrefix(label, dm.top()) == label.GetSize()-1 {
+				dm.addDebugMarker()
+			}
+			dm.pop()
+		}
+	}
 }
 
 func getCommandHierarchyNames() *api.HierarchyNames {
@@ -88,55 +154,6 @@ func getMaxCommonPrefix(label1 *api.Label, label2 *api.Label) int {
 	return size
 }
 
-func addDebugMarker(builder *labelForVulkanCommands, from, to int) {
-	level := builder.labelsInsideDebugMarkers[len(builder.labelsInsideDebugMarkers)-1].GetSize() - 1
-	builder.numberOfDebugMarker++
-	for i := from; i <= to; i++ {
-		builder.labelsInsideDebugMarkers[i].Insert(level, VK_CMD_DEBUG_MARKER, builder.numberOfDebugMarker)
-	}
-}
-
-func checkDebugMarkers(builder *labelForVulkanCommands, currentLabel *api.Label) {
-	commandName := currentLabel.GetCommandName()
-	positionOfLastDebugMarkerBegin := 0
-	labelOfLastDebugMarkerBegin := &api.Label{}
-	if len(builder.positionOfDebugMarkersBegin) > 0 {
-		positionOfLastDebugMarkerBegin = builder.positionOfDebugMarkersBegin[len(builder.positionOfDebugMarkersBegin)-1]
-		labelOfLastDebugMarkerBegin = builder.labelsInsideDebugMarkers[positionOfLastDebugMarkerBegin]
-	}
-
-	if commandName == VK_CMD_DEBUG_MARKER_BEGIN {
-		if len(builder.positionOfDebugMarkersBegin) > 0 {
-			if currentLabel.GetSize() < labelOfLastDebugMarkerBegin.GetSize() {
-				builder.positionOfDebugMarkersBegin = builder.positionOfDebugMarkersBegin[:len(builder.positionOfDebugMarkersBegin)-1]
-			}
-		}
-		builder.labelsInsideDebugMarkers = append(builder.labelsInsideDebugMarkers, currentLabel)
-		builder.positionOfDebugMarkersBegin = append(builder.positionOfDebugMarkersBegin, len(builder.labelsInsideDebugMarkers)-1)
-
-	} else if commandName == VK_CMD_DEBUG_MARKER_END {
-
-		if len(builder.positionOfDebugMarkersBegin) > 0 {
-			if currentLabel.GetSize() != labelOfLastDebugMarkerBegin.GetSize() {
-				builder.positionOfDebugMarkersBegin = builder.positionOfDebugMarkersBegin[:len(builder.positionOfDebugMarkersBegin)-1]
-
-			} else if getMaxCommonPrefix(labelOfLastDebugMarkerBegin, currentLabel) == currentLabel.GetSize()-1 {
-				builder.labelsInsideDebugMarkers = append(builder.labelsInsideDebugMarkers, currentLabel)
-				builder.positionOfDebugMarkersBegin = append(builder.positionOfDebugMarkersBegin, len(builder.labelsInsideDebugMarkers)-1)
-				addDebugMarker(builder, positionOfLastDebugMarkerBegin, len(builder.labelsInsideDebugMarkers)-1)
-				builder.positionOfDebugMarkersBegin = builder.positionOfDebugMarkersBegin[:len(builder.positionOfDebugMarkersBegin)-1]
-			}
-		}
-
-	} else if len(builder.positionOfDebugMarkersBegin) > 0 {
-		if currentLabel.GetSize() < labelOfLastDebugMarkerBegin.GetSize() {
-			builder.positionOfDebugMarkersBegin = builder.positionOfDebugMarkersBegin[:len(builder.positionOfDebugMarkersBegin)-1]
-		} else {
-			builder.labelsInsideDebugMarkers = append(builder.labelsInsideDebugMarkers, currentLabel)
-		}
-	}
-}
-
 func getCommandBufferId(command api.Cmd) (VkCommandBuffer, bool) {
 	parameters := command.CmdParams()
 	for _, parameter := range parameters {
@@ -151,6 +168,7 @@ func getCommandBufferId(command api.Cmd) (VkCommandBuffer, bool) {
 func (builder *labelForVulkanCommands) GetCommandLabel(command api.Cmd, cmdId uint64) *api.Label {
 	label := &api.Label{}
 	commandName := command.CmdName()
+	nameAndId := ""
 	if commandBufferId, ok := getCommandBufferId(command); ok {
 		hierarchy, ok := builder.commandBufferIdToHierarchy[commandBufferId]
 		if !ok {
@@ -161,10 +179,18 @@ func (builder *labelForVulkanCommands) GetCommandLabel(command api.Cmd, cmdId ui
 		label.PushBack(COMMAND_BUFFER, builder.commandBufferIdToOrderNumber[commandBufferId])
 		label.PushBackLabel(getLabelFromHierarchy(commandName, commandHierarchyNames, hierarchy))
 		label.PushBack(commandName, int(cmdId))
+		nameAndId = fmt.Sprintf("%s%d", COMMAND_BUFFER, builder.commandBufferIdToOrderNumber[commandBufferId])
 	} else {
 		label.PushBack(commandName, int(cmdId))
+		nameAndId = fmt.Sprintf("%s%d", commandName, cmdId)
+	}
+
+	currentDebugMarker, ok := builder.nameAndIdToDebugMarkerStack[nameAndId]
+	if !ok {
+		currentDebugMarker = &debugMarkerStack{}
+		builder.nameAndIdToDebugMarkerStack[nameAndId] = currentDebugMarker
 	}
-	checkDebugMarkers(builder, label)
+	currentDebugMarker.checkDebugMarkers(label)
 	return label
 }
 
@@ -193,9 +219,15 @@ func (builder *labelForVulkanCommands) GetSubCommandLabel(index api.SubCmdIdx, c
 	labelFromHierarchy := getLabelFromHierarchy(subCommandName, subCommandHierarchyNames, hierarchy)
 	labelFromHierarchy.PushBack(subCommandName, int(index[len(index)-1]))
 	builder.subCommandIndexNameToHierarchyLabel[subCommandIndexName.String()] = labelFromHierarchy
-
 	label.PushBackLabel(labelFromHierarchy)
-	checkDebugMarkers(builder, label)
+
+	nameAndId := fmt.Sprintf("%s%d", commandName, cmdId)
+	currentDebugMarker, ok := builder.nameAndIdToDebugMarkerStack[nameAndId]
+	if !ok {
+		currentDebugMarker = &debugMarkerStack{}
+		builder.nameAndIdToDebugMarkerStack[nameAndId] = currentDebugMarker
+	}
+	currentDebugMarker.checkDebugMarkers(label)
 	return label
 }
 
@@ -233,5 +265,6 @@ func (API) GetGraphVisualizationBuilder() api.GraphVisualizationBuilder {
 		subCommandIndexNameToHierarchyLabel: map[string]*api.Label{},
 		commandBufferIdToHierarchy:          map[VkCommandBuffer]*api.Hierarchy{},
 		commandBufferIdToOrderNumber:        map[VkCommandBuffer]int{},
+		nameAndIdToDebugMarkerStack:         map[string]*debugMarkerStack{},
 	}
 }