Concatenate event metadata into "paths".

Stackdriver (SD) imposes a limit of 10 labels per custom metric. Simply
copying event metadata into SD labels results in some metrics hitting
that limit.

To address this, drop eventType and the org/space/application UUID labels
completely, and instead create two "path" labels, originPath and
applicationPath, that concatenate together related pieces of event
metadata into a hierarchy.

The origin path hierarchy is /job/origin.
The application path hierarchy is /org/space/application.

Keep the VM and application index labels separate so that higher level
aggregations of the same metric data can be created, and the deployment
label separate to allow it to be used to distinguish between multiple
PCF instances in the same GCP project.

src/stackdriver-nozzle/nozzle/label_maker.go

@@ -17,8 +17,10 @@
 package nozzle
 
 import (
+	"bytes"
 	"encoding/binary"
 	"fmt"
+	"strings"
 
 	"github.com/cloudfoundry-community/stackdriver-tools/src/stackdriver-nozzle/cloudfoundry"
 	"github.com/cloudfoundry/sonde-go/events"
@@ -36,67 +38,126 @@ type labelMaker struct {
 	appInfoRepository cloudfoundry.AppInfoRepository
 }
 
-func (lm *labelMaker) Build(envelope *events.Envelope) map[string]string {
-	labels := map[string]string{}
+type labelMap map[string]string
 
-	if envelope.Origin != nil {
-		labels["origin"] = envelope.GetOrigin()
+func (labels labelMap) setIfNotEmpty(key, value string) {
+	if value != "" {
+		labels[key] = value
 	}
+}
 
-	if envelope.EventType != nil {
-		labels["eventType"] = envelope.GetEventType().String()
+func (labels labelMap) setValueOrUnknown(key, value string) {
+	if value == "" {
+		labels[key] = "unknown_" + key
+	} else {
+		labels[key] = value
 	}
+}
 
-	if envelope.Job != nil {
-		labels["job"] = envelope.GetJob()
+func (labels labelMap) path(keys ...string) string {
+	var b bytes.Buffer
+	for _, k := range keys {
+		if _, ok := labels[k]; !ok {
+			continue
+		}
+		b.WriteByte('/')
+		b.WriteString(labels[k])
 	}
+	return b.String()
+}
 
-	if envelope.Index != nil {
-		labels["index"] = envelope.GetIndex()
-	}
+// Build extracts metric metadata from the event envelope and event contained
+// within, and constructs a set of StackDriver (SD) metric labels from them.
+//
+// Since SD only allows 10 custom labels per metric, we collapse most metadata
+// into two "paths" one representing the metric origin and one representing the
+// serving application. We maintain vm and application instance indexes as
+// separate labels so that it is easy to aggregate across multiple instances.
+// We maintain "deployment" as a separate label to facilitate monitoring
+// multple PCF instances within a GCP project, though this does require
+// users to name their PCF deployment on bosh something other than "cf".:
+func (lm *labelMaker) Build(envelope *events.Envelope) map[string]string {
+	labels := labelMap{}
 
-	if appId := lm.getApplicationId(envelope); appId != "" {
-		labels["applicationId"] = appId
-		lm.buildAppMetadataLabels(appId, labels, envelope)
+	// Copy over tags from the envelope into labels first so they cannot
+	// overwrite the ones derived from envelope/event data.
+	for k, v := range envelope.GetTags() {
+		safe := strings.Map(sanitize, k)
+		labels[safe] = v
 	}
 
+	labels.setIfNotEmpty("deployment", envelope.GetDeployment())
+	labels.setIfNotEmpty("originPath", lm.getOriginPath(envelope))
+	labels.setIfNotEmpty("index", envelope.GetIndex())
+	labels.setIfNotEmpty("applicationPath", lm.getApplicationPath(envelope))
+	labels.setIfNotEmpty("instanceIndex", getInstanceIndex(envelope))
+
 	return labels
 }
 
-func (lm *labelMaker) getApplicationId(envelope *events.Envelope) string {
-	if envelope.GetEventType() == events.Envelope_HttpStartStop {
-		return formatUUID(envelope.GetHttpStartStop().GetApplicationId())
-	} else if envelope.GetEventType() == events.Envelope_LogMessage {
-		return envelope.GetLogMessage().GetAppId()
-	} else if envelope.GetEventType() == events.Envelope_ContainerMetric {
-		return envelope.GetContainerMetric().GetApplicationId()
-	} else {
-		return ""
-	}
+// getOriginPath returns a path that uniquely identifies a metric origin.
+// The path hierarchy is /job/origin, e.g.
+//     /diego_brain/tps_listener
+func (lm *labelMaker) getOriginPath(envelope *events.Envelope) string {
+	labels := labelMap{}
+	labels.setValueOrUnknown("job", envelope.GetJob())
+	labels.setValueOrUnknown("origin", envelope.GetOrigin())
+	return labels.path("job", "origin")
 }
 
-func (lm *labelMaker) buildAppMetadataLabels(guid string, labels map[string]string, envelope *events.Envelope) {
-	app := lm.appInfoRepository.GetAppInfo(guid)
-
-	if app.AppName != "" {
-		labels["appName"] = app.AppName
+// getApplicationPath returns a path that uniquely identifies a
+// collection of instances of a given application running in an org + space.
+// The path heirarchy is /deployment/org/space/application, e.g.
+//     /system/autoscaling/autoscale
+func (lm *labelMaker) getApplicationPath(envelope *events.Envelope) string {
+	appID := getApplicationId(envelope)
+	if appID == "" {
+		return ""
 	}
-
-	if app.SpaceName != "" {
-		labels["spaceName"] = app.SpaceName
+	app := lm.appInfoRepository.GetAppInfo(appID)
+	if app.AppName == "" {
+		return ""
 	}
 
-	if app.SpaceGUID != "" {
-		labels["spaceGuid"] = app.SpaceGUID
-	}
+	labels := labelMap{}
+	labels.setValueOrUnknown("org", app.OrgName)
+	labels.setValueOrUnknown("space", app.SpaceName)
+	labels.setValueOrUnknown("application", app.AppName)
+
+	return labels.path("org", "space", "application")
+}
 
-	if app.OrgName != "" {
-		labels["orgName"] = app.OrgName
+// getApplicationId extracts the application UUID from the event contained
+// within the envelope, for those events that have application IDs.
+func getApplicationId(envelope *events.Envelope) string {
+	switch envelope.GetEventType() {
+	case events.Envelope_HttpStartStop:
+		return formatUUID(envelope.GetHttpStartStop().GetApplicationId())
+	case events.Envelope_LogMessage:
+		return envelope.GetLogMessage().GetAppId()
+	case events.Envelope_ContainerMetric:
+		return envelope.GetContainerMetric().GetApplicationId()
 	}
+	return ""
+}
 
-	if app.OrgGUID != "" {
-		labels["orgGuid"] = app.OrgGUID
+// getInstanceIndex extracts the instance index or UUID from the event
+// contained within the envelope, for those events that have instance IDs.
+func getInstanceIndex(envelope *events.Envelope) string {
+	switch envelope.GetEventType() {
+	case events.Envelope_HttpStartStop:
+		hss := envelope.GetHttpStartStop()
+		if hss != nil && hss.InstanceIndex != nil {
+			return fmt.Sprintf("%d", hss.GetInstanceIndex())
+		}
+		// Sometimes InstanceIndex is not set but InstanceId is; fall back.
+		return hss.GetInstanceId()
+	case events.Envelope_LogMessage:
+		return envelope.GetLogMessage().GetSourceInstance()
+	case events.Envelope_ContainerMetric:
+		return fmt.Sprintf("%d", envelope.GetContainerMetric().GetInstanceIndex())
 	}
+	return ""
 }
 
 func formatUUID(uuid *events.UUID) string {
@@ -108,3 +169,18 @@ func formatUUID(uuid *events.UUID) string {
 	binary.LittleEndian.PutUint64(uuidBytes[8:], uuid.GetHigh())
 	return fmt.Sprintf("%x-%x-%x-%x-%x", uuidBytes[0:4], uuidBytes[4:6], uuidBytes[6:8], uuidBytes[8:10], uuidBytes[10:])
 }
+
+// sanitize is a function usable with strings.Map to perform
+// s/[^A-Za-z0-9_]/_/g, for sanitizing label key names.
+func sanitize(r rune) rune {
+	switch {
+	case r >= 'a' && r <= 'z':
+		return r
+	case r >= 'A' && r <= 'Z':
+		return r
+	case r >= '0' && r <= '9':
+		return r
+	default:
+		return '_'
+	}
+}