From 46af0bfab83f295ba04a9934fb70ab23e0fac215 Mon Sep 17 00:00:00 2001 From: Isaac Duarte Date: Wed, 16 Dec 2020 16:24:27 -0500 Subject: [PATCH 1/2] fix bug with unmarshalling flags and add test --- pkg/skaffold/instrumentation/meter.go | 10 ++-- pkg/skaffold/instrumentation/meter_test.go | 69 ++++++++++++++++++---- 2 files changed, 64 insertions(+), 15 deletions(-) diff --git a/pkg/skaffold/instrumentation/meter.go b/pkg/skaffold/instrumentation/meter.go index c73ec256994..ae662d3eef8 100644 --- a/pkg/skaffold/instrumentation/meter.go +++ b/pkg/skaffold/instrumentation/meter.go @@ -42,7 +42,7 @@ import ( "github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/cmd/statik" - // import embedded secret for uploading metrics + // import embedded secret for uploading metrics _ "github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/secret/statik" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/constants" "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest" @@ -61,7 +61,7 @@ type skaffoldMeter struct { Arch string PlatformType string Deployers []string - EnumFlags map[string]*flag.Flag + EnumFlags map[string]string Builders map[string]int SyncType map[string]bool DevIterations []devIteration @@ -79,7 +79,7 @@ var ( meter = skaffoldMeter{ OS: runtime.GOOS, Arch: runtime.GOARCH, - EnumFlags: map[string]*flag.Flag{}, + EnumFlags: map[string]string{}, Builders: map[string]int{}, SyncType: map[string]bool{}, DevIterations: []devIteration{}, @@ -148,7 +148,9 @@ func AddDevIterationErr(i int, errorCode proto.StatusCode) { } func AddFlag(flag *flag.Flag) { - meter.EnumFlags[flag.Name] = flag + if flag.Changed { + meter.EnumFlags[flag.Name] = flag.Value.String() + } } func ExportMetrics(exitCode int) error { diff --git a/pkg/skaffold/instrumentation/meter_test.go b/pkg/skaffold/instrumentation/meter_test.go index fbc99ccc12c..2281aa5eb2a 100644 --- a/pkg/skaffold/instrumentation/meter_test.go +++ b/pkg/skaffold/instrumentation/meter_test.go @@ -25,13 +25,19 @@ import ( "testing" "time" - "github.com/spf13/pflag" - "github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/cmd/statik" "github.com/GoogleContainerTools/skaffold/proto" "github.com/GoogleContainerTools/skaffold/testutil" ) +var testKey = `{ + "client_id": "test_id", + "client_secret": "test_secret", + "project_id": "test_project", + "refresh_token": "test_token", + "type": "authorized_user" +}` + func TestOfflineExportMetrics(t *testing.T) { startTime, _ := time.Parse(time.ANSIC, "Mon Jan 2 15:04:05 -0700 MST 2006") validMeter := skaffoldMeter{ @@ -41,20 +47,14 @@ func TestOfflineExportMetrics(t *testing.T) { Arch: "test arch", OS: "test os", Builders: map[string]int{"docker": 1, "buildpacks": 1}, - EnumFlags: map[string]*pflag.Flag{"test": {Name: "test", Shorthand: "t"}}, + EnumFlags: map[string]string{"test": "test_value"}, StartTime: startTime, Duration: time.Minute, } validMeterBytes, _ := json.Marshal(validMeter) fs := &testutil.FakeFileSystem{ Files: map[string][]byte{ - "/keys.json": []byte(`{ - "client_id": "test_id", - "client_secret": "test_secret", - "project_id": "test_project", - "refresh_token": "test_token", - "type": "authorized_user" - }`), + "/keys.json": []byte(testKey), }, } @@ -83,7 +83,7 @@ func TestOfflineExportMetrics(t *testing.T) { PlatformType: "test platform", Deployers: []string{"test helm", "test kpt"}, SyncType: map[string]bool{"manual": true}, - EnumFlags: map[string]*pflag.Flag{"test_run": {Name: "test_run", Shorthand: "r"}}, + EnumFlags: map[string]string{"test_run": "test_run_value"}, ErrorCode: proto.StatusCode_BUILD_CANCELLED, StartTime: startTime.Add(time.Hour * 24 * 30), Duration: time.Minute, @@ -130,3 +130,50 @@ func TestOfflineExportMetrics(t *testing.T) { }) } } + +func TestInitCloudMonitoring(t *testing.T) { + tests := []struct { + name string + fileSystem *testutil.FakeFileSystem + pusherIsNil bool + shouldError bool + }{ + { + name: "if key present pusher is not nil", + fileSystem: &testutil.FakeFileSystem{ + Files: map[string][]byte{"/keys.json": []byte(testKey)}, + }, + }, + { + name: "key not present returns nill err", + fileSystem: &testutil.FakeFileSystem{ + Files: map[string][]byte{}, + }, + pusherIsNil: true, + }, + { + name: "credentials without project_id returns an error", + fileSystem: &testutil.FakeFileSystem{ + Files: map[string][]byte{ + "/keys.json": []byte(`{ + "client_id": "test_id", + "client_secret": "test_secret", + "refresh_token": "test_token", + "type": "authorized_user" + }`, + )}, + }, + shouldError: true, + pusherIsNil: true, + }, + } + for _, test := range tests { + testutil.Run(t, test.name, func(t *testutil.T) { + t.Override(&statik.FS, func() (http.FileSystem, error) { return test.fileSystem, nil }) + + p, err := initCloudMonitoringExporterMetrics() + + t.CheckErrorAndDeepEqual(test.shouldError, err, test.pusherIsNil || test.shouldError, p == nil) + }) + } +} From ae04b18bee11991aac5982caaa8c97c08e6377f1 Mon Sep 17 00:00:00 2001 From: Isaac Duarte Date: Thu, 17 Dec 2020 15:13:42 -0500 Subject: [PATCH 2/2] test that otel metrics are created --- go.mod | 3 +- go.sum | 9 + pkg/skaffold/instrumentation/meter.go | 36 ++-- pkg/skaffold/instrumentation/meter_test.go | 229 +++++++++++++++++---- 4 files changed, 218 insertions(+), 59 deletions(-) diff --git a/go.mod b/go.mod index e01080676e5..b41a77b0172 100644 --- a/go.mod +++ b/go.mod @@ -68,9 +68,10 @@ require ( github.com/spf13/pflag v1.0.5 github.com/tektoncd/pipeline v0.5.1-0.20190731183258-9d7e37e85bf8 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9 go.opentelemetry.io/otel v0.13.0 + go.opentelemetry.io/otel/exporters/stdout v0.13.0 go.opentelemetry.io/otel/sdk v0.13.0 + golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9 golang.org/x/mod v0.3.0 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb // indirect golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58 diff --git a/go.sum b/go.sum index 3278006c9b1..1648db92a0d 100644 --- a/go.sum +++ b/go.sum @@ -1274,10 +1274,19 @@ go.opentelemetry.io/otel v0.13.0 h1:2isEnyzjjJZq6r2EKMsFj4TxiQiexsM04AVhwbR/oBA= go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY= go.opentelemetry.io/otel v0.14.0 h1:YFBEfjCk9MTjaytCNSUkp9Q8lF7QJezA06T71FbQxLQ= go.opentelemetry.io/otel v0.14.0/go.mod h1:vH5xEuwy7Rts0GNtsCW3HYQoZDY+OmBJ6t1bFGGlxgw= +go.opentelemetry.io/otel v0.15.0 h1:CZFy2lPhxd4HlhZnYK8gRyDotksO3Ip9rBweY1vVYJw= +go.opentelemetry.io/otel v0.15.0/go.mod h1:e4GKElweB8W2gWUqbghw0B8t5MCTccc9212eNHnOHwA= +go.opentelemetry.io/otel/exporters/stdout v0.13.0 h1:A+XiGIPQbGoJoBOJfKAKnZyiUSjSWvL3XWETUvtom5k= +go.opentelemetry.io/otel/exporters/stdout v0.13.0/go.mod h1:JJt8RpNY6K+ft9ir3iKpceCvT/rhzJXEExGrWFCbv1o= +go.opentelemetry.io/otel/exporters/stdout v0.14.0/go.mod h1:KG9w470+KbZZexYbC/g3TPKgluS0VgBJHh4KlnJpG18= +go.opentelemetry.io/otel/exporters/stdout v0.15.0 h1:/i7NvRnB+L7R/uxwpfolovicyBFnFa527NBs2yIhPUo= +go.opentelemetry.io/otel/exporters/stdout v0.15.0/go.mod h1:1d+FA51tyW9NDD0VXUsk5K5S3LAOt9GBWU3TNelHhxA= go.opentelemetry.io/otel/sdk v0.13.0 h1:4VCfpKamZ8GtnepXxMRurSpHpMKkcxhtO33z1S4rGDQ= go.opentelemetry.io/otel/sdk v0.13.0/go.mod h1:dKvLH8Uu8LcEPlSAUsfW7kMGaJBhk/1NYvpPZ6wIMbU= go.opentelemetry.io/otel/sdk v0.14.0 h1:Pqgd85y5XhyvHQlOxkKW+FD4DAX7AoeaNIDKC2VhfHQ= go.opentelemetry.io/otel/sdk v0.14.0/go.mod h1:kGO5pEMSNqSJppHAm8b73zztLxB5fgDQnD56/dl5xqE= +go.opentelemetry.io/otel/sdk v0.15.0 h1:Hf2dl1Ad9Hn03qjcAuAq51GP5Pv1SV5puIkS2nRhdd8= +go.opentelemetry.io/otel/sdk v0.15.0/go.mod h1:Qudkwgq81OcA9GYVlbyZ62wkLieeS1eWxIL0ufxgwoc= go.starlark.net v0.0.0-20190528202925-30ae18b8564f/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= diff --git a/pkg/skaffold/instrumentation/meter.go b/pkg/skaffold/instrumentation/meter.go index ae662d3eef8..9a2272a7b62 100644 --- a/pkg/skaffold/instrumentation/meter.go +++ b/pkg/skaffold/instrumentation/meter.go @@ -71,8 +71,8 @@ type skaffoldMeter struct { } type devIteration struct { - intent string - errorCode proto.StatusCode + Intent string + ErrorCode proto.StatusCode } var ( @@ -92,6 +92,7 @@ var ( meteredCommands = util.NewStringSet() doesBuild = util.NewStringSet() doesDeploy = util.NewStringSet() + initExporter = initCloudMonitoringExporterMetrics isOnline bool ) @@ -113,11 +114,7 @@ func init() { func InitMeterFromConfig(config *latest.SkaffoldConfig) { meter.PlatformType = yamltags.GetYamlTag(config.Build.BuildType) for _, artifact := range config.Pipeline.Build.Artifacts { - if _, ok := meter.Builders[yamltags.GetYamlTag(artifact.ArtifactType)]; ok { - meter.Builders[yamltags.GetYamlTag(artifact.ArtifactType)]++ - } else { - meter.Builders[yamltags.GetYamlTag(artifact.ArtifactType)] = 1 - } + meter.Builders[yamltags.GetYamlTag(artifact.ArtifactType)]++ if artifact.Sync != nil { meter.SyncType[yamltags.GetYamlTag(artifact.Sync)] = true } @@ -137,14 +134,14 @@ func SetErrorCode(errorCode proto.StatusCode) { } func AddDevIteration(intent string) { - meter.DevIterations = append(meter.DevIterations, devIteration{intent: intent}) + meter.DevIterations = append(meter.DevIterations, devIteration{Intent: intent}) } func AddDevIterationErr(i int, errorCode proto.StatusCode) { if len(meter.DevIterations) == i { - meter.DevIterations = append(meter.DevIterations, devIteration{intent: "error"}) + meter.DevIterations = append(meter.DevIterations, devIteration{Intent: "error"}) } - meter.DevIterations[i].errorCode = errorCode + meter.DevIterations[i].ErrorCode = errorCode } func AddFlag(flag *flag.Flag) { @@ -170,7 +167,7 @@ func ExportMetrics(exitCode int) error { func exportMetrics(ctx context.Context, filename string, meter skaffoldMeter) error { logrus.Debug("exporting metrics") - p, err := initCloudMonitoringExporterMetrics() + p, err := initExporter() if p == nil { return err } @@ -191,11 +188,13 @@ func exportMetrics(ctx context.Context, filename string, meter skaffoldMeter) er return ioutil.WriteFile(filename, b, 0666) } + start := time.Now() p.Start() for _, m := range meters { createMetrics(ctx, m) } p.Stop() + logrus.Debugf("metrics uploading complete in %s", time.Since(start).String()) if fileExists { return os.Remove(filename) @@ -223,8 +222,8 @@ func initCloudMonitoringExporterMetrics() (*push.Controller, error) { var c creds err = json.Unmarshal(b, &c) - if err != nil { - return nil, fmt.Errorf("error unmarsharling metrics credentials: %v", err) + if c.ProjectID == "" || err != nil { + return nil, fmt.Errorf("no project id found in metrics credentials") } formatter := func(desc *metric.Descriptor) string { @@ -297,14 +296,11 @@ func commandMetrics(ctx context.Context, meter skaffoldMeter, m metric.Meter, ra counts := make(map[string]map[proto.StatusCode]int) for _, iteration := range meter.DevIterations { - if _, ok := counts[iteration.intent]; !ok { - counts[iteration.intent] = make(map[proto.StatusCode]int) - } - m := counts[iteration.intent] - if _, ok := m[iteration.errorCode]; !ok { - m[iteration.errorCode] = 0 + if _, ok := counts[iteration.Intent]; !ok { + counts[iteration.Intent] = make(map[proto.StatusCode]int) } - m[iteration.errorCode]++ + m := counts[iteration.Intent] + m[iteration.ErrorCode]++ } for intention, errorCounts := range counts { for errorCode, count := range errorCounts { diff --git a/pkg/skaffold/instrumentation/meter_test.go b/pkg/skaffold/instrumentation/meter_test.go index 2281aa5eb2a..5b1af051b8b 100644 --- a/pkg/skaffold/instrumentation/meter_test.go +++ b/pkg/skaffold/instrumentation/meter_test.go @@ -19,12 +19,18 @@ package instrumentation import ( "context" "encoding/json" + "fmt" "io/ioutil" "net/http" "os" + "strconv" + "strings" "testing" "time" + "go.opentelemetry.io/otel/exporters/stdout" + "go.opentelemetry.io/otel/sdk/metric/controller/push" + "github.com/GoogleContainerTools/skaffold/cmd/skaffold/app/cmd/statik" "github.com/GoogleContainerTools/skaffold/proto" "github.com/GoogleContainerTools/skaffold/testutil" @@ -38,20 +44,36 @@ var testKey = `{ "type": "authorized_user" }` -func TestOfflineExportMetrics(t *testing.T) { +func TestExportMetrics(t *testing.T) { startTime, _ := time.Parse(time.ANSIC, "Mon Jan 2 15:04:05 -0700 MST 2006") - validMeter := skaffoldMeter{ + buildMeter := skaffoldMeter{ Command: "build", BuildArtifacts: 5, Version: "vTest.0", Arch: "test arch", OS: "test os", + Deployers: []string{"test kubectl"}, Builders: map[string]int{"docker": 1, "buildpacks": 1}, EnumFlags: map[string]string{"test": "test_value"}, StartTime: startTime, Duration: time.Minute, } - validMeterBytes, _ := json.Marshal(validMeter) + devMeter := skaffoldMeter{ + Command: "dev", + Version: "vTest.1", + Arch: "test arch 2", + OS: "test os 2", + PlatformType: "test platform", + Deployers: []string{"test helm", "test kpt"}, + SyncType: map[string]bool{"manual": true}, + EnumFlags: map[string]string{"test_run": "test_run_value"}, + Builders: map[string]int{"kustomize": 3, "buildpacks": 2}, + DevIterations: []devIteration{{"sync", 0}, {"build", 400}, {"build", 0}, {"sync", 200}, {"deploy", 0}}, + ErrorCode: proto.StatusCode_BUILD_CANCELLED, + StartTime: startTime.Add(time.Hour * 24 * 30), + Duration: time.Minute * 2, + } + devBuildBytes, _ := json.Marshal([]skaffoldMeter{buildMeter, devMeter}) fs := &testutil.FakeFileSystem{ Files: map[string][]byte{ "/keys.json": []byte(testKey), @@ -62,70 +84,86 @@ func TestOfflineExportMetrics(t *testing.T) { name string meter skaffoldMeter savedMetrics []byte - shouldSkip bool shouldFailUnmarshal bool + isOnline bool }{ - { - name: "skips exporting if command is not set", - shouldSkip: true, - }, { name: "saves meter to a new file", - meter: validMeter, + meter: buildMeter, }, { - name: "meter is appended to previously saved metrics", - meter: skaffoldMeter{ - Command: "dev", - Version: "vTest.1", - Arch: "test arch 2", - OS: "test os 2", - PlatformType: "test platform", - Deployers: []string{"test helm", "test kpt"}, - SyncType: map[string]bool{"manual": true}, - EnumFlags: map[string]string{"test_run": "test_run_value"}, - ErrorCode: proto.StatusCode_BUILD_CANCELLED, - StartTime: startTime.Add(time.Hour * 24 * 30), - Duration: time.Minute, - }, - savedMetrics: validMeterBytes, + name: "meter is appended to previously saved metrics", + meter: devMeter, + savedMetrics: devBuildBytes, }, { name: "meter does not re-save invalid metrics", - meter: validMeter, + meter: buildMeter, savedMetrics: []byte("[{\"Command\":\"run\", Invalid\": 10000000000010202301230}]"), shouldFailUnmarshal: true, }, + { + name: "test creating builder otel metrics", + meter: buildMeter, + isOnline: true, + }, + { + name: "test creating dev otel metrics", + meter: devMeter, + isOnline: true, + }, + { + name: "test otel metrics include offline metrics", + meter: devMeter, + savedMetrics: devBuildBytes, + isOnline: true, + }, } for _, test := range tests { testutil.Run(t, test.name, func(t *testutil.T) { - t.Override(&isOnline, false) - t.Override(&statik.FS, func() (http.FileSystem, error) { return fs, nil }) - filename := "metrics" - tmp := t.NewTempDir() + var actual []skaffoldMeter var savedMetrics []skaffoldMeter - _ = json.Unmarshal(test.savedMetrics, &savedMetrics) - if len(test.savedMetrics) > 0 { - err := ioutil.WriteFile(tmp.Path(filename), test.savedMetrics, 0666) + tmp := t.NewTempDir() + filename := "metrics" + openTelFilename := "otel_metrics" + + t.Override(&statik.FS, func() (http.FileSystem, error) { return fs, nil }) + t.Override(&isOnline, test.isOnline) + + if test.isOnline { + tmpFile, err := os.OpenFile(tmp.Path(openTelFilename), os.O_RDWR|os.O_CREATE, os.ModePerm) if err != nil { t.Error(err) } + t.Override(&initExporter, func() (*push.Controller, error) { + return stdout.InstallNewPipeline([]stdout.Option{ + stdout.WithQuantiles([]float64{0.5}), + stdout.WithPrettyPrint(), + stdout.WithWriter(tmpFile), + }, nil) + }) } + if len(test.savedMetrics) > 0 { + json.Unmarshal(test.savedMetrics, &savedMetrics) + tmp.Write(filename, string(test.savedMetrics)) + } + _ = exportMetrics(context.Background(), tmp.Path(filename), test.meter) + b, err := ioutil.ReadFile(tmp.Path(filename)) - if test.shouldSkip { - _, err := os.Stat(filename) - t.CheckDeepEqual(true, os.IsNotExist(err)) - } else { - b, _ := ioutil.ReadFile(tmp.Path(filename)) - var actual []skaffoldMeter + if !test.isOnline { _ = json.Unmarshal(b, &actual) expected := append(savedMetrics, test.meter) if test.shouldFailUnmarshal { expected = []skaffoldMeter{test.meter} } t.CheckDeepEqual(expected, actual) + } else { + t.CheckDeepEqual(true, os.IsNotExist(err)) + b, err := ioutil.ReadFile(tmp.Path(openTelFilename)) + t.CheckError(false, err) + checkOutput(t, append(savedMetrics, test.meter), b) } }) } @@ -177,3 +215,118 @@ func TestInitCloudMonitoring(t *testing.T) { }) } } + +func checkOutput(t *testutil.T, meters []skaffoldMeter, b []byte) { + osCount := make(map[interface{}]int) + versionCount := make(map[interface{}]int) + archCount := make(map[interface{}]int) + durationCount := make(map[interface{}]int) + commandCount := make(map[interface{}]int) + errorCount := make(map[interface{}]int) + builders := make(map[interface{}]int) + devIterations := make(map[interface{}]int) + deployers := make(map[interface{}]int) + + testMaps := []map[interface{}]int{ + osCount, versionCount, archCount, durationCount, commandCount, errorCount, builders, devIterations, deployers} + + for _, meter := range meters { + osCount[meter.OS]++ + versionCount[meter.Version]++ + durationCount[fmt.Sprintf("%s:%f", meter.Command, meter.Duration.Seconds())]++ + archCount[meter.Arch]++ + commandCount[meter.Command]++ + errorCount[meter.ErrorCode]++ + + if doesBuild.Contains(meter.Command) { + for k, v := range meter.Builders { + builders[k] += v + } + } + if meter.Command == "dev" { + for _, devI := range meter.DevIterations { + devIterations[devI]++ + } + } + if doesDeploy.Contains(meter.Command) { + for _, d := range meter.Deployers { + deployers[d]++ + } + } + } + + var lines []*line + json.Unmarshal(b, &lines) + + for _, l := range lines { + l.initLine() + switch l.Name { + case "launches": + archCount[l.Labels["arch"]]-- + osCount[l.Labels["os"]]-- + versionCount[l.Labels["version"]]-- + e, _ := strconv.Atoi(l.Labels["error"]) + if e == int(proto.StatusCode_OK) { + errorCount[proto.StatusCode(e)]-- + } + case "launch/duration": + durationCount[fmt.Sprintf("%s:%f", l.Labels["command"], l.value().(float64))]-- + case "artifacts": + builders[l.Labels["builder"]] -= int(l.value().(float64)) - 1 + case "builders": + builders[l.Labels["builder"]]-- + case "deployer": + deployers[l.Labels["deployer"]]-- + case "dev/iterations": + e, _ := strconv.Atoi(l.Labels["error"]) + devIterations[devIteration{l.Labels["intent"], proto.StatusCode(e)}]-- + case "errors": + e, _ := strconv.Atoi(l.Labels["error"]) + errorCount[proto.StatusCode(e)]-- + default: + switch { + case meteredCommands.Contains(l.Name): + commandCount[l.Name]-- + default: + t.Error("unexpected metric with name", l.Name) + } + } + } + + for _, m := range testMaps { + for n, v := range m { + t.Logf("Checking %s", n) + t.CheckDeepEqual(0, v) + } + } +} + +// Derived from go.opentelemetry.io/otel/exporters/stdout/metric.go +type line struct { + Name string `json:"Name"` + Count interface{} `json:"Count,omitempty"` + Quantiles []quantile `json:"Quantiles,omitempty"` + Labels map[string]string +} + +type quantile struct { + Quantile interface{} `json:"Quantile"` + Value interface{} `json:"Value"` +} + +func (l *line) initLine() { + l.Labels = make(map[string]string) + leftBracket := strings.Index(l.Name, "{") + rightBracket := strings.Index(l.Name, "}") + + labels := strings.Split(l.Name[leftBracket+1:rightBracket], ",")[1:] + for _, lbl := range labels { + ll := strings.Split(lbl, "=") + l.Labels[ll[0]] = ll[1] + } + l.Name = l.Name[:leftBracket] +} + +func (l *line) value() interface{} { + return l.Quantiles[0].Value +}