From b577f116bc88d305d56958c4f0123c612716d3d6 Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Mon, 1 Aug 2022 16:44:56 -0500 Subject: [PATCH 01/11] feat: added profile/ to output pprof style constraint profiler --- frontend/cs/r1cs/builder.go | 2 + frontend/cs/scs/builder.go | 3 +- go.mod | 1 + go.sum | 2 + profile/profile.go | 248 ++++++++++++++++++++++++++++++++++++ 5 files changed, 255 insertions(+), 1 deletion(-) create mode 100644 profile/profile.go diff --git a/frontend/cs/r1cs/builder.go b/frontend/cs/r1cs/builder.go index 2ef6a05129..adab8258da 100644 --- a/frontend/cs/r1cs/builder.go +++ b/frontend/cs/r1cs/builder.go @@ -46,6 +46,7 @@ import ( tinyfieldr1cs "github.com/consensys/gnark/internal/tinyfield/cs" "github.com/consensys/gnark/internal/utils" "github.com/consensys/gnark/logger" + "github.com/consensys/gnark/profile" ) // NewBuilder returns a new R1CS compiler @@ -174,6 +175,7 @@ func newR1C(_l, _r, _o frontend.Variable) compiled.R1C { } func (system *r1cs) addConstraint(r1c compiled.R1C, debugID ...int) { + profile.Sample() system.Constraints = append(system.Constraints, r1c) if len(debugID) > 0 { system.MDebug[len(system.Constraints)-1] = debugID[0] diff --git a/frontend/cs/scs/builder.go b/frontend/cs/scs/builder.go index bd244aa5a9..10fbb371fb 100644 --- a/frontend/cs/scs/builder.go +++ b/frontend/cs/scs/builder.go @@ -46,6 +46,7 @@ import ( tinyfieldr1cs "github.com/consensys/gnark/internal/tinyfield/cs" "github.com/consensys/gnark/internal/utils" "github.com/consensys/gnark/logger" + "github.com/consensys/gnark/profile" ) func NewBuilder(field *big.Int, config frontend.CompileConfig) (frontend.Builder, error) { @@ -87,7 +88,7 @@ func newBuilder(field *big.Int, config frontend.CompileConfig) *scs { // addPlonkConstraint creates a constraint of the for al+br+clr+k=0 //func (system *SparseR1CS) addPlonkConstraint(l, r, o frontend.Variable, cidl, cidr, cidm1, cidm2, cido, k int, debugID ...int) { func (system *scs) addPlonkConstraint(l, r, o compiled.Term, cidl, cidr, cidm1, cidm2, cido, k int, debugID ...int) { - + profile.Sample() if len(debugID) > 0 { system.MDebug[len(system.Constraints)] = debugID[0] } else if debug.Debug { diff --git a/go.mod b/go.mod index 48352d00b6..835a6361d6 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/consensys/gnark-crypto v0.7.1-0.20220603201101-938eff486457 github.com/fxamacker/cbor/v2 v2.2.0 github.com/google/go-cmp v0.5.8 + github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 github.com/leanovate/gopter v0.2.9 github.com/rs/zerolog v1.26.1 github.com/stretchr/testify v1.7.1 diff --git a/go.sum b/go.sum index 2888afb28c..70e099c4d9 100644 --- a/go.sum +++ b/go.sum @@ -14,6 +14,8 @@ github.com/fxamacker/cbor/v2 v2.2.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrt github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1 h1:8pyqKJvrJqUYaKS851Ule26pwWvey6IDMiczaBLDKLQ= +github.com/google/pprof v0.0.0-20220729232143-a41b82acbcb1/go.mod h1:gSuNB+gJaOiQKLEZ+q+PK9Mq3SOzhRcw2GsGS/FhYDk= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= diff --git a/profile/profile.go b/profile/profile.go new file mode 100644 index 0000000000..96c6eb6c9d --- /dev/null +++ b/profile/profile.go @@ -0,0 +1,248 @@ +// partially derived from: https://github.com/pkg/profile +// Original copyright: +// Copyright (c) 2013 Dave Cheney. All rights reserved. + +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: + +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. + +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package profile + +import ( + "io/ioutil" + "os" + "os/signal" + "path/filepath" + "runtime" + "strings" + "sync" + "sync/atomic" + + "github.com/consensys/gnark/logger" + "github.com/google/pprof/profile" +) + +// Profile represents an active constraint system profiling session. +type Profile struct { + // path holds the base path where various profiling files are written. + // If blank, the base path will be generated by ioutil.TempDir. + path string + + // closer holds a cleanup function that run after each profile + closer func() + + // stopped records if a call to profile.Stop has been made + stopped uint32 + + // noShutdownHook controls whether the profiling package should + // hook SIGINT to write profiles cleanly. + noShutdownHook bool + + // actual pprof profile struct + pprof profile.Profile + + functions map[string]*profile.Function + locations map[uint64]*profile.Location + + onceSetName sync.Once +} + +// NoShutdownHook controls whether the profiling package should +// hook SIGINT to write profiles cleanly. +// Programs with more sophisticated signal handling should set +// this to true and ensure the Stop() function returned from Start() +// is called during shutdown. +func NoShutdownHook(p *Profile) { p.noShutdownHook = true } + +// ProfilePath controls the base path where various profiling +// files are written. If blank, the base path will be generated +// by ioutil.TempDir. +func ProfilePath(path string) func(*Profile) { + return func(p *Profile) { + p.path = path + } +} + +func (p *Profile) Stop() { + if !atomic.CompareAndSwapUint32(&p.stopped, 0, 1) { + // someone has already called close + return + } + p.closer() + atomic.StoreUint32(&started, 0) +} + +// started is non zero if a profile is running. +var started uint32 + +// this is a global var, need to improve that; +// not a big risk given than circuit compilation runs in a single go routine, but ... ugly. +var _profile *Profile + +func Start(options ...func(*Profile)) interface { + Stop() +} { + log := logger.Logger() + if !atomic.CompareAndSwapUint32(&started, 0, 1) { + log.Fatal().Msg("Start() already called") + } + prof := Profile{ + functions: make(map[string]*profile.Function), + locations: make(map[uint64]*profile.Location), + } + for _, option := range options { + option(&prof) + } + + path, err := func() (string, error) { + if p := prof.path; p != "" { + return p, os.MkdirAll(p, 0777) + } + return ioutil.TempDir("", "profile") + }() + + if err != nil { + log.Fatal().Err(err).Msg("could not create initial output directory") + } + + fn := filepath.Join(path, "gnark.pprof") + f, err := os.Create(fn) + if err != nil { + log.Fatal().Err(err).Msg("could not create gnark profile") + } + log.Debug().Msgf("gnark profiling enabled, %s", fn) + + prof.pprof.SampleType = []*profile.ValueType{{ + Type: "constraints", + Unit: "count", + }} + + prof.closer = func() { + if err := prof.pprof.Write(f); err != nil { + log.Error().Err(err).Msg("writing profile") + } + f.Close() + log.Debug().Msgf("gnark profiling disabled, %s", fn) + } + + if !prof.noShutdownHook { + go func() { + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + <-c + + log.Info().Msg("caught interrupt, stopping profiles") + prof.Stop() + + os.Exit(0) + }() + } + + _profile = &prof + return &prof +} + +func Sample() { + if s := atomic.LoadUint32(&started); s == 0 { + return // do nothing, no profile running. + } + + sample := &profile.Sample{ + Value: []int64{1}, + } + + // 1 collect the stack . + // Ask runtime.Callers for up to 10 pcs + pc := make([]uintptr, 20) + n := runtime.Callers(3, pc) + if n == 0 { + // No pcs available. Stop now. + // This can happen if the first argument to runtime.Callers is large. + return + } + pc = pc[:n] // pass only valid pcs to runtime.CallersFrames + frames := runtime.CallersFrames(pc) + // Loop to get frames. + // A fixed number of pcs can expand to an indefinite number of Frames. + for { + frame, more := frames.Next() + if strings.HasSuffix(frame.Function, ".func1") { + // TODO @gbotrel filter lambda func better + continue + } + + // TODO @gbotrel [...] -> from generics display poorly in pprof + // fix may be coming in go 1.19 + frame.Function = strings.Replace(frame.Function, "[...]", "[T]", -1) + + sample.Location = append(sample.Location, _profile.getLocation(&frame)) + + if !more { + break + } + if strings.HasSuffix(frame.Function, "Define") { + _profile.onceSetName.Do(func() { + fe := strings.Split(frame.Function, "/") + circuitName := strings.TrimSuffix(fe[len(fe)-1], ".Define") + _profile.pprof.Mapping = []*profile.Mapping{ + {ID: 1, File: circuitName}, + } + }) + break + } + } + + _profile.pprof.Sample = append(_profile.pprof.Sample, sample) +} + +func (p *Profile) getLocation(frame *runtime.Frame) *profile.Location { + + // location + // locationID := frame.File + strconv.Itoa(frame.Line) + l, ok := p.locations[uint64(frame.PC)] + if !ok { + // first let's see if we have the function. + f, ok := p.functions[frame.File+frame.Function] + if !ok { + fe := strings.Split(frame.Function, "/") + fName := fe[len(fe)-1] + f = &profile.Function{ + ID: uint64(len(p.functions) + 1), + Name: fName, + SystemName: frame.Function, + Filename: frame.File, + } + + p.functions[frame.File+frame.Function] = f + p.pprof.Function = append(p.pprof.Function, f) + } + + l = &profile.Location{ + ID: uint64(len(p.locations) + 1), + Line: []profile.Line{{Function: f, Line: int64(frame.Line)}}, + } + p.locations[uint64(frame.PC)] = l + p.pprof.Location = append(p.pprof.Location, l) + } + + return l +} From f5873507833c529dc052f9a75dd203b1b40426ca Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Tue, 2 Aug 2022 13:20:31 -0500 Subject: [PATCH 02/11] feat: gnark/profile allows multiple overlapping profiling sessions --- frontend/cs/r1cs/builder.go | 2 +- frontend/cs/scs/builder.go | 2 +- profile/profile.go | 232 ++++++++++++------------------------ profile/profile_worker.go | 125 +++++++++++++++++++ 4 files changed, 205 insertions(+), 156 deletions(-) create mode 100644 profile/profile_worker.go diff --git a/frontend/cs/r1cs/builder.go b/frontend/cs/r1cs/builder.go index adab8258da..0b4d2ffeeb 100644 --- a/frontend/cs/r1cs/builder.go +++ b/frontend/cs/r1cs/builder.go @@ -175,7 +175,7 @@ func newR1C(_l, _r, _o frontend.Variable) compiled.R1C { } func (system *r1cs) addConstraint(r1c compiled.R1C, debugID ...int) { - profile.Sample() + profile.RecordConstraint() system.Constraints = append(system.Constraints, r1c) if len(debugID) > 0 { system.MDebug[len(system.Constraints)-1] = debugID[0] diff --git a/frontend/cs/scs/builder.go b/frontend/cs/scs/builder.go index 10fbb371fb..08d15886ec 100644 --- a/frontend/cs/scs/builder.go +++ b/frontend/cs/scs/builder.go @@ -88,7 +88,7 @@ func newBuilder(field *big.Int, config frontend.CompileConfig) *scs { // addPlonkConstraint creates a constraint of the for al+br+clr+k=0 //func (system *SparseR1CS) addPlonkConstraint(l, r, o frontend.Variable, cidl, cidr, cidm1, cidm2, cido, k int, debugID ...int) { func (system *scs) addPlonkConstraint(l, r, o compiled.Term, cidl, cidr, cidm1, cidm2, cido, k int, debugID ...int) { - profile.Sample() + profile.RecordConstraint() if len(debugID) > 0 { system.MDebug[len(system.Constraints)] = debugID[0] } else if debug.Debug { diff --git a/profile/profile.go b/profile/profile.go index 96c6eb6c9d..b6dece84e5 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -1,36 +1,11 @@ -// partially derived from: https://github.com/pkg/profile -// Original copyright: -// Copyright (c) 2013 Dave Cheney. All rights reserved. - -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: - -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. - -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - +// Package profile provides a simple way to generate pprof compatible gnark circuit profile. +// +// Since the gnark frontend compiler is not thread safe and operates in a single go-routine, +// this package is also NOT thread safe and is meant to be called in the same go-routine. package profile import ( - "io/ioutil" "os" - "os/signal" "path/filepath" "runtime" "strings" @@ -41,177 +16,126 @@ import ( "github.com/google/pprof/profile" ) +var ( + sessions []*Profile // active sessions + activeSessions uint32 +) + // Profile represents an active constraint system profiling session. type Profile struct { - // path holds the base path where various profiling files are written. - // If blank, the base path will be generated by ioutil.TempDir. - path string - - // closer holds a cleanup function that run after each profile - closer func() - - // stopped records if a call to profile.Stop has been made - stopped uint32 - - // noShutdownHook controls whether the profiling package should - // hook SIGINT to write profiles cleanly. - noShutdownHook bool + // defaults to ./gnark.pprof + // if blank, profiile is not written to disk + filePath string // actual pprof profile struct + // details on pprof format: https://github.com/google/pprof/blob/main/proto/README.md pprof profile.Profile functions map[string]*profile.Function locations map[uint64]*profile.Location onceSetName sync.Once -} -// NoShutdownHook controls whether the profiling package should -// hook SIGINT to write profiles cleanly. -// Programs with more sophisticated signal handling should set -// this to true and ensure the Stop() function returned from Start() -// is called during shutdown. -func NoShutdownHook(p *Profile) { p.noShutdownHook = true } + chDone chan struct{} +} -// ProfilePath controls the base path where various profiling -// files are written. If blank, the base path will be generated -// by ioutil.TempDir. +// ProfilePath controls the profile destination file. If blank, profile is not written. +// +// Defaults to ./gnark.pprof. func ProfilePath(path string) func(*Profile) { return func(p *Profile) { - p.path = path - } -} - -func (p *Profile) Stop() { - if !atomic.CompareAndSwapUint32(&p.stopped, 0, 1) { - // someone has already called close - return + p.filePath = path } - p.closer() - atomic.StoreUint32(&started, 0) } -// started is non zero if a profile is running. -var started uint32 +// Start creates a new active profiling session. When Stop() is called, this session is removed from +// active profiling sessions and may be serialized to disk as a pprof compatible file (see ProfilePath option). +// +// All calls to profile.Start() and Stop() are meant to be executed in the same go routine (frontend.Compile). +// +// It is allowed to create multiple overlapping profiling sessions in one circuit. +func Start(options ...func(*Profile)) *Profile { -// this is a global var, need to improve that; -// not a big risk given than circuit compilation runs in a single go routine, but ... ugly. -var _profile *Profile + // start the worker first time a profiling session starts. + onceInit.Do(func() { + go worker() + }) -func Start(options ...func(*Profile)) interface { - Stop() -} { - log := logger.Logger() - if !atomic.CompareAndSwapUint32(&started, 0, 1) { - log.Fatal().Msg("Start() already called") - } prof := Profile{ functions: make(map[string]*profile.Function), locations: make(map[uint64]*profile.Location), + filePath: filepath.Join(".", "gnark.pprof"), + chDone: make(chan struct{}), } + prof.pprof.SampleType = []*profile.ValueType{{ + Type: "constraints", + Unit: "count", + }} + for _, option := range options { option(&prof) } - path, err := func() (string, error) { - if p := prof.path; p != "" { - return p, os.MkdirAll(p, 0777) - } - return ioutil.TempDir("", "profile") - }() - - if err != nil { - log.Fatal().Err(err).Msg("could not create initial output directory") + log := logger.Logger() + if prof.filePath == "" { + log.Warn().Msg("gnark profiling enabled [not writting to disk]") + } else { + log.Info().Str("path", prof.filePath).Msg("gnark profiling enabled") } - fn := filepath.Join(path, "gnark.pprof") - f, err := os.Create(fn) - if err != nil { - log.Fatal().Err(err).Msg("could not create gnark profile") - } - log.Debug().Msgf("gnark profiling enabled, %s", fn) + // add the session to active sessions + chCommands <- command{p: &prof} + atomic.AddUint32(&activeSessions, 1) - prof.pprof.SampleType = []*profile.ValueType{{ - Type: "constraints", - Unit: "count", - }} + return &prof +} - prof.closer = func() { - if err := prof.pprof.Write(f); err != nil { - log.Error().Err(err).Msg("writing profile") - } - f.Close() - log.Debug().Msgf("gnark profiling disabled, %s", fn) +// Stop removes the profile from active session and may write the pprof file to disk. See ProfilePath option. +func (p *Profile) Stop() { + log := logger.Logger() + + if p.chDone == nil { + log.Fatal().Msg("gnark profile stopped multiple times") } - if !prof.noShutdownHook { - go func() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - <-c + // ask worker routine to remove ourselves from the active sessions + chCommands <- command{p: p, remove: true} - log.Info().Msg("caught interrupt, stopping profiles") - prof.Stop() + // wait for worker routine to remove us. + <-p.chDone + p.chDone = nil - os.Exit(0) - }() + // if filePath is set, serialize profile to disk in pprof format + if p.filePath != "" { + f, err := os.Create(p.filePath) + if err != nil { + log.Fatal().Err(err).Msg("could not create gnark profile") + } + if err := p.pprof.Write(f); err != nil { + log.Error().Err(err).Msg("writing profile") + } + f.Close() + log.Info().Str("path", p.filePath).Msg("gnark profiling disabled") + } else { + log.Warn().Msg("gnark profiling disabled [not writting to disk]") } - _profile = &prof - return &prof } -func Sample() { - if s := atomic.LoadUint32(&started); s == 0 { - return // do nothing, no profile running. - } - - sample := &profile.Sample{ - Value: []int64{1}, +// RecordConstraint add a sample (with count == 1) to all the active profiling sessions. +func RecordConstraint() { + if n := atomic.LoadUint32(&activeSessions); n == 0 { + return // do nothing, no active session. } - // 1 collect the stack . - // Ask runtime.Callers for up to 10 pcs + // collect the stack and send it async to the worker pc := make([]uintptr, 20) n := runtime.Callers(3, pc) if n == 0 { - // No pcs available. Stop now. - // This can happen if the first argument to runtime.Callers is large. return } - pc = pc[:n] // pass only valid pcs to runtime.CallersFrames - frames := runtime.CallersFrames(pc) - // Loop to get frames. - // A fixed number of pcs can expand to an indefinite number of Frames. - for { - frame, more := frames.Next() - if strings.HasSuffix(frame.Function, ".func1") { - // TODO @gbotrel filter lambda func better - continue - } - - // TODO @gbotrel [...] -> from generics display poorly in pprof - // fix may be coming in go 1.19 - frame.Function = strings.Replace(frame.Function, "[...]", "[T]", -1) - - sample.Location = append(sample.Location, _profile.getLocation(&frame)) - - if !more { - break - } - if strings.HasSuffix(frame.Function, "Define") { - _profile.onceSetName.Do(func() { - fe := strings.Split(frame.Function, "/") - circuitName := strings.TrimSuffix(fe[len(fe)-1], ".Define") - _profile.pprof.Mapping = []*profile.Mapping{ - {ID: 1, File: circuitName}, - } - }) - break - } - } - - _profile.pprof.Sample = append(_profile.pprof.Sample, sample) + pc = pc[:n] + chCommands <- command{pc: pc} } func (p *Profile) getLocation(frame *runtime.Frame) *profile.Location { diff --git a/profile/profile_worker.go b/profile/profile_worker.go new file mode 100644 index 0000000000..c035011b04 --- /dev/null +++ b/profile/profile_worker.go @@ -0,0 +1,125 @@ +package profile + +import ( + "runtime" + "strings" + "sync" + "sync/atomic" + + "github.com/google/pprof/profile" +) + +// since we are assuming usage of this package from a single go routine, this channel only has +// one "producer", and one "consumer". it's purpose is to guarantee the order of execution of +// adding / removing a profiling session and sampling events, while enabling the caller +// (frontend.Compile) to sample the events asynchronously. +var chCommands = make(chan command, 100) +var onceInit sync.Once + +type command struct { + p *Profile + pc []uintptr + remove bool +} + +func worker() { + for c := range chCommands { + if c.p != nil { + if c.remove { + for i := 0; i < len(sessions); i++ { + if sessions[i] == c.p { + sessions[i] = sessions[len(sessions)-1] + sessions = sessions[:len(sessions)-1] + break + } + } + close(c.p.chDone) + + // decrement active sessions + atomic.AddUint32(&activeSessions, ^uint32(0)) + } else { + sessions = append(sessions, c.p) + } + continue + } + + // it's a sampling of event + collectSample(c.pc) + } + +} + +// collectSample must be called from the worker go routine +func collectSample(pc []uintptr) { + // for each session we may have a distinct sample, since ids of functions and locations may mismatch + samples := make([]*profile.Sample, len(sessions)) + for i := 0; i < len(samples); i++ { + samples[i] = &profile.Sample{Value: []int64{1}} // for now, we just collect new constraints count + } + + frames := runtime.CallersFrames(pc) + // Loop to get frames. + // A fixed number of pcs can expand to an indefinite number of Frames. + for { + frame, more := frames.Next() + + if strings.HasSuffix(frame.Function, ".func1") { + // TODO @gbotrel filter anonymous func better + continue + } + + // to avoid aving a location that concentrates 99% of the calls, we transfer the "addConstraint" + // occuring in Mul to the previous level in the stack + if strings.Contains(frame.Function, "github.com/consensys/gnark/frontend/cs/r1cs.(*r1cs).Mul") { + continue + } + + if strings.HasPrefix(frame.Function, "github.com/consensys/gnark/frontend/cs/scs.(*scs).Mul") { + continue + } + + if strings.HasPrefix(frame.Function, "github.com/consensys/gnark/frontend/cs/scs.(*scs).split") { + continue + } + + // with scs.Builder (Plonk) Add and Sub always add a constraint --> we record the caller as the constraint adder + // but in the future we may record a different type of sample for these + if strings.HasPrefix(frame.Function, "github.com/consensys/gnark/frontend/cs/scs.(*scs).Add") { + continue + } + if strings.HasPrefix(frame.Function, "github.com/consensys/gnark/frontend/cs/scs.(*scs).Sub") { + continue + } + + // TODO @gbotrel [...] -> from generics display poorly in pprof + // https://github.com/golang/go/issues/54105 + frame.Function = strings.Replace(frame.Function, "[...]", "[T]", -1) + + for i := 0; i < len(samples); i++ { + samples[i].Location = append(samples[i].Location, sessions[i].getLocation(&frame)) + } + + if !more { + break + } + if strings.HasSuffix(frame.Function, ".Define") { + for i := 0; i < len(sessions); i++ { + sessions[i].onceSetName.Do(func() { + // once per profile session, we set the "name of the binary" + // here we grep the struct name where "Define" exist: hopefully the circuit Name + fe := strings.Split(frame.Function, "/") + circuitName := strings.TrimSuffix(fe[len(fe)-1], ".Define") + sessions[i].pprof.Mapping = []*profile.Mapping{ + {ID: 1, File: circuitName}, + } + }) + } + break + } + } + + for i := 0; i < len(sessions); i++ { + sessions[i].pprof.Sample = append(sessions[i].pprof.Sample, samples[i]) + } + +} From 3af4b03186f45faab41f3d336c31771562366b04 Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Tue, 2 Aug 2022 14:20:50 -0500 Subject: [PATCH 03/11] feat: added profile.Top() --- examples/rollup/circuit_test.go | 3 + profile/internal/graph/dotgraph.go | 494 ++++++++ profile/internal/graph/graph.go | 1170 +++++++++++++++++++ profile/internal/measurement/measurement.go | 293 +++++ profile/internal/report/report.go | 1093 +++++++++++++++++ profile/internal/report/synth.go | 39 + profile/profile.go | 34 +- 7 files changed, 3116 insertions(+), 10 deletions(-) create mode 100644 profile/internal/graph/dotgraph.go create mode 100644 profile/internal/graph/graph.go create mode 100644 profile/internal/measurement/measurement.go create mode 100644 profile/internal/report/report.go create mode 100644 profile/internal/report/synth.go diff --git a/examples/rollup/circuit_test.go b/examples/rollup/circuit_test.go index 0fc65549c7..4b51fc907c 100644 --- a/examples/rollup/circuit_test.go +++ b/examples/rollup/circuit_test.go @@ -22,6 +22,7 @@ import ( "github.com/consensys/gnark-crypto/ecc" "github.com/consensys/gnark/backend" "github.com/consensys/gnark/frontend" + "github.com/consensys/gnark/profile" "github.com/consensys/gnark/std/hash/mimc" "github.com/consensys/gnark/test" ) @@ -266,10 +267,12 @@ func TestCircuitFull(t *testing.T) { } // TODO full circuit has some unconstrained inputs, that's odd. + p := profile.Start() assert.ProverSucceeded( &rollupCircuit, &operator.witnesses, test.WithCurves(ecc.BN254), test.WithBackends(backend.GROTH16)) + p.Stop() } diff --git a/profile/internal/graph/dotgraph.go b/profile/internal/graph/dotgraph.go new file mode 100644 index 0000000000..6282bf9272 --- /dev/null +++ b/profile/internal/graph/dotgraph.go @@ -0,0 +1,494 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +import ( + "fmt" + "io" + "math" + "path/filepath" + "strings" + + "github.com/consensys/gnark/profile/internal/measurement" +) + +// DotAttributes contains details about the graph itself, giving +// insight into how its elements should be rendered. +type DotAttributes struct { + Nodes map[*Node]*DotNodeAttributes // A map allowing each Node to have its own visualization option +} + +// DotNodeAttributes contains Node specific visualization options. +type DotNodeAttributes struct { + Shape string // The optional shape of the node when rendered visually + Bold bool // If the node should be bold or not + Peripheries int // An optional number of borders to place around a node + URL string // An optional url link to add to a node + Formatter func(*NodeInfo) string // An optional formatter for the node's label +} + +// DotConfig contains attributes about how a graph should be +// constructed and how it should look. +type DotConfig struct { + Title string // The title of the DOT graph + LegendURL string // The URL to link to from the legend. + Labels []string // The labels for the DOT's legend + + FormatValue func(int64) string // A formatting function for values + Total int64 // The total weight of the graph, used to compute percentages +} + +const maxNodelets = 4 // Number of nodelets for labels (both numeric and non) + +// ComposeDot creates and writes a in the DOT format to the writer, using +// the configurations given. +func ComposeDot(w io.Writer, g *Graph, a *DotAttributes, c *DotConfig) { + builder := &builder{w, a, c} + + // Begin constructing DOT by adding a title and legend. + builder.start() + defer builder.finish() + builder.addLegend() + + if len(g.Nodes) == 0 { + return + } + + // Preprocess graph to get id map and find max flat. + nodeIDMap := make(map[*Node]int) + hasNodelets := make(map[*Node]bool) + + maxFlat := float64(abs64(g.Nodes[0].FlatValue())) + for i, n := range g.Nodes { + nodeIDMap[n] = i + 1 + if float64(abs64(n.FlatValue())) > maxFlat { + maxFlat = float64(abs64(n.FlatValue())) + } + } + + edges := EdgeMap{} + + // Add nodes and nodelets to DOT builder. + for _, n := range g.Nodes { + builder.addNode(n, nodeIDMap[n], maxFlat) + hasNodelets[n] = builder.addNodelets(n, nodeIDMap[n]) + + // Collect all edges. Use a fake node to support multiple incoming edges. + for _, e := range n.Out { + edges[&Node{}] = e + } + } + + // Add edges to DOT builder. Sort edges by frequency as a hint to the graph layout engine. + for _, e := range edges.Sort() { + builder.addEdge(e, nodeIDMap[e.Src], nodeIDMap[e.Dest], hasNodelets[e.Src]) + } +} + +// builder wraps an io.Writer and understands how to compose DOT formatted elements. +type builder struct { + io.Writer + attributes *DotAttributes + config *DotConfig +} + +// start generates a title and initial node in DOT format. +func (b *builder) start() { + graphname := "unnamed" + if b.config.Title != "" { + graphname = b.config.Title + } + fmt.Fprintln(b, `digraph "`+graphname+`" {`) + fmt.Fprintln(b, `node [style=filled fillcolor="#f8f8f8"]`) +} + +// finish closes the opening curly bracket in the constructed DOT buffer. +func (b *builder) finish() { + fmt.Fprintln(b, "}") +} + +// addLegend generates a legend in DOT format. +func (b *builder) addLegend() { + labels := b.config.Labels + if len(labels) == 0 { + return + } + title := labels[0] + fmt.Fprintf(b, `subgraph cluster_L { "%s" [shape=box fontsize=16`, escapeForDot(title)) + fmt.Fprintf(b, ` label="%s\l"`, strings.Join(escapeAllForDot(labels), `\l`)) + if b.config.LegendURL != "" { + fmt.Fprintf(b, ` URL="%s" target="_blank"`, b.config.LegendURL) + } + if b.config.Title != "" { + fmt.Fprintf(b, ` tooltip="%s"`, b.config.Title) + } + fmt.Fprintf(b, "] }\n") +} + +// addNode generates a graph node in DOT format. +func (b *builder) addNode(node *Node, nodeID int, maxFlat float64) { + flat, cum := node.FlatValue(), node.CumValue() + attrs := b.attributes.Nodes[node] + + // Populate label for node. + var label string + if attrs != nil && attrs.Formatter != nil { + label = attrs.Formatter(&node.Info) + } else { + label = multilinePrintableName(&node.Info) + } + + flatValue := b.config.FormatValue(flat) + if flat != 0 { + label = label + fmt.Sprintf(`%s (%s)`, + flatValue, + strings.TrimSpace(measurement.Percentage(flat, b.config.Total))) + } else { + label = label + "0" + } + cumValue := flatValue + if cum != flat { + if flat != 0 { + label = label + `\n` + } else { + label = label + " " + } + cumValue = b.config.FormatValue(cum) + label = label + fmt.Sprintf(`of %s (%s)`, + cumValue, + strings.TrimSpace(measurement.Percentage(cum, b.config.Total))) + } + + // Scale font sizes from 8 to 24 based on percentage of flat frequency. + // Use non linear growth to emphasize the size difference. + baseFontSize, maxFontGrowth := 8, 16.0 + fontSize := baseFontSize + if maxFlat != 0 && flat != 0 && float64(abs64(flat)) <= maxFlat { + fontSize += int(math.Ceil(maxFontGrowth * math.Sqrt(float64(abs64(flat))/maxFlat))) + } + + // Determine node shape. + shape := "box" + if attrs != nil && attrs.Shape != "" { + shape = attrs.Shape + } + + // Create DOT attribute for node. + attr := fmt.Sprintf(`label="%s" id="node%d" fontsize=%d shape=%s tooltip="%s (%s)" color="%s" fillcolor="%s"`, + label, nodeID, fontSize, shape, escapeForDot(node.Info.PrintableName()), cumValue, + dotColor(float64(node.CumValue())/float64(abs64(b.config.Total)), false), + dotColor(float64(node.CumValue())/float64(abs64(b.config.Total)), true)) + + // Add on extra attributes if provided. + if attrs != nil { + // Make bold if specified. + if attrs.Bold { + attr += ` style="bold,filled"` + } + + // Add peripheries if specified. + if attrs.Peripheries != 0 { + attr += fmt.Sprintf(` peripheries=%d`, attrs.Peripheries) + } + + // Add URL if specified. target="_blank" forces the link to open in a new tab. + if attrs.URL != "" { + attr += fmt.Sprintf(` URL="%s" target="_blank"`, attrs.URL) + } + } + + fmt.Fprintf(b, "N%d [%s]\n", nodeID, attr) +} + +// addNodelets generates the DOT boxes for the node tags if they exist. +func (b *builder) addNodelets(node *Node, nodeID int) bool { + var nodelets string + + // Populate two Tag slices, one for LabelTags and one for NumericTags. + var ts []*Tag + lnts := make(map[string][]*Tag) + for _, t := range node.LabelTags { + ts = append(ts, t) + } + for l, tm := range node.NumericTags { + for _, t := range tm { + lnts[l] = append(lnts[l], t) + } + } + + // For leaf nodes, print cumulative tags (includes weight from + // children that have been deleted). + // For internal nodes, print only flat tags. + flatTags := len(node.Out) > 0 + + // Select the top maxNodelets alphanumeric labels by weight. + SortTags(ts, flatTags) + if len(ts) > maxNodelets { + ts = ts[:maxNodelets] + } + for i, t := range ts { + w := t.CumValue() + if flatTags { + w = t.FlatValue() + } + if w == 0 { + continue + } + weight := b.config.FormatValue(w) + nodelets += fmt.Sprintf(`N%d_%d [label = "%s" id="N%d_%d" fontsize=8 shape=box3d tooltip="%s"]`+"\n", nodeID, i, t.Name, nodeID, i, weight) + nodelets += fmt.Sprintf(`N%d -> N%d_%d [label=" %s" weight=100 tooltip="%s" labeltooltip="%s"]`+"\n", nodeID, nodeID, i, weight, weight, weight) + if nts := lnts[t.Name]; nts != nil { + nodelets += b.numericNodelets(nts, maxNodelets, flatTags, fmt.Sprintf(`N%d_%d`, nodeID, i)) + } + } + + if nts := lnts[""]; nts != nil { + nodelets += b.numericNodelets(nts, maxNodelets, flatTags, fmt.Sprintf(`N%d`, nodeID)) + } + + fmt.Fprint(b, nodelets) + return nodelets != "" +} + +func (b *builder) numericNodelets(nts []*Tag, maxNumNodelets int, flatTags bool, source string) string { + nodelets := "" + + // Collapse numeric labels into maxNumNodelets buckets, of the form: + // 1MB..2MB, 3MB..5MB, ... + for j, t := range b.collapsedTags(nts, maxNumNodelets, flatTags) { + w, attr := t.CumValue(), ` style="dotted"` + if flatTags || t.FlatValue() == t.CumValue() { + w, attr = t.FlatValue(), "" + } + if w != 0 { + weight := b.config.FormatValue(w) + nodelets += fmt.Sprintf(`N%s_%d [label = "%s" id="N%s_%d" fontsize=8 shape=box3d tooltip="%s"]`+"\n", source, j, t.Name, source, j, weight) + nodelets += fmt.Sprintf(`%s -> N%s_%d [label=" %s" weight=100 tooltip="%s" labeltooltip="%s"%s]`+"\n", source, source, j, weight, weight, weight, attr) + } + } + return nodelets +} + +// addEdge generates a graph edge in DOT format. +func (b *builder) addEdge(edge *Edge, from, to int, hasNodelets bool) { + var inline string + if edge.Inline { + inline = `\n (inline)` + } + w := b.config.FormatValue(edge.WeightValue()) + attr := fmt.Sprintf(`label=" %s%s"`, w, inline) + if b.config.Total != 0 { + // Note: edge.weight > b.config.Total is possible for profile diffs. + if weight := 1 + int(min64(abs64(edge.WeightValue()*100/b.config.Total), 100)); weight > 1 { + attr = fmt.Sprintf(`%s weight=%d`, attr, weight) + } + if width := 1 + int(min64(abs64(edge.WeightValue()*5/b.config.Total), 5)); width > 1 { + attr = fmt.Sprintf(`%s penwidth=%d`, attr, width) + } + attr = fmt.Sprintf(`%s color="%s"`, attr, + dotColor(float64(edge.WeightValue())/float64(abs64(b.config.Total)), false)) + } + arrow := "->" + if edge.Residual { + arrow = "..." + } + tooltip := fmt.Sprintf(`"%s %s %s (%s)"`, + escapeForDot(edge.Src.Info.PrintableName()), arrow, + escapeForDot(edge.Dest.Info.PrintableName()), w) + attr = fmt.Sprintf(`%s tooltip=%s labeltooltip=%s`, attr, tooltip, tooltip) + + if edge.Residual { + attr = attr + ` style="dotted"` + } + + if hasNodelets { + // Separate children further if source has tags. + attr = attr + " minlen=2" + } + + fmt.Fprintf(b, "N%d -> N%d [%s]\n", from, to, attr) +} + +// dotColor returns a color for the given score (between -1.0 and +// 1.0), with -1.0 colored green, 0.0 colored grey, and 1.0 colored +// red. If isBackground is true, then a light (low-saturation) +// color is returned (suitable for use as a background color); +// otherwise, a darker color is returned (suitable for use as a +// foreground color). +func dotColor(score float64, isBackground bool) string { + // A float between 0.0 and 1.0, indicating the extent to which + // colors should be shifted away from grey (to make positive and + // negative values easier to distinguish, and to make more use of + // the color range.) + const shift = 0.7 + + // Saturation and value (in hsv colorspace) for background colors. + const bgSaturation = 0.1 + const bgValue = 0.93 + + // Saturation and value (in hsv colorspace) for foreground colors. + const fgSaturation = 1.0 + const fgValue = 0.7 + + // Choose saturation and value based on isBackground. + var saturation float64 + var value float64 + if isBackground { + saturation = bgSaturation + value = bgValue + } else { + saturation = fgSaturation + value = fgValue + } + + // Limit the score values to the range [-1.0, 1.0]. + score = math.Max(-1.0, math.Min(1.0, score)) + + // Reduce saturation near score=0 (so it is colored grey, rather than yellow). + if math.Abs(score) < 0.2 { + saturation *= math.Abs(score) / 0.2 + } + + // Apply 'shift' to move scores away from 0.0 (grey). + if score > 0.0 { + score = math.Pow(score, (1.0 - shift)) + } + if score < 0.0 { + score = -math.Pow(-score, (1.0 - shift)) + } + + var r, g, b float64 // red, green, blue + if score < 0.0 { + g = value + r = value * (1 + saturation*score) + } else { + r = value + g = value * (1 - saturation*score) + } + b = value * (1 - saturation) + return fmt.Sprintf("#%02x%02x%02x", uint8(r*255.0), uint8(g*255.0), uint8(b*255.0)) +} + +func multilinePrintableName(info *NodeInfo) string { + infoCopy := *info + infoCopy.Name = escapeForDot(ShortenFunctionName(infoCopy.Name)) + infoCopy.Name = strings.Replace(infoCopy.Name, "::", `\n`, -1) + // Go type parameters are reported as "[...]" by Go pprof profiles. + // Keep this ellipsis rather than replacing with newlines below. + infoCopy.Name = strings.Replace(infoCopy.Name, "[...]", "[…]", -1) + infoCopy.Name = strings.Replace(infoCopy.Name, ".", `\n`, -1) + if infoCopy.File != "" { + infoCopy.File = filepath.Base(infoCopy.File) + } + return strings.Join(infoCopy.NameComponents(), `\n`) + `\n` +} + +// collapsedTags trims and sorts a slice of tags. +func (b *builder) collapsedTags(ts []*Tag, count int, flatTags bool) []*Tag { + ts = SortTags(ts, flatTags) + if len(ts) <= count { + return ts + } + + tagGroups := make([][]*Tag, count) + for i, t := range (ts)[:count] { + tagGroups[i] = []*Tag{t} + } + for _, t := range (ts)[count:] { + g, d := 0, tagDistance(t, tagGroups[0][0]) + for i := 1; i < count; i++ { + if nd := tagDistance(t, tagGroups[i][0]); nd < d { + g, d = i, nd + } + } + tagGroups[g] = append(tagGroups[g], t) + } + + var nts []*Tag + for _, g := range tagGroups { + l, w, c := b.tagGroupLabel(g) + nts = append(nts, &Tag{ + Name: l, + Flat: w, + Cum: c, + }) + } + return SortTags(nts, flatTags) +} + +func tagDistance(t, u *Tag) float64 { + v, _ := measurement.Scale(u.Value, u.Unit, t.Unit) + if v < float64(t.Value) { + return float64(t.Value) - v + } + return v - float64(t.Value) +} + +func (b *builder) tagGroupLabel(g []*Tag) (label string, flat, cum int64) { + if len(g) == 1 { + t := g[0] + return measurement.Label(t.Value, t.Unit), t.FlatValue(), t.CumValue() + } + min := g[0] + max := g[0] + df, f := min.FlatDiv, min.Flat + dc, c := min.CumDiv, min.Cum + for _, t := range g[1:] { + if v, _ := measurement.Scale(t.Value, t.Unit, min.Unit); int64(v) < min.Value { + min = t + } + if v, _ := measurement.Scale(t.Value, t.Unit, max.Unit); int64(v) > max.Value { + max = t + } + f += t.Flat + df += t.FlatDiv + c += t.Cum + dc += t.CumDiv + } + if df != 0 { + f = f / df + } + if dc != 0 { + c = c / dc + } + + // Tags are not scaled with the selected output unit because tags are often + // much smaller than other values which appear, so the range of tag sizes + // sometimes would appear to be "0..0" when scaled to the selected output unit. + return measurement.Label(min.Value, min.Unit) + ".." + measurement.Label(max.Value, max.Unit), f, c +} + +func min64(a, b int64) int64 { + if a < b { + return a + } + return b +} + +// escapeAllForDot applies escapeForDot to all strings in the given slice. +func escapeAllForDot(in []string) []string { + var out = make([]string, len(in)) + for i := range in { + out[i] = escapeForDot(in[i]) + } + return out +} + +// escapeForDot escapes double quotes and backslashes, and replaces Graphviz's +// "center" character (\n) with a left-justified character. +// See https://graphviz.org/docs/attr-types/escString/ for more info. +func escapeForDot(str string) string { + return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(str, `\`, `\\`), `"`, `\"`), "\n", `\l`) +} diff --git a/profile/internal/graph/graph.go b/profile/internal/graph/graph.go new file mode 100644 index 0000000000..74b904c402 --- /dev/null +++ b/profile/internal/graph/graph.go @@ -0,0 +1,1170 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package graph collects a set of samples into a directed graph. +package graph + +import ( + "fmt" + "math" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/google/pprof/profile" +) + +var ( + // Removes package name and method arguments for Java method names. + // See tests for examples. + javaRegExp = regexp.MustCompile(`^(?:[a-z]\w*\.)*([A-Z][\w\$]*\.(?:|[a-z][\w\$]*(?:\$\d+)?))(?:(?:\()|$)`) + // Removes package name and method arguments for Go function names. + // See tests for examples. + goRegExp = regexp.MustCompile(`^(?:[\w\-\.]+\/)+(.+)`) + // Removes potential module versions in a package path. + goVerRegExp = regexp.MustCompile(`^(.*?)/v(?:[2-9]|[1-9][0-9]+)([./].*)$`) + // Strips C++ namespace prefix from a C++ function / method name. + // NOTE: Make sure to keep the template parameters in the name. Normally, + // template parameters are stripped from the C++ names but when + // -symbolize=demangle=templates flag is used, they will not be. + // See tests for examples. + cppRegExp = regexp.MustCompile(`^(?:[_a-zA-Z]\w*::)+(_*[A-Z]\w*::~?[_a-zA-Z]\w*(?:<.*>)?)`) + cppAnonymousPrefixRegExp = regexp.MustCompile(`^\(anonymous namespace\)::`) +) + +// Graph summarizes a performance profile into a format that is +// suitable for visualization. +type Graph struct { + Nodes Nodes +} + +// Options encodes the options for constructing a graph +type Options struct { + SampleValue func(s []int64) int64 // Function to compute the value of a sample + SampleMeanDivisor func(s []int64) int64 // Function to compute the divisor for mean graphs, or nil + FormatTag func(int64, string) string // Function to format a sample tag value into a string + ObjNames bool // Always preserve obj filename + OrigFnNames bool // Preserve original (eg mangled) function names + + CallTree bool // Build a tree instead of a graph + DropNegative bool // Drop nodes with overall negative values + + KeptNodes NodeSet // If non-nil, only use nodes in this set +} + +// Nodes is an ordered collection of graph nodes. +type Nodes []*Node + +// Node is an entry on a profiling report. It represents a unique +// program location. +type Node struct { + // Info describes the source location associated to this node. + Info NodeInfo + + // Function represents the function that this node belongs to. On + // graphs with sub-function resolution (eg line number or + // addresses), two nodes in a NodeMap that are part of the same + // function have the same value of Node.Function. If the Node + // represents the whole function, it points back to itself. + Function *Node + + // Values associated to this node. Flat is exclusive to this node, + // Cum includes all descendents. + Flat, FlatDiv, Cum, CumDiv int64 + + // In and out Contains the nodes immediately reaching or reached by + // this node. + In, Out EdgeMap + + // LabelTags provide additional information about subsets of a sample. + LabelTags TagMap + + // NumericTags provide additional values for subsets of a sample. + // Numeric tags are optionally associated to a label tag. The key + // for NumericTags is the name of the LabelTag they are associated + // to, or "" for numeric tags not associated to a label tag. + NumericTags map[string]TagMap +} + +// FlatValue returns the exclusive value for this node, computing the +// mean if a divisor is available. +func (n *Node) FlatValue() int64 { + if n.FlatDiv == 0 { + return n.Flat + } + return n.Flat / n.FlatDiv +} + +// CumValue returns the inclusive value for this node, computing the +// mean if a divisor is available. +func (n *Node) CumValue() int64 { + if n.CumDiv == 0 { + return n.Cum + } + return n.Cum / n.CumDiv +} + +// AddToEdge increases the weight of an edge between two nodes. If +// there isn't such an edge one is created. +func (n *Node) AddToEdge(to *Node, v int64, residual, inline bool) { + n.AddToEdgeDiv(to, 0, v, residual, inline) +} + +// AddToEdgeDiv increases the weight of an edge between two nodes. If +// there isn't such an edge one is created. +func (n *Node) AddToEdgeDiv(to *Node, dv, v int64, residual, inline bool) { + if n.Out[to] != to.In[n] { + panic(fmt.Errorf("asymmetric edges %v %v", *n, *to)) + } + + if e := n.Out[to]; e != nil { + e.WeightDiv += dv + e.Weight += v + if residual { + e.Residual = true + } + if !inline { + e.Inline = false + } + return + } + + info := &Edge{Src: n, Dest: to, WeightDiv: dv, Weight: v, Residual: residual, Inline: inline} + n.Out[to] = info + to.In[n] = info +} + +// NodeInfo contains the attributes for a node. +type NodeInfo struct { + Name string + OrigName string + Address uint64 + File string + StartLine, Lineno int + Objfile string +} + +// PrintableName calls the Node's Formatter function with a single space separator. +func (i *NodeInfo) PrintableName() string { + return strings.Join(i.NameComponents(), " ") +} + +// NameComponents returns the components of the printable name to be used for a node. +func (i *NodeInfo) NameComponents() []string { + var name []string + if i.Address != 0 { + name = append(name, fmt.Sprintf("%016x", i.Address)) + } + if fun := i.Name; fun != "" { + name = append(name, fun) + } + + switch { + case i.Lineno != 0: + // User requested line numbers, provide what we have. + name = append(name, fmt.Sprintf("%s:%d", i.File, i.Lineno)) + case i.File != "": + // User requested file name, provide it. + name = append(name, i.File) + case i.Name != "": + // User requested function name. It was already included. + case i.Objfile != "": + // Only binary name is available + name = append(name, "["+filepath.Base(i.Objfile)+"]") + default: + // Do not leave it empty if there is no information at all. + name = append(name, "") + } + return name +} + +// NodeMap maps from a node info struct to a node. It is used to merge +// report entries with the same info. +type NodeMap map[NodeInfo]*Node + +// NodeSet is a collection of node info structs. +type NodeSet map[NodeInfo]bool + +// NodePtrSet is a collection of nodes. Trimming a graph or tree requires a set +// of objects which uniquely identify the nodes to keep. In a graph, NodeInfo +// works as a unique identifier; however, in a tree multiple nodes may share +// identical NodeInfos. A *Node does uniquely identify a node so we can use that +// instead. Though a *Node also uniquely identifies a node in a graph, +// currently, during trimming, graphs are rebuilt from scratch using only the +// NodeSet, so there would not be the required context of the initial graph to +// allow for the use of *Node. +type NodePtrSet map[*Node]bool + +// FindOrInsertNode takes the info for a node and either returns a matching node +// from the node map if one exists, or adds one to the map if one does not. +// If kept is non-nil, nodes are only added if they can be located on it. +func (nm NodeMap) FindOrInsertNode(info NodeInfo, kept NodeSet) *Node { + if kept != nil { + if _, ok := kept[info]; !ok { + return nil + } + } + + if n, ok := nm[info]; ok { + return n + } + + n := &Node{ + Info: info, + In: make(EdgeMap), + Out: make(EdgeMap), + LabelTags: make(TagMap), + NumericTags: make(map[string]TagMap), + } + nm[info] = n + if info.Address == 0 && info.Lineno == 0 { + // This node represents the whole function, so point Function + // back to itself. + n.Function = n + return n + } + // Find a node that represents the whole function. + info.Address = 0 + info.Lineno = 0 + n.Function = nm.FindOrInsertNode(info, nil) + return n +} + +// EdgeMap is used to represent the incoming/outgoing edges from a node. +type EdgeMap map[*Node]*Edge + +// Edge contains any attributes to be represented about edges in a graph. +type Edge struct { + Src, Dest *Node + // The summary weight of the edge + Weight, WeightDiv int64 + + // residual edges connect nodes that were connected through a + // separate node, which has been removed from the report. + Residual bool + // An inline edge represents a call that was inlined into the caller. + Inline bool +} + +// WeightValue returns the weight value for this edge, normalizing if a +// divisor is available. +func (e *Edge) WeightValue() int64 { + if e.WeightDiv == 0 { + return e.Weight + } + return e.Weight / e.WeightDiv +} + +// Tag represent sample annotations +type Tag struct { + Name string + Unit string // Describe the value, "" for non-numeric tags + Value int64 + Flat, FlatDiv int64 + Cum, CumDiv int64 +} + +// FlatValue returns the exclusive value for this tag, computing the +// mean if a divisor is available. +func (t *Tag) FlatValue() int64 { + if t.FlatDiv == 0 { + return t.Flat + } + return t.Flat / t.FlatDiv +} + +// CumValue returns the inclusive value for this tag, computing the +// mean if a divisor is available. +func (t *Tag) CumValue() int64 { + if t.CumDiv == 0 { + return t.Cum + } + return t.Cum / t.CumDiv +} + +// TagMap is a collection of tags, classified by their name. +type TagMap map[string]*Tag + +// SortTags sorts a slice of tags based on their weight. +func SortTags(t []*Tag, flat bool) []*Tag { + ts := tags{t, flat} + sort.Sort(ts) + return ts.t +} + +// New summarizes performance data from a profile into a graph. +func New(prof *profile.Profile, o *Options) *Graph { + if o.CallTree { + return newTree(prof, o) + } + g, _ := newGraph(prof, o) + return g +} + +// newGraph computes a graph from a profile. It returns the graph, and +// a map from the profile location indices to the corresponding graph +// nodes. +func newGraph(prof *profile.Profile, o *Options) (*Graph, map[uint64]Nodes) { + nodes, locationMap := CreateNodes(prof, o) + seenNode := make(map[*Node]bool) + seenEdge := make(map[nodePair]bool) + for _, sample := range prof.Sample { + var w, dw int64 + w = o.SampleValue(sample.Value) + if o.SampleMeanDivisor != nil { + dw = o.SampleMeanDivisor(sample.Value) + } + if dw == 0 && w == 0 { + continue + } + for k := range seenNode { + delete(seenNode, k) + } + for k := range seenEdge { + delete(seenEdge, k) + } + var parent *Node + // A residual edge goes over one or more nodes that were not kept. + residual := false + + labels := joinLabels(sample) + // Group the sample frames, based on a global map. + for i := len(sample.Location) - 1; i >= 0; i-- { + l := sample.Location[i] + locNodes := locationMap[l.ID] + for ni := len(locNodes) - 1; ni >= 0; ni-- { + n := locNodes[ni] + if n == nil { + residual = true + continue + } + // Add cum weight to all nodes in stack, avoiding double counting. + if _, ok := seenNode[n]; !ok { + seenNode[n] = true + n.addSample(dw, w, labels, sample.NumLabel, sample.NumUnit, o.FormatTag, false) + } + // Update edge weights for all edges in stack, avoiding double counting. + if _, ok := seenEdge[nodePair{n, parent}]; !ok && parent != nil && n != parent { + seenEdge[nodePair{n, parent}] = true + parent.AddToEdgeDiv(n, dw, w, residual, ni != len(locNodes)-1) + } + parent = n + residual = false + } + } + if parent != nil && !residual { + // Add flat weight to leaf node. + parent.addSample(dw, w, labels, sample.NumLabel, sample.NumUnit, o.FormatTag, true) + } + } + + return selectNodesForGraph(nodes, o.DropNegative), locationMap +} + +func selectNodesForGraph(nodes Nodes, dropNegative bool) *Graph { + // Collect nodes into a graph. + gNodes := make(Nodes, 0, len(nodes)) + for _, n := range nodes { + if n == nil { + continue + } + if n.Cum == 0 && n.Flat == 0 { + continue + } + if dropNegative && isNegative(n) { + continue + } + gNodes = append(gNodes, n) + } + return &Graph{gNodes} +} + +type nodePair struct { + src, dest *Node +} + +func newTree(prof *profile.Profile, o *Options) (g *Graph) { + parentNodeMap := make(map[*Node]NodeMap, len(prof.Sample)) + for _, sample := range prof.Sample { + var w, dw int64 + w = o.SampleValue(sample.Value) + if o.SampleMeanDivisor != nil { + dw = o.SampleMeanDivisor(sample.Value) + } + if dw == 0 && w == 0 { + continue + } + var parent *Node + labels := joinLabels(sample) + // Group the sample frames, based on a per-node map. + for i := len(sample.Location) - 1; i >= 0; i-- { + l := sample.Location[i] + lines := l.Line + if len(lines) == 0 { + lines = []profile.Line{{}} // Create empty line to include location info. + } + for lidx := len(lines) - 1; lidx >= 0; lidx-- { + nodeMap := parentNodeMap[parent] + if nodeMap == nil { + nodeMap = make(NodeMap) + parentNodeMap[parent] = nodeMap + } + n := nodeMap.findOrInsertLine(l, lines[lidx], o) + if n == nil { + continue + } + n.addSample(dw, w, labels, sample.NumLabel, sample.NumUnit, o.FormatTag, false) + if parent != nil { + parent.AddToEdgeDiv(n, dw, w, false, lidx != len(lines)-1) + } + parent = n + } + } + if parent != nil { + parent.addSample(dw, w, labels, sample.NumLabel, sample.NumUnit, o.FormatTag, true) + } + } + + nodes := make(Nodes, len(prof.Location)) + for _, nm := range parentNodeMap { + nodes = append(nodes, nm.nodes()...) + } + return selectNodesForGraph(nodes, o.DropNegative) +} + +// ShortenFunctionName returns a shortened version of a function's name. +func ShortenFunctionName(f string) string { + f = cppAnonymousPrefixRegExp.ReplaceAllString(f, "") + f = goVerRegExp.ReplaceAllString(f, `${1}${2}`) + for _, re := range []*regexp.Regexp{goRegExp, javaRegExp, cppRegExp} { + if matches := re.FindStringSubmatch(f); len(matches) >= 2 { + return strings.Join(matches[1:], "") + } + } + return f +} + +// TrimTree trims a Graph in forest form, keeping only the nodes in kept. This +// will not work correctly if even a single node has multiple parents. +func (g *Graph) TrimTree(kept NodePtrSet) { + // Creates a new list of nodes + oldNodes := g.Nodes + g.Nodes = make(Nodes, 0, len(kept)) + + for _, cur := range oldNodes { + // A node may not have multiple parents + if len(cur.In) > 1 { + panic("TrimTree only works on trees") + } + + // If a node should be kept, add it to the new list of nodes + if _, ok := kept[cur]; ok { + g.Nodes = append(g.Nodes, cur) + continue + } + + // If a node has no parents, then delete all of the in edges of its + // children to make them each roots of their own trees. + if len(cur.In) == 0 { + for _, outEdge := range cur.Out { + delete(outEdge.Dest.In, cur) + } + continue + } + + // Get the parent. This works since at this point cur.In must contain only + // one element. + if len(cur.In) != 1 { + panic("Get parent assertion failed. cur.In expected to be of length 1.") + } + var parent *Node + for _, edge := range cur.In { + parent = edge.Src + } + + parentEdgeInline := parent.Out[cur].Inline + + // Remove the edge from the parent to this node + delete(parent.Out, cur) + + // Reconfigure every edge from the current node to now begin at the parent. + for _, outEdge := range cur.Out { + child := outEdge.Dest + + delete(child.In, cur) + child.In[parent] = outEdge + parent.Out[child] = outEdge + + outEdge.Src = parent + outEdge.Residual = true + // If the edge from the parent to the current node and the edge from the + // current node to the child are both inline, then this resulting residual + // edge should also be inline + outEdge.Inline = parentEdgeInline && outEdge.Inline + } + } + g.RemoveRedundantEdges() +} + +func joinLabels(s *profile.Sample) string { + if len(s.Label) == 0 { + return "" + } + + var labels []string + for key, vals := range s.Label { + for _, v := range vals { + labels = append(labels, key+":"+v) + } + } + sort.Strings(labels) + return strings.Join(labels, `\n`) +} + +// isNegative returns true if the node is considered as "negative" for the +// purposes of drop_negative. +func isNegative(n *Node) bool { + switch { + case n.Flat < 0: + return true + case n.Flat == 0 && n.Cum < 0: + return true + default: + return false + } +} + +// CreateNodes creates graph nodes for all locations in a profile. It +// returns set of all nodes, plus a mapping of each location to the +// set of corresponding nodes (one per location.Line). +func CreateNodes(prof *profile.Profile, o *Options) (Nodes, map[uint64]Nodes) { + locations := make(map[uint64]Nodes, len(prof.Location)) + nm := make(NodeMap, len(prof.Location)) + for _, l := range prof.Location { + lines := l.Line + if len(lines) == 0 { + lines = []profile.Line{{}} // Create empty line to include location info. + } + nodes := make(Nodes, len(lines)) + for ln := range lines { + nodes[ln] = nm.findOrInsertLine(l, lines[ln], o) + } + locations[l.ID] = nodes + } + return nm.nodes(), locations +} + +func (nm NodeMap) nodes() Nodes { + nodes := make(Nodes, 0, len(nm)) + for _, n := range nm { + nodes = append(nodes, n) + } + return nodes +} + +func (nm NodeMap) findOrInsertLine(l *profile.Location, li profile.Line, o *Options) *Node { + var objfile string + if m := l.Mapping; m != nil && m.File != "" { + objfile = m.File + } + + if ni := nodeInfo(l, li, objfile, o); ni != nil { + return nm.FindOrInsertNode(*ni, o.KeptNodes) + } + return nil +} + +func nodeInfo(l *profile.Location, line profile.Line, objfile string, o *Options) *NodeInfo { + if line.Function == nil { + return &NodeInfo{Address: l.Address, Objfile: objfile} + } + ni := &NodeInfo{ + Address: l.Address, + Lineno: int(line.Line), + Name: line.Function.Name, + } + if fname := line.Function.Filename; fname != "" { + ni.File = filepath.Clean(fname) + } + if o.OrigFnNames { + ni.OrigName = line.Function.SystemName + } + if o.ObjNames || (ni.Name == "" && ni.OrigName == "") { + ni.Objfile = objfile + ni.StartLine = int(line.Function.StartLine) + } + return ni +} + +type tags struct { + t []*Tag + flat bool +} + +func (t tags) Len() int { return len(t.t) } +func (t tags) Swap(i, j int) { t.t[i], t.t[j] = t.t[j], t.t[i] } +func (t tags) Less(i, j int) bool { + if !t.flat { + if t.t[i].Cum != t.t[j].Cum { + return abs64(t.t[i].Cum) > abs64(t.t[j].Cum) + } + } + if t.t[i].Flat != t.t[j].Flat { + return abs64(t.t[i].Flat) > abs64(t.t[j].Flat) + } + return t.t[i].Name < t.t[j].Name +} + +// Sum adds the flat and cum values of a set of nodes. +func (ns Nodes) Sum() (flat int64, cum int64) { + for _, n := range ns { + flat += n.Flat + cum += n.Cum + } + return +} + +func (n *Node) addSample(dw, w int64, labels string, numLabel map[string][]int64, numUnit map[string][]string, format func(int64, string) string, flat bool) { + // Update sample value + if flat { + n.FlatDiv += dw + n.Flat += w + } else { + n.CumDiv += dw + n.Cum += w + } + + // Add string tags + if labels != "" { + t := n.LabelTags.findOrAddTag(labels, "", 0) + if flat { + t.FlatDiv += dw + t.Flat += w + } else { + t.CumDiv += dw + t.Cum += w + } + } + + numericTags := n.NumericTags[labels] + if numericTags == nil { + numericTags = TagMap{} + n.NumericTags[labels] = numericTags + } + // Add numeric tags + if format == nil { + format = defaultLabelFormat + } + for k, nvals := range numLabel { + units := numUnit[k] + for i, v := range nvals { + var t *Tag + if len(units) > 0 { + t = numericTags.findOrAddTag(format(v, units[i]), units[i], v) + } else { + t = numericTags.findOrAddTag(format(v, k), k, v) + } + if flat { + t.FlatDiv += dw + t.Flat += w + } else { + t.CumDiv += dw + t.Cum += w + } + } + } +} + +func defaultLabelFormat(v int64, key string) string { + return strconv.FormatInt(v, 10) +} + +func (m TagMap) findOrAddTag(label, unit string, value int64) *Tag { + l := m[label] + if l == nil { + l = &Tag{ + Name: label, + Unit: unit, + Value: value, + } + m[label] = l + } + return l +} + +// String returns a text representation of a graph, for debugging purposes. +func (g *Graph) String() string { + var s []string + + nodeIndex := make(map[*Node]int, len(g.Nodes)) + + for i, n := range g.Nodes { + nodeIndex[n] = i + 1 + } + + for i, n := range g.Nodes { + name := n.Info.PrintableName() + var in, out []int + + for _, from := range n.In { + in = append(in, nodeIndex[from.Src]) + } + for _, to := range n.Out { + out = append(out, nodeIndex[to.Dest]) + } + s = append(s, fmt.Sprintf("%d: %s[flat=%d cum=%d] %x -> %v ", i+1, name, n.Flat, n.Cum, in, out)) + } + return strings.Join(s, "\n") +} + +// DiscardLowFrequencyNodes returns a set of the nodes at or over a +// specific cum value cutoff. +func (g *Graph) DiscardLowFrequencyNodes(nodeCutoff int64) NodeSet { + return makeNodeSet(g.Nodes, nodeCutoff) +} + +// DiscardLowFrequencyNodePtrs returns a NodePtrSet of nodes at or over a +// specific cum value cutoff. +func (g *Graph) DiscardLowFrequencyNodePtrs(nodeCutoff int64) NodePtrSet { + cutNodes := getNodesAboveCumCutoff(g.Nodes, nodeCutoff) + kept := make(NodePtrSet, len(cutNodes)) + for _, n := range cutNodes { + kept[n] = true + } + return kept +} + +func makeNodeSet(nodes Nodes, nodeCutoff int64) NodeSet { + cutNodes := getNodesAboveCumCutoff(nodes, nodeCutoff) + kept := make(NodeSet, len(cutNodes)) + for _, n := range cutNodes { + kept[n.Info] = true + } + return kept +} + +// getNodesAboveCumCutoff returns all the nodes which have a Cum value greater +// than or equal to cutoff. +func getNodesAboveCumCutoff(nodes Nodes, nodeCutoff int64) Nodes { + cutoffNodes := make(Nodes, 0, len(nodes)) + for _, n := range nodes { + if abs64(n.Cum) < nodeCutoff { + continue + } + cutoffNodes = append(cutoffNodes, n) + } + return cutoffNodes +} + +// TrimLowFrequencyTags removes tags that have less than +// the specified weight. +func (g *Graph) TrimLowFrequencyTags(tagCutoff int64) { + // Remove nodes with value <= total*nodeFraction + for _, n := range g.Nodes { + n.LabelTags = trimLowFreqTags(n.LabelTags, tagCutoff) + for s, nt := range n.NumericTags { + n.NumericTags[s] = trimLowFreqTags(nt, tagCutoff) + } + } +} + +func trimLowFreqTags(tags TagMap, minValue int64) TagMap { + kept := TagMap{} + for s, t := range tags { + if abs64(t.Flat) >= minValue || abs64(t.Cum) >= minValue { + kept[s] = t + } + } + return kept +} + +// TrimLowFrequencyEdges removes edges that have less than +// the specified weight. Returns the number of edges removed +func (g *Graph) TrimLowFrequencyEdges(edgeCutoff int64) int { + var droppedEdges int + for _, n := range g.Nodes { + for src, e := range n.In { + if abs64(e.Weight) < edgeCutoff { + delete(n.In, src) + delete(src.Out, n) + droppedEdges++ + } + } + } + return droppedEdges +} + +// SortNodes sorts the nodes in a graph based on a specific heuristic. +func (g *Graph) SortNodes(cum bool, visualMode bool) { + // Sort nodes based on requested mode + switch { + case visualMode: + // Specialized sort to produce a more visually-interesting graph + g.Nodes.Sort(EntropyOrder) + case cum: + g.Nodes.Sort(CumNameOrder) + default: + g.Nodes.Sort(FlatNameOrder) + } +} + +// SelectTopNodePtrs returns a set of the top maxNodes *Node in a graph. +func (g *Graph) SelectTopNodePtrs(maxNodes int, visualMode bool) NodePtrSet { + set := make(NodePtrSet) + for _, node := range g.selectTopNodes(maxNodes, visualMode) { + set[node] = true + } + return set +} + +// SelectTopNodes returns a set of the top maxNodes nodes in a graph. +func (g *Graph) SelectTopNodes(maxNodes int, visualMode bool) NodeSet { + return makeNodeSet(g.selectTopNodes(maxNodes, visualMode), 0) +} + +// selectTopNodes returns a slice of the top maxNodes nodes in a graph. +func (g *Graph) selectTopNodes(maxNodes int, visualMode bool) Nodes { + if maxNodes > 0 { + if visualMode { + var count int + // If generating a visual graph, count tags as nodes. Update + // maxNodes to account for them. + for i, n := range g.Nodes { + tags := countTags(n) + if tags > maxNodelets { + tags = maxNodelets + } + if count += tags + 1; count >= maxNodes { + maxNodes = i + 1 + break + } + } + } + } + if maxNodes > len(g.Nodes) { + maxNodes = len(g.Nodes) + } + return g.Nodes[:maxNodes] +} + +// countTags counts the tags with flat count. This underestimates the +// number of tags being displayed, but in practice is close enough. +func countTags(n *Node) int { + count := 0 + for _, e := range n.LabelTags { + if e.Flat != 0 { + count++ + } + } + for _, t := range n.NumericTags { + for _, e := range t { + if e.Flat != 0 { + count++ + } + } + } + return count +} + +// RemoveRedundantEdges removes residual edges if the destination can +// be reached through another path. This is done to simplify the graph +// while preserving connectivity. +func (g *Graph) RemoveRedundantEdges() { + // Walk the nodes and outgoing edges in reverse order to prefer + // removing edges with the lowest weight. + for i := len(g.Nodes); i > 0; i-- { + n := g.Nodes[i-1] + in := n.In.Sort() + for j := len(in); j > 0; j-- { + e := in[j-1] + if !e.Residual { + // Do not remove edges heavier than a non-residual edge, to + // avoid potential confusion. + break + } + if isRedundantEdge(e) { + delete(e.Src.Out, e.Dest) + delete(e.Dest.In, e.Src) + } + } + } +} + +// isRedundantEdge determines if there is a path that allows e.Src +// to reach e.Dest after removing e. +func isRedundantEdge(e *Edge) bool { + src, n := e.Src, e.Dest + seen := map[*Node]bool{n: true} + queue := Nodes{n} + for len(queue) > 0 { + n := queue[0] + queue = queue[1:] + for _, ie := range n.In { + if e == ie || seen[ie.Src] { + continue + } + if ie.Src == src { + return true + } + seen[ie.Src] = true + queue = append(queue, ie.Src) + } + } + return false +} + +// nodeSorter is a mechanism used to allow a report to be sorted +// in different ways. +type nodeSorter struct { + rs Nodes + less func(l, r *Node) bool +} + +func (s nodeSorter) Len() int { return len(s.rs) } +func (s nodeSorter) Swap(i, j int) { s.rs[i], s.rs[j] = s.rs[j], s.rs[i] } +func (s nodeSorter) Less(i, j int) bool { return s.less(s.rs[i], s.rs[j]) } + +// Sort reorders a slice of nodes based on the specified ordering +// criteria. The result is sorted in decreasing order for (absolute) +// numeric quantities, alphabetically for text, and increasing for +// addresses. +func (ns Nodes) Sort(o NodeOrder) error { + var s nodeSorter + + switch o { + case FlatNameOrder: + s = nodeSorter{ns, + func(l, r *Node) bool { + if iv, jv := abs64(l.Flat), abs64(r.Flat); iv != jv { + return iv > jv + } + if iv, jv := l.Info.PrintableName(), r.Info.PrintableName(); iv != jv { + return iv < jv + } + if iv, jv := abs64(l.Cum), abs64(r.Cum); iv != jv { + return iv > jv + } + return compareNodes(l, r) + }, + } + case FlatCumNameOrder: + s = nodeSorter{ns, + func(l, r *Node) bool { + if iv, jv := abs64(l.Flat), abs64(r.Flat); iv != jv { + return iv > jv + } + if iv, jv := abs64(l.Cum), abs64(r.Cum); iv != jv { + return iv > jv + } + if iv, jv := l.Info.PrintableName(), r.Info.PrintableName(); iv != jv { + return iv < jv + } + return compareNodes(l, r) + }, + } + case NameOrder: + s = nodeSorter{ns, + func(l, r *Node) bool { + if iv, jv := l.Info.Name, r.Info.Name; iv != jv { + return iv < jv + } + return compareNodes(l, r) + }, + } + case FileOrder: + s = nodeSorter{ns, + func(l, r *Node) bool { + if iv, jv := l.Info.File, r.Info.File; iv != jv { + return iv < jv + } + if iv, jv := l.Info.StartLine, r.Info.StartLine; iv != jv { + return iv < jv + } + return compareNodes(l, r) + }, + } + case AddressOrder: + s = nodeSorter{ns, + func(l, r *Node) bool { + if iv, jv := l.Info.Address, r.Info.Address; iv != jv { + return iv < jv + } + return compareNodes(l, r) + }, + } + case CumNameOrder, EntropyOrder: + // Hold scoring for score-based ordering + var score map[*Node]int64 + scoreOrder := func(l, r *Node) bool { + if iv, jv := abs64(score[l]), abs64(score[r]); iv != jv { + return iv > jv + } + if iv, jv := l.Info.PrintableName(), r.Info.PrintableName(); iv != jv { + return iv < jv + } + if iv, jv := abs64(l.Flat), abs64(r.Flat); iv != jv { + return iv > jv + } + return compareNodes(l, r) + } + + switch o { + case CumNameOrder: + score = make(map[*Node]int64, len(ns)) + for _, n := range ns { + score[n] = n.Cum + } + s = nodeSorter{ns, scoreOrder} + case EntropyOrder: + score = make(map[*Node]int64, len(ns)) + for _, n := range ns { + score[n] = entropyScore(n) + } + s = nodeSorter{ns, scoreOrder} + } + default: + return fmt.Errorf("report: unrecognized sort ordering: %d", o) + } + sort.Sort(s) + return nil +} + +// compareNodes compares two nodes to provide a deterministic ordering +// between them. Two nodes cannot have the same Node.Info value. +func compareNodes(l, r *Node) bool { + return fmt.Sprint(l.Info) < fmt.Sprint(r.Info) +} + +// entropyScore computes a score for a node representing how important +// it is to include this node on a graph visualization. It is used to +// sort the nodes and select which ones to display if we have more +// nodes than desired in the graph. This number is computed by looking +// at the flat and cum weights of the node and the incoming/outgoing +// edges. The fundamental idea is to penalize nodes that have a simple +// fallthrough from their incoming to the outgoing edge. +func entropyScore(n *Node) int64 { + score := float64(0) + + if len(n.In) == 0 { + score++ // Favor entry nodes + } else { + score += edgeEntropyScore(n, n.In, 0) + } + + if len(n.Out) == 0 { + score++ // Favor leaf nodes + } else { + score += edgeEntropyScore(n, n.Out, n.Flat) + } + + return int64(score*float64(n.Cum)) + n.Flat +} + +// edgeEntropyScore computes the entropy value for a set of edges +// coming in or out of a node. Entropy (as defined in information +// theory) refers to the amount of information encoded by the set of +// edges. A set of edges that have a more interesting distribution of +// samples gets a higher score. +func edgeEntropyScore(n *Node, edges EdgeMap, self int64) float64 { + score := float64(0) + total := self + for _, e := range edges { + if e.Weight > 0 { + total += abs64(e.Weight) + } + } + if total != 0 { + for _, e := range edges { + frac := float64(abs64(e.Weight)) / float64(total) + score += -frac * math.Log2(frac) + } + if self > 0 { + frac := float64(abs64(self)) / float64(total) + score += -frac * math.Log2(frac) + } + } + return score +} + +// NodeOrder sets the ordering for a Sort operation +type NodeOrder int + +// Sorting options for node sort. +const ( + FlatNameOrder NodeOrder = iota + FlatCumNameOrder + CumNameOrder + NameOrder + FileOrder + AddressOrder + EntropyOrder +) + +// Sort returns a slice of the edges in the map, in a consistent +// order. The sort order is first based on the edge weight +// (higher-to-lower) and then by the node names to avoid flakiness. +func (e EdgeMap) Sort() []*Edge { + el := make(edgeList, 0, len(e)) + for _, w := range e { + el = append(el, w) + } + + sort.Sort(el) + return el +} + +// Sum returns the total weight for a set of nodes. +func (e EdgeMap) Sum() int64 { + var ret int64 + for _, edge := range e { + ret += edge.Weight + } + return ret +} + +type edgeList []*Edge + +func (el edgeList) Len() int { + return len(el) +} + +func (el edgeList) Less(i, j int) bool { + if el[i].Weight != el[j].Weight { + return abs64(el[i].Weight) > abs64(el[j].Weight) + } + + from1 := el[i].Src.Info.PrintableName() + from2 := el[j].Src.Info.PrintableName() + if from1 != from2 { + return from1 < from2 + } + + to1 := el[i].Dest.Info.PrintableName() + to2 := el[j].Dest.Info.PrintableName() + + return to1 < to2 +} + +func (el edgeList) Swap(i, j int) { + el[i], el[j] = el[j], el[i] +} + +func abs64(i int64) int64 { + if i < 0 { + return -i + } + return i +} diff --git a/profile/internal/measurement/measurement.go b/profile/internal/measurement/measurement.go new file mode 100644 index 0000000000..b5fcfbc3e4 --- /dev/null +++ b/profile/internal/measurement/measurement.go @@ -0,0 +1,293 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package measurement export utility functions to manipulate/format performance profile sample values. +package measurement + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/google/pprof/profile" +) + +// ScaleProfiles updates the units in a set of profiles to make them +// compatible. It scales the profiles to the smallest unit to preserve +// data. +func ScaleProfiles(profiles []*profile.Profile) error { + if len(profiles) == 0 { + return nil + } + periodTypes := make([]*profile.ValueType, 0, len(profiles)) + for _, p := range profiles { + if p.PeriodType != nil { + periodTypes = append(periodTypes, p.PeriodType) + } + } + periodType, err := CommonValueType(periodTypes) + if err != nil { + return fmt.Errorf("period type: %v", err) + } + + // Identify common sample types + numSampleTypes := len(profiles[0].SampleType) + for _, p := range profiles[1:] { + if numSampleTypes != len(p.SampleType) { + return fmt.Errorf("inconsistent samples type count: %d != %d", numSampleTypes, len(p.SampleType)) + } + } + sampleType := make([]*profile.ValueType, numSampleTypes) + for i := 0; i < numSampleTypes; i++ { + sampleTypes := make([]*profile.ValueType, len(profiles)) + for j, p := range profiles { + sampleTypes[j] = p.SampleType[i] + } + sampleType[i], err = CommonValueType(sampleTypes) + if err != nil { + return fmt.Errorf("sample types: %v", err) + } + } + + for _, p := range profiles { + if p.PeriodType != nil && periodType != nil { + period, _ := Scale(p.Period, p.PeriodType.Unit, periodType.Unit) + p.Period, p.PeriodType.Unit = int64(period), periodType.Unit + } + ratios := make([]float64, len(p.SampleType)) + for i, st := range p.SampleType { + if sampleType[i] == nil { + ratios[i] = 1 + continue + } + ratios[i], _ = Scale(1, st.Unit, sampleType[i].Unit) + p.SampleType[i].Unit = sampleType[i].Unit + } + if err := p.ScaleN(ratios); err != nil { + return fmt.Errorf("scale: %v", err) + } + } + return nil +} + +// CommonValueType returns the finest type from a set of compatible +// types. +func CommonValueType(ts []*profile.ValueType) (*profile.ValueType, error) { + if len(ts) <= 1 { + return nil, nil + } + minType := ts[0] + for _, t := range ts[1:] { + if !compatibleValueTypes(minType, t) { + return nil, fmt.Errorf("incompatible types: %v %v", *minType, *t) + } + if ratio, _ := Scale(1, t.Unit, minType.Unit); ratio < 1 { + minType = t + } + } + rcopy := *minType + return &rcopy, nil +} + +func compatibleValueTypes(v1, v2 *profile.ValueType) bool { + if v1 == nil || v2 == nil { + return true // No grounds to disqualify. + } + // Remove trailing 's' to permit minor mismatches. + if t1, t2 := strings.TrimSuffix(v1.Type, "s"), strings.TrimSuffix(v2.Type, "s"); t1 != t2 { + return false + } + + if v1.Unit == v2.Unit { + return true + } + for _, ut := range unitTypes { + if ut.sniffUnit(v1.Unit) != nil && ut.sniffUnit(v2.Unit) != nil { + return true + } + } + return false +} + +// Scale a measurement from an unit to a different unit and returns +// the scaled value and the target unit. The returned target unit +// will be empty if uninteresting (could be skipped). +func Scale(value int64, fromUnit, toUnit string) (float64, string) { + // Avoid infinite recursion on overflow. + if value < 0 && -value > 0 { + v, u := Scale(-value, fromUnit, toUnit) + return -v, u + } + for _, ut := range unitTypes { + if v, u, ok := ut.convertUnit(value, fromUnit, toUnit); ok { + return v, u + } + } + // Skip non-interesting units. + switch toUnit { + case "count", "sample", "unit", "minimum", "auto": + return float64(value), "" + default: + return float64(value), toUnit + } +} + +// Label returns the label used to describe a certain measurement. +func Label(value int64, unit string) string { + return ScaledLabel(value, unit, "auto") +} + +// ScaledLabel scales the passed-in measurement (if necessary) and +// returns the label used to describe a float measurement. +func ScaledLabel(value int64, fromUnit, toUnit string) string { + v, u := Scale(value, fromUnit, toUnit) + sv := strings.TrimSuffix(fmt.Sprintf("%.2f", v), ".00") + if sv == "0" || sv == "-0" { + return "0" + } + return sv + u +} + +// Percentage computes the percentage of total of a value, and encodes +// it as a string. At least two digits of precision are printed. +func Percentage(value, total int64) string { + var ratio float64 + if total != 0 { + ratio = math.Abs(float64(value)/float64(total)) * 100 + } + switch { + case math.Abs(ratio) >= 99.95 && math.Abs(ratio) <= 100.05: + return " 100%" + case math.Abs(ratio) >= 1.0: + return fmt.Sprintf("%5.2f%%", ratio) + default: + return fmt.Sprintf("%5.2g%%", ratio) + } +} + +// unit includes a list of aliases representing a specific unit and a factor +// which one can multiple a value in the specified unit by to get the value +// in terms of the base unit. +type unit struct { + canonicalName string + aliases []string + factor float64 +} + +// unitType includes a list of units that are within the same category (i.e. +// memory or time units) and a default unit to use for this type of unit. +type unitType struct { + defaultUnit unit + units []unit +} + +// findByAlias returns the unit associated with the specified alias. It returns +// nil if the unit with such alias is not found. +func (ut unitType) findByAlias(alias string) *unit { + for _, u := range ut.units { + for _, a := range u.aliases { + if alias == a { + return &u + } + } + } + return nil +} + +// sniffUnit simpifies the input alias and returns the unit associated with the +// specified alias. It returns nil if the unit with such alias is not found. +func (ut unitType) sniffUnit(unit string) *unit { + unit = strings.ToLower(unit) + if len(unit) > 2 { + unit = strings.TrimSuffix(unit, "s") + } + return ut.findByAlias(unit) +} + +// autoScale takes in the value with units of the base unit and returns +// that value scaled to a reasonable unit if a reasonable unit is +// found. +func (ut unitType) autoScale(value float64) (float64, string, bool) { + var f float64 + var unit string + for _, u := range ut.units { + if u.factor >= f && (value/u.factor) >= 1.0 { + f = u.factor + unit = u.canonicalName + } + } + if f == 0 { + return 0, "", false + } + return value / f, unit, true +} + +// convertUnit converts a value from the fromUnit to the toUnit, autoscaling +// the value if the toUnit is "minimum" or "auto". If the fromUnit is not +// included in the unitType, then a false boolean will be returned. If the +// toUnit is not in the unitType, the value will be returned in terms of the +// default unitType. +func (ut unitType) convertUnit(value int64, fromUnitStr, toUnitStr string) (float64, string, bool) { + fromUnit := ut.sniffUnit(fromUnitStr) + if fromUnit == nil { + return 0, "", false + } + v := float64(value) * fromUnit.factor + if toUnitStr == "minimum" || toUnitStr == "auto" { + if v, u, ok := ut.autoScale(v); ok { + return v, u, true + } + return v / ut.defaultUnit.factor, ut.defaultUnit.canonicalName, true + } + toUnit := ut.sniffUnit(toUnitStr) + if toUnit == nil { + return v / ut.defaultUnit.factor, ut.defaultUnit.canonicalName, true + } + return v / toUnit.factor, toUnit.canonicalName, true +} + +var unitTypes = []unitType{{ + units: []unit{ + {"B", []string{"b", "byte"}, 1}, + {"kB", []string{"kb", "kbyte", "kilobyte"}, float64(1 << 10)}, + {"MB", []string{"mb", "mbyte", "megabyte"}, float64(1 << 20)}, + {"GB", []string{"gb", "gbyte", "gigabyte"}, float64(1 << 30)}, + {"TB", []string{"tb", "tbyte", "terabyte"}, float64(1 << 40)}, + {"PB", []string{"pb", "pbyte", "petabyte"}, float64(1 << 50)}, + }, + defaultUnit: unit{"B", []string{"b", "byte"}, 1}, +}, { + units: []unit{ + {"ns", []string{"ns", "nanosecond"}, float64(time.Nanosecond)}, + {"us", []string{"μs", "us", "microsecond"}, float64(time.Microsecond)}, + {"ms", []string{"ms", "millisecond"}, float64(time.Millisecond)}, + {"s", []string{"s", "sec", "second"}, float64(time.Second)}, + {"hrs", []string{"hour", "hr"}, float64(time.Hour)}, + }, + defaultUnit: unit{"s", []string{}, float64(time.Second)}, +}, { + units: []unit{ + {"n*GCU", []string{"nanogcu"}, 1e-9}, + {"u*GCU", []string{"microgcu"}, 1e-6}, + {"m*GCU", []string{"milligcu"}, 1e-3}, + {"GCU", []string{"gcu"}, 1}, + {"k*GCU", []string{"kilogcu"}, 1e3}, + {"M*GCU", []string{"megagcu"}, 1e6}, + {"G*GCU", []string{"gigagcu"}, 1e9}, + {"T*GCU", []string{"teragcu"}, 1e12}, + {"P*GCU", []string{"petagcu"}, 1e15}, + }, + defaultUnit: unit{"GCU", []string{}, 1.0}, +}} diff --git a/profile/internal/report/report.go b/profile/internal/report/report.go new file mode 100644 index 0000000000..2207d5f78d --- /dev/null +++ b/profile/internal/report/report.go @@ -0,0 +1,1093 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package report summarizes a performance profile into a +// human-readable report. +package report + +import ( + "fmt" + "io" + "path/filepath" + "regexp" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/consensys/gnark/profile/internal/graph" + "github.com/consensys/gnark/profile/internal/measurement" + "github.com/google/pprof/profile" +) + +// Output formats. +const ( + Callgrind = iota + Comments + Dis + Dot + List + Proto + Raw + Tags + Text + TopProto + Traces + Tree + WebList +) + +// Options are the formatting and filtering options used to generate a +// profile. +type Options struct { + OutputFormat int + + CumSort bool + CallTree bool + DropNegative bool + CompactLabels bool + Ratio float64 + Title string + ProfileLabels []string + ActiveFilters []string + NumLabelUnits map[string]string + + NodeCount int + NodeFraction float64 + EdgeFraction float64 + + SampleValue func(s []int64) int64 + SampleMeanDivisor func(s []int64) int64 + SampleType string + SampleUnit string // Unit for the sample data from the profile. + + OutputUnit string // Units for data formatting in report. + + Symbol *regexp.Regexp // Symbols to include on disassembly report. + SourcePath string // Search path for source files. + TrimPath string // Paths to trim from source file paths. + + IntelSyntax bool // Whether or not to print assembly in Intel syntax. +} + +// Generate generates a report as directed by the Report. +func Generate(w io.Writer, rpt *Report) error { + o := rpt.options + + switch o.OutputFormat { + case Comments: + return printComments(w, rpt) + case Dot: + return printDOT(w, rpt) + case Tree: + return printTree(w, rpt) + case Text: + return printText(w, rpt) + case Traces: + return printTraces(w, rpt) + case Raw: + fmt.Fprint(w, rpt.prof.String()) + return nil + case Tags: + return printTags(w, rpt) + case Proto: + return printProto(w, rpt) + case TopProto: + return printTopProto(w, rpt) + case Callgrind: + return printCallgrind(w, rpt) + } + return fmt.Errorf("unexpected output format") +} + +// newTrimmedGraph creates a graph for this report, trimmed according +// to the report options. +func (rpt *Report) newTrimmedGraph() (g *graph.Graph, origCount, droppedNodes, droppedEdges int) { + o := rpt.options + + // Build a graph and refine it. On each refinement step we must rebuild the graph from the samples, + // as the graph itself doesn't contain enough information to preserve full precision. + visualMode := o.OutputFormat == Dot + cumSort := o.CumSort + + // The call_tree option is only honored when generating visual representations of the callgraph. + callTree := o.CallTree && (o.OutputFormat == Dot || o.OutputFormat == Callgrind) + + // First step: Build complete graph to identify low frequency nodes, based on their cum weight. + g = rpt.newGraph(nil) + totalValue, _ := g.Nodes.Sum() + nodeCutoff := abs64(int64(float64(totalValue) * o.NodeFraction)) + edgeCutoff := abs64(int64(float64(totalValue) * o.EdgeFraction)) + + // Filter out nodes with cum value below nodeCutoff. + if nodeCutoff > 0 { + if callTree { + if nodesKept := g.DiscardLowFrequencyNodePtrs(nodeCutoff); len(g.Nodes) != len(nodesKept) { + droppedNodes = len(g.Nodes) - len(nodesKept) + g.TrimTree(nodesKept) + } + } else { + if nodesKept := g.DiscardLowFrequencyNodes(nodeCutoff); len(g.Nodes) != len(nodesKept) { + droppedNodes = len(g.Nodes) - len(nodesKept) + g = rpt.newGraph(nodesKept) + } + } + } + origCount = len(g.Nodes) + + // Second step: Limit the total number of nodes. Apply specialized heuristics to improve + // visualization when generating dot output. + g.SortNodes(cumSort, visualMode) + if nodeCount := o.NodeCount; nodeCount > 0 { + // Remove low frequency tags and edges as they affect selection. + g.TrimLowFrequencyTags(nodeCutoff) + g.TrimLowFrequencyEdges(edgeCutoff) + if callTree { + if nodesKept := g.SelectTopNodePtrs(nodeCount, visualMode); len(g.Nodes) != len(nodesKept) { + g.TrimTree(nodesKept) + g.SortNodes(cumSort, visualMode) + } + } else { + if nodesKept := g.SelectTopNodes(nodeCount, visualMode); len(g.Nodes) != len(nodesKept) { + g = rpt.newGraph(nodesKept) + g.SortNodes(cumSort, visualMode) + } + } + } + + // Final step: Filter out low frequency tags and edges, and remove redundant edges that clutter + // the graph. + g.TrimLowFrequencyTags(nodeCutoff) + droppedEdges = g.TrimLowFrequencyEdges(edgeCutoff) + if visualMode { + g.RemoveRedundantEdges() + } + return +} + +func (rpt *Report) selectOutputUnit(g *graph.Graph) { + o := rpt.options + + // Select best unit for profile output. + // Find the appropriate units for the smallest non-zero sample + if o.OutputUnit != "minimum" || len(g.Nodes) == 0 { + return + } + var minValue int64 + + for _, n := range g.Nodes { + nodeMin := abs64(n.FlatValue()) + if nodeMin == 0 { + nodeMin = abs64(n.CumValue()) + } + if nodeMin > 0 && (minValue == 0 || nodeMin < minValue) { + minValue = nodeMin + } + } + maxValue := rpt.total + if minValue == 0 { + minValue = maxValue + } + + if r := o.Ratio; r > 0 && r != 1 { + minValue = int64(float64(minValue) * r) + maxValue = int64(float64(maxValue) * r) + } + + _, minUnit := measurement.Scale(minValue, o.SampleUnit, "minimum") + _, maxUnit := measurement.Scale(maxValue, o.SampleUnit, "minimum") + + unit := minUnit + if minUnit != maxUnit && minValue*100 < maxValue && o.OutputFormat != Callgrind { + // Minimum and maximum values have different units. Scale + // minimum by 100 to use larger units, allowing minimum value to + // be scaled down to 0.01, except for callgrind reports since + // they can only represent integer values. + _, unit = measurement.Scale(100*minValue, o.SampleUnit, "minimum") + } + + if unit != "" { + o.OutputUnit = unit + } else { + o.OutputUnit = o.SampleUnit + } +} + +// newGraph creates a new graph for this report. If nodes is non-nil, +// only nodes whose info matches are included. Otherwise, all nodes +// are included, without trimming. +func (rpt *Report) newGraph(nodes graph.NodeSet) *graph.Graph { + o := rpt.options + + // Clean up file paths using heuristics. + prof := rpt.prof + for _, f := range prof.Function { + f.Filename = trimPath(f.Filename, o.TrimPath, o.SourcePath) + } + // Removes all numeric tags except for the bytes tag prior + // to making graph. + // TODO: modify to select first numeric tag if no bytes tag + for _, s := range prof.Sample { + numLabels := make(map[string][]int64, len(s.NumLabel)) + numUnits := make(map[string][]string, len(s.NumLabel)) + for k, vs := range s.NumLabel { + if k == "bytes" { + unit := o.NumLabelUnits[k] + numValues := make([]int64, len(vs)) + numUnit := make([]string, len(vs)) + for i, v := range vs { + numValues[i] = v + numUnit[i] = unit + } + numLabels[k] = append(numLabels[k], numValues...) + numUnits[k] = append(numUnits[k], numUnit...) + } + } + s.NumLabel = numLabels + s.NumUnit = numUnits + } + + // Remove label marking samples from the base profiles, so it does not appear + // as a nodelet in the graph view. + prof.RemoveLabel("pprof::base") + + formatTag := func(v int64, key string) string { + return measurement.ScaledLabel(v, key, o.OutputUnit) + } + + gopt := &graph.Options{ + SampleValue: o.SampleValue, + SampleMeanDivisor: o.SampleMeanDivisor, + FormatTag: formatTag, + CallTree: o.CallTree && (o.OutputFormat == Dot || o.OutputFormat == Callgrind), + DropNegative: o.DropNegative, + KeptNodes: nodes, + } + + // Only keep binary names for disassembly-based reports, otherwise + // remove it to allow merging of functions across binaries. + switch o.OutputFormat { + case Raw, List, WebList, Dis, Callgrind: + gopt.ObjNames = true + } + + return graph.New(rpt.prof, gopt) +} + +// printProto writes the incoming proto via thw writer w. +// If the divide_by option has been specified, samples are scaled appropriately. +func printProto(w io.Writer, rpt *Report) error { + p, o := rpt.prof, rpt.options + + // Apply the sample ratio to all samples before saving the profile. + if r := o.Ratio; r > 0 && r != 1 { + for _, sample := range p.Sample { + for i, v := range sample.Value { + sample.Value[i] = int64(float64(v) * r) + } + } + } + return p.Write(w) +} + +// printTopProto writes a list of the hottest routines in a profile as a profile.proto. +func printTopProto(w io.Writer, rpt *Report) error { + p := rpt.prof + o := rpt.options + g, _, _, _ := rpt.newTrimmedGraph() + rpt.selectOutputUnit(g) + + out := profile.Profile{ + SampleType: []*profile.ValueType{ + {Type: "cum", Unit: o.OutputUnit}, + {Type: "flat", Unit: o.OutputUnit}, + }, + TimeNanos: p.TimeNanos, + DurationNanos: p.DurationNanos, + PeriodType: p.PeriodType, + Period: p.Period, + } + functionMap := make(functionMap) + for i, n := range g.Nodes { + f, added := functionMap.findOrAdd(n.Info) + if added { + out.Function = append(out.Function, f) + } + flat, cum := n.FlatValue(), n.CumValue() + l := &profile.Location{ + ID: uint64(i + 1), + Address: n.Info.Address, + Line: []profile.Line{ + { + Line: int64(n.Info.Lineno), + Function: f, + }, + }, + } + + fv, _ := measurement.Scale(flat, o.SampleUnit, o.OutputUnit) + cv, _ := measurement.Scale(cum, o.SampleUnit, o.OutputUnit) + s := &profile.Sample{ + Location: []*profile.Location{l}, + Value: []int64{int64(cv), int64(fv)}, + } + out.Location = append(out.Location, l) + out.Sample = append(out.Sample, s) + } + + return out.Write(w) +} + +type functionMap map[string]*profile.Function + +// findOrAdd takes a node representing a function, adds the function +// represented by the node to the map if the function is not already present, +// and returns the function the node represents. This also returns a boolean, +// which is true if the function was added and false otherwise. +func (fm functionMap) findOrAdd(ni graph.NodeInfo) (*profile.Function, bool) { + fName := fmt.Sprintf("%q%q%q%d", ni.Name, ni.OrigName, ni.File, ni.StartLine) + + if f := fm[fName]; f != nil { + return f, false + } + + f := &profile.Function{ + ID: uint64(len(fm) + 1), + Name: ni.Name, + SystemName: ni.OrigName, + Filename: ni.File, + StartLine: int64(ni.StartLine), + } + fm[fName] = f + return f, true +} + +type assemblyInstruction struct { + address uint64 + instruction string + function string + file string + line int + flat, cum int64 + flatDiv, cumDiv int64 + startsBlock bool + inlineCalls []callID +} + +type callID struct { + file string + line int +} + +func (a *assemblyInstruction) flatValue() int64 { + if a.flatDiv != 0 { + return a.flat / a.flatDiv + } + return a.flat +} + +func (a *assemblyInstruction) cumValue() int64 { + if a.cumDiv != 0 { + return a.cum / a.cumDiv + } + return a.cum +} + +// valueOrDot formats a value according to a report, intercepting zero +// values. +func valueOrDot(value int64, rpt *Report) string { + if value == 0 { + return "." + } + return rpt.formatValue(value) +} + +// printTags collects all tags referenced in the profile and prints +// them in a sorted table. +func printTags(w io.Writer, rpt *Report) error { + p := rpt.prof + + o := rpt.options + formatTag := func(v int64, key string) string { + return measurement.ScaledLabel(v, key, o.OutputUnit) + } + + // Hashtable to keep accumulate tags as key,value,count. + tagMap := make(map[string]map[string]int64) + for _, s := range p.Sample { + for key, vals := range s.Label { + for _, val := range vals { + valueMap, ok := tagMap[key] + if !ok { + valueMap = make(map[string]int64) + tagMap[key] = valueMap + } + valueMap[val] += o.SampleValue(s.Value) + } + } + for key, vals := range s.NumLabel { + unit := o.NumLabelUnits[key] + for _, nval := range vals { + val := formatTag(nval, unit) + valueMap, ok := tagMap[key] + if !ok { + valueMap = make(map[string]int64) + tagMap[key] = valueMap + } + valueMap[val] += o.SampleValue(s.Value) + } + } + } + + tagKeys := make([]*graph.Tag, 0, len(tagMap)) + for key := range tagMap { + tagKeys = append(tagKeys, &graph.Tag{Name: key}) + } + tabw := tabwriter.NewWriter(w, 0, 0, 1, ' ', tabwriter.AlignRight) + for _, tagKey := range graph.SortTags(tagKeys, true) { + var total int64 + key := tagKey.Name + tags := make([]*graph.Tag, 0, len(tagMap[key])) + for t, c := range tagMap[key] { + total += c + tags = append(tags, &graph.Tag{Name: t, Flat: c}) + } + + f, u := measurement.Scale(total, o.SampleUnit, o.OutputUnit) + fmt.Fprintf(tabw, "%s:\t Total %.1f%s\n", key, f, u) + for _, t := range graph.SortTags(tags, true) { + f, u := measurement.Scale(t.FlatValue(), o.SampleUnit, o.OutputUnit) + if total > 0 { + fmt.Fprintf(tabw, " \t%.1f%s (%s):\t %s\n", f, u, measurement.Percentage(t.FlatValue(), total), t.Name) + } else { + fmt.Fprintf(tabw, " \t%.1f%s:\t %s\n", f, u, t.Name) + } + } + fmt.Fprintln(tabw) + } + return tabw.Flush() +} + +// printComments prints all freeform comments in the profile. +func printComments(w io.Writer, rpt *Report) error { + p := rpt.prof + + for _, c := range p.Comments { + fmt.Fprintln(w, c) + } + return nil +} + +// TextItem holds a single text report entry. +type TextItem struct { + Name string + InlineLabel string // Not empty if inlined + Flat, Cum int64 // Raw values + FlatFormat, CumFormat string // Formatted values +} + +// TextItems returns a list of text items from the report and a list +// of labels that describe the report. +func TextItems(rpt *Report) ([]TextItem, []string) { + g, origCount, droppedNodes, _ := rpt.newTrimmedGraph() + rpt.selectOutputUnit(g) + labels := reportLabels(rpt, g, origCount, droppedNodes, 0, false) + + var items []TextItem + var flatSum int64 + for _, n := range g.Nodes { + name, flat, cum := n.Info.PrintableName(), n.FlatValue(), n.CumValue() + + var inline, noinline bool + for _, e := range n.In { + if e.Inline { + inline = true + } else { + noinline = true + } + } + + var inl string + if inline { + if noinline { + inl = "(partial-inline)" + } else { + inl = "(inline)" + } + } + + flatSum += flat + items = append(items, TextItem{ + Name: name, + InlineLabel: inl, + Flat: flat, + Cum: cum, + FlatFormat: rpt.formatValue(flat), + CumFormat: rpt.formatValue(cum), + }) + } + return items, labels +} + +// printText prints a flat text report for a profile. +func printText(w io.Writer, rpt *Report) error { + items, labels := TextItems(rpt) + fmt.Fprintln(w, strings.Join(labels, "\n")) + fmt.Fprintf(w, "%10s %5s%% %5s%% %10s %5s%%\n", + "flat", "flat", "sum", "cum", "cum") + var flatSum int64 + for _, item := range items { + inl := item.InlineLabel + if inl != "" { + inl = " " + inl + } + flatSum += item.Flat + fmt.Fprintf(w, "%10s %s %s %10s %s %s%s\n", + item.FlatFormat, measurement.Percentage(item.Flat, rpt.total), + measurement.Percentage(flatSum, rpt.total), + item.CumFormat, measurement.Percentage(item.Cum, rpt.total), + item.Name, inl) + } + return nil +} + +// printTraces prints all traces from a profile. +func printTraces(w io.Writer, rpt *Report) error { + fmt.Fprintln(w, strings.Join(ProfileLabels(rpt), "\n")) + + prof := rpt.prof + o := rpt.options + + const separator = "-----------+-------------------------------------------------------" + + _, locations := graph.CreateNodes(prof, &graph.Options{}) + for _, sample := range prof.Sample { + type stk struct { + *graph.NodeInfo + inline bool + } + var stack []stk + for _, loc := range sample.Location { + nodes := locations[loc.ID] + for i, n := range nodes { + // The inline flag may be inaccurate if 'show' or 'hide' filter is + // used. See https://github.com/google/pprof/issues/511. + inline := i != len(nodes)-1 + stack = append(stack, stk{&n.Info, inline}) + } + } + + if len(stack) == 0 { + continue + } + + fmt.Fprintln(w, separator) + // Print any text labels for the sample. + var labels []string + for s, vs := range sample.Label { + labels = append(labels, fmt.Sprintf("%10s: %s\n", s, strings.Join(vs, " "))) + } + sort.Strings(labels) + fmt.Fprint(w, strings.Join(labels, "")) + + // Print any numeric labels for the sample + var numLabels []string + for key, vals := range sample.NumLabel { + unit := o.NumLabelUnits[key] + numValues := make([]string, len(vals)) + for i, vv := range vals { + numValues[i] = measurement.Label(vv, unit) + } + numLabels = append(numLabels, fmt.Sprintf("%10s: %s\n", key, strings.Join(numValues, " "))) + } + sort.Strings(numLabels) + fmt.Fprint(w, strings.Join(numLabels, "")) + + var d, v int64 + v = o.SampleValue(sample.Value) + if o.SampleMeanDivisor != nil { + d = o.SampleMeanDivisor(sample.Value) + } + // Print call stack. + if d != 0 { + v = v / d + } + for i, s := range stack { + var vs, inline string + if i == 0 { + vs = rpt.formatValue(v) + } + if s.inline { + inline = " (inline)" + } + fmt.Fprintf(w, "%10s %s%s\n", vs, s.PrintableName(), inline) + } + } + fmt.Fprintln(w, separator) + return nil +} + +// printCallgrind prints a graph for a profile on callgrind format. +func printCallgrind(w io.Writer, rpt *Report) error { + o := rpt.options + rpt.options.NodeFraction = 0 + rpt.options.EdgeFraction = 0 + rpt.options.NodeCount = 0 + + g, _, _, _ := rpt.newTrimmedGraph() + rpt.selectOutputUnit(g) + + nodeNames := getDisambiguatedNames(g) + + fmt.Fprintln(w, "positions: instr line") + fmt.Fprintln(w, "events:", o.SampleType+"("+o.OutputUnit+")") + + objfiles := make(map[string]int) + files := make(map[string]int) + names := make(map[string]int) + + // prevInfo points to the previous NodeInfo. + // It is used to group cost lines together as much as possible. + var prevInfo *graph.NodeInfo + for _, n := range g.Nodes { + if prevInfo == nil || n.Info.Objfile != prevInfo.Objfile || n.Info.File != prevInfo.File || n.Info.Name != prevInfo.Name { + fmt.Fprintln(w) + fmt.Fprintln(w, "ob="+callgrindName(objfiles, n.Info.Objfile)) + fmt.Fprintln(w, "fl="+callgrindName(files, n.Info.File)) + fmt.Fprintln(w, "fn="+callgrindName(names, n.Info.Name)) + } + + addr := callgrindAddress(prevInfo, n.Info.Address) + sv, _ := measurement.Scale(n.FlatValue(), o.SampleUnit, o.OutputUnit) + fmt.Fprintf(w, "%s %d %d\n", addr, n.Info.Lineno, int64(sv)) + + // Print outgoing edges. + for _, out := range n.Out.Sort() { + c, _ := measurement.Scale(out.Weight, o.SampleUnit, o.OutputUnit) + callee := out.Dest + fmt.Fprintln(w, "cfl="+callgrindName(files, callee.Info.File)) + fmt.Fprintln(w, "cfn="+callgrindName(names, nodeNames[callee])) + // pprof doesn't have a flat weight for a call, leave as 0. + fmt.Fprintf(w, "calls=0 %s %d\n", callgrindAddress(prevInfo, callee.Info.Address), callee.Info.Lineno) + // TODO: This address may be in the middle of a call + // instruction. It would be best to find the beginning + // of the instruction, but the tools seem to handle + // this OK. + fmt.Fprintf(w, "* * %d\n", int64(c)) + } + + prevInfo = &n.Info + } + + return nil +} + +// getDisambiguatedNames returns a map from each node in the graph to +// the name to use in the callgrind output. Callgrind merges all +// functions with the same [file name, function name]. Add a [%d/n] +// suffix to disambiguate nodes with different values of +// node.Function, which we want to keep separate. In particular, this +// affects graphs created with --call_tree, where nodes from different +// contexts are associated to different Functions. +func getDisambiguatedNames(g *graph.Graph) map[*graph.Node]string { + nodeName := make(map[*graph.Node]string, len(g.Nodes)) + + type names struct { + file, function string + } + + // nameFunctionIndex maps the callgrind names (filename, function) + // to the node.Function values found for that name, and each + // node.Function value to a sequential index to be used on the + // disambiguated name. + nameFunctionIndex := make(map[names]map[*graph.Node]int) + for _, n := range g.Nodes { + nm := names{n.Info.File, n.Info.Name} + p, ok := nameFunctionIndex[nm] + if !ok { + p = make(map[*graph.Node]int) + nameFunctionIndex[nm] = p + } + if _, ok := p[n.Function]; !ok { + p[n.Function] = len(p) + } + } + + for _, n := range g.Nodes { + nm := names{n.Info.File, n.Info.Name} + nodeName[n] = n.Info.Name + if p := nameFunctionIndex[nm]; len(p) > 1 { + // If there is more than one function, add suffix to disambiguate. + nodeName[n] += fmt.Sprintf(" [%d/%d]", p[n.Function]+1, len(p)) + } + } + return nodeName +} + +// callgrindName implements the callgrind naming compression scheme. +// For names not previously seen returns "(N) name", where N is a +// unique index. For names previously seen returns "(N)" where N is +// the index returned the first time. +func callgrindName(names map[string]int, name string) string { + if name == "" { + return "" + } + if id, ok := names[name]; ok { + return fmt.Sprintf("(%d)", id) + } + id := len(names) + 1 + names[name] = id + return fmt.Sprintf("(%d) %s", id, name) +} + +// callgrindAddress implements the callgrind subposition compression scheme if +// possible. If prevInfo != nil, it contains the previous address. The current +// address can be given relative to the previous address, with an explicit +/- +// to indicate it is relative, or * for the same address. +func callgrindAddress(prevInfo *graph.NodeInfo, curr uint64) string { + abs := fmt.Sprintf("%#x", curr) + if prevInfo == nil { + return abs + } + + prev := prevInfo.Address + if prev == curr { + return "*" + } + + diff := int64(curr - prev) + relative := fmt.Sprintf("%+d", diff) + + // Only bother to use the relative address if it is actually shorter. + if len(relative) < len(abs) { + return relative + } + + return abs +} + +// printTree prints a tree-based report in text form. +func printTree(w io.Writer, rpt *Report) error { + const separator = "----------------------------------------------------------+-------------" + const legend = " flat flat% sum% cum cum% calls calls% + context " + + g, origCount, droppedNodes, _ := rpt.newTrimmedGraph() + rpt.selectOutputUnit(g) + + fmt.Fprintln(w, strings.Join(reportLabels(rpt, g, origCount, droppedNodes, 0, false), "\n")) + + fmt.Fprintln(w, separator) + fmt.Fprintln(w, legend) + var flatSum int64 + + rx := rpt.options.Symbol + matched := 0 + for _, n := range g.Nodes { + name, flat, cum := n.Info.PrintableName(), n.FlatValue(), n.CumValue() + + // Skip any entries that do not match the regexp (for the "peek" command). + if rx != nil && !rx.MatchString(name) { + continue + } + matched++ + + fmt.Fprintln(w, separator) + // Print incoming edges. + inEdges := n.In.Sort() + for _, in := range inEdges { + var inline string + if in.Inline { + inline = " (inline)" + } + fmt.Fprintf(w, "%50s %s | %s%s\n", rpt.formatValue(in.Weight), + measurement.Percentage(in.Weight, cum), in.Src.Info.PrintableName(), inline) + } + + // Print current node. + flatSum += flat + fmt.Fprintf(w, "%10s %s %s %10s %s | %s\n", + rpt.formatValue(flat), + measurement.Percentage(flat, rpt.total), + measurement.Percentage(flatSum, rpt.total), + rpt.formatValue(cum), + measurement.Percentage(cum, rpt.total), + name) + + // Print outgoing edges. + outEdges := n.Out.Sort() + for _, out := range outEdges { + var inline string + if out.Inline { + inline = " (inline)" + } + fmt.Fprintf(w, "%50s %s | %s%s\n", rpt.formatValue(out.Weight), + measurement.Percentage(out.Weight, cum), out.Dest.Info.PrintableName(), inline) + } + } + if len(g.Nodes) > 0 { + fmt.Fprintln(w, separator) + } + if rx != nil && matched == 0 { + return fmt.Errorf("no matches found for regexp: %s", rx) + } + return nil +} + +// GetDOT returns a graph suitable for dot processing along with some +// configuration information. +func GetDOT(rpt *Report) (*graph.Graph, *graph.DotConfig) { + g, origCount, droppedNodes, droppedEdges := rpt.newTrimmedGraph() + rpt.selectOutputUnit(g) + labels := reportLabels(rpt, g, origCount, droppedNodes, droppedEdges, true) + + c := &graph.DotConfig{ + Title: rpt.options.Title, + Labels: labels, + FormatValue: rpt.formatValue, + Total: rpt.total, + } + return g, c +} + +// printDOT prints an annotated callgraph in DOT format. +func printDOT(w io.Writer, rpt *Report) error { + g, c := GetDOT(rpt) + graph.ComposeDot(w, g, &graph.DotAttributes{}, c) + return nil +} + +// ProfileLabels returns printable labels for a profile. +func ProfileLabels(rpt *Report) []string { + label := []string{} + prof := rpt.prof + o := rpt.options + if len(prof.Mapping) > 0 { + if prof.Mapping[0].File != "" { + label = append(label, "File: "+filepath.Base(prof.Mapping[0].File)) + } + if prof.Mapping[0].BuildID != "" { + label = append(label, "Build ID: "+prof.Mapping[0].BuildID) + } + } + // Only include comments that do not start with '#'. + for _, c := range prof.Comments { + if !strings.HasPrefix(c, "#") { + label = append(label, c) + } + } + if o.SampleType != "" { + label = append(label, "Type: "+o.SampleType) + } + if prof.TimeNanos != 0 { + const layout = "Jan 2, 2006 at 3:04pm (MST)" + label = append(label, "Time: "+time.Unix(0, prof.TimeNanos).Format(layout)) + } + if prof.DurationNanos != 0 { + duration := measurement.Label(prof.DurationNanos, "nanoseconds") + totalNanos, totalUnit := measurement.Scale(rpt.total, o.SampleUnit, "nanoseconds") + var ratio string + if totalUnit == "ns" && totalNanos != 0 { + ratio = "(" + measurement.Percentage(int64(totalNanos), prof.DurationNanos) + ")" + } + label = append(label, fmt.Sprintf("Duration: %s, Total samples = %s %s", duration, rpt.formatValue(rpt.total), ratio)) + } + return label +} + +// reportLabels returns printable labels for a report. Includes +// profileLabels. +func reportLabels(rpt *Report, g *graph.Graph, origCount, droppedNodes, droppedEdges int, fullHeaders bool) []string { + nodeFraction := rpt.options.NodeFraction + edgeFraction := rpt.options.EdgeFraction + nodeCount := len(g.Nodes) + + var label []string + if len(rpt.options.ProfileLabels) > 0 { + label = append(label, rpt.options.ProfileLabels...) + } else if fullHeaders || !rpt.options.CompactLabels { + label = ProfileLabels(rpt) + } + + var flatSum int64 + for _, n := range g.Nodes { + flatSum = flatSum + n.FlatValue() + } + + if len(rpt.options.ActiveFilters) > 0 { + activeFilters := legendActiveFilters(rpt.options.ActiveFilters) + label = append(label, activeFilters...) + } + + label = append(label, fmt.Sprintf("Showing nodes accounting for %s, %s of %s total", rpt.formatValue(flatSum), strings.TrimSpace(measurement.Percentage(flatSum, rpt.total)), rpt.formatValue(rpt.total))) + + if rpt.total != 0 { + if droppedNodes > 0 { + label = append(label, genLabel(droppedNodes, "node", "cum", + rpt.formatValue(abs64(int64(float64(rpt.total)*nodeFraction))))) + } + if droppedEdges > 0 { + label = append(label, genLabel(droppedEdges, "edge", "freq", + rpt.formatValue(abs64(int64(float64(rpt.total)*edgeFraction))))) + } + if nodeCount > 0 && nodeCount < origCount { + label = append(label, fmt.Sprintf("Showing top %d nodes out of %d", + nodeCount, origCount)) + } + } + + // Help new users understand the graph. + // A new line is intentionally added here to better show this message. + if fullHeaders { + label = append(label, "\nSee https://git.io/JfYMW for how to read the graph") + } + + return label +} + +func legendActiveFilters(activeFilters []string) []string { + legendActiveFilters := make([]string, len(activeFilters)+1) + legendActiveFilters[0] = "Active filters:" + for i, s := range activeFilters { + if len(s) > 80 { + s = s[:80] + "…" + } + legendActiveFilters[i+1] = " " + s + } + return legendActiveFilters +} + +func genLabel(d int, n, l, f string) string { + if d > 1 { + n = n + "s" + } + return fmt.Sprintf("Dropped %d %s (%s <= %s)", d, n, l, f) +} + +// New builds a new report indexing the sample values interpreting the +// samples with the provided function. +func New(prof *profile.Profile, o *Options) *Report { + format := func(v int64) string { + if r := o.Ratio; r > 0 && r != 1 { + fv := float64(v) * r + v = int64(fv) + } + return measurement.ScaledLabel(v, o.SampleUnit, o.OutputUnit) + } + return &Report{prof, computeTotal(prof, o.SampleValue, o.SampleMeanDivisor), + o, format} +} + +// NewDefault builds a new report indexing the last sample value +// available. +func NewDefault(prof *profile.Profile, options Options) *Report { + index := len(prof.SampleType) - 1 + o := &options + if o.Title == "" && len(prof.Mapping) > 0 && prof.Mapping[0].File != "" { + o.Title = filepath.Base(prof.Mapping[0].File) + } + o.SampleType = prof.SampleType[index].Type + o.SampleUnit = strings.ToLower(prof.SampleType[index].Unit) + o.SampleValue = func(v []int64) int64 { + return v[index] + } + return New(prof, o) +} + +// computeTotal computes the sum of the absolute value of all sample values. +// If any samples have label indicating they belong to the diff base, then the +// total will only include samples with that label. +func computeTotal(prof *profile.Profile, value, meanDiv func(v []int64) int64) int64 { + var div, total, diffDiv, diffTotal int64 + for _, sample := range prof.Sample { + var d, v int64 + v = value(sample.Value) + if meanDiv != nil { + d = meanDiv(sample.Value) + } + if v < 0 { + v = -v + } + total += v + div += d + if sample.DiffBaseSample() { + diffTotal += v + diffDiv += d + } + } + if diffTotal > 0 { + total = diffTotal + div = diffDiv + } + if div != 0 { + return total / div + } + return total +} + +// Report contains the data and associated routines to extract a +// report from a profile. +type Report struct { + prof *profile.Profile + total int64 + options *Options + formatValue func(int64) string +} + +// Total returns the total number of samples in a report. +func (rpt *Report) Total() int64 { return rpt.total } + +func abs64(i int64) int64 { + if i < 0 { + return -i + } + return i +} + +func trimPath(path, trimPath, searchPath string) string { + const gnarkRoot = "github.com/consensys/gnark/" + + // Keep path variable intact as it's used below to form the return value. + sPath, searchPath := filepath.ToSlash(path), filepath.ToSlash(searchPath) + + if idx := strings.Index(sPath, gnarkRoot); idx != -1 { + sPath = sPath[idx+len(gnarkRoot):] + return sPath + } + + if trimPath == "" { + // If the trim path is not configured, try to guess it heuristically: + // search for basename of each search path in the original path and, if + // found, strip everything up to and including the basename. So, for + // example, given original path "/some/remote/path/my-project/foo/bar.c" + // and search path "/my/local/path/my-project" the heuristic will return + // "/my/local/path/my-project/foo/bar.c". + for _, dir := range filepath.SplitList(searchPath) { + want := "/" + filepath.Base(dir) + "/" + if found := strings.Index(sPath, want); found != -1 { + return path[found+len(want):] + } + } + } + // Trim configured trim prefixes. + trimPaths := append(filepath.SplitList(filepath.ToSlash(trimPath)), "/proc/self/cwd/./", "/proc/self/cwd/") + for _, trimPath := range trimPaths { + if !strings.HasSuffix(trimPath, "/") { + trimPath += "/" + } + if strings.HasPrefix(sPath, trimPath) { + return path[len(trimPath):] + } + } + return path +} diff --git a/profile/internal/report/synth.go b/profile/internal/report/synth.go new file mode 100644 index 0000000000..7a35bbcda8 --- /dev/null +++ b/profile/internal/report/synth.go @@ -0,0 +1,39 @@ +package report + +import ( + "github.com/google/pprof/profile" +) + +// synthCode assigns addresses to locations without an address. +type synthCode struct { + next uint64 + addr map[*profile.Location]uint64 // Synthesized address assigned to a location +} + +func newSynthCode(mappings []*profile.Mapping) *synthCode { + // Find a larger address than any mapping. + s := &synthCode{next: 1} + for _, m := range mappings { + if s.next < m.Limit { + s.next = m.Limit + } + } + return s +} + +// address returns the synthetic address for loc, creating one if needed. +func (s *synthCode) address(loc *profile.Location) uint64 { + if loc.Address != 0 { + panic("can only synthesize addresses for locations without an address") + } + if addr, ok := s.addr[loc]; ok { + return addr + } + if s.addr == nil { + s.addr = map[*profile.Location]uint64{} + } + addr := s.next + s.next++ + s.addr[loc] = addr + return addr +} diff --git a/profile/profile.go b/profile/profile.go index b6dece84e5..e8d6ec6233 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -5,6 +5,7 @@ package profile import ( + "bytes" "os" "path/filepath" "runtime" @@ -13,6 +14,7 @@ import ( "sync/atomic" "github.com/consensys/gnark/logger" + "github.com/consensys/gnark/profile/internal/report" "github.com/google/pprof/profile" ) @@ -61,33 +63,33 @@ func Start(options ...func(*Profile)) *Profile { go worker() }) - prof := Profile{ + p := Profile{ functions: make(map[string]*profile.Function), locations: make(map[uint64]*profile.Location), filePath: filepath.Join(".", "gnark.pprof"), chDone: make(chan struct{}), } - prof.pprof.SampleType = []*profile.ValueType{{ + p.pprof.SampleType = []*profile.ValueType{{ Type: "constraints", Unit: "count", }} for _, option := range options { - option(&prof) + option(&p) } log := logger.Logger() - if prof.filePath == "" { + if p.filePath == "" { log.Warn().Msg("gnark profiling enabled [not writting to disk]") } else { - log.Info().Str("path", prof.filePath).Msg("gnark profiling enabled") + log.Info().Str("path", p.filePath).Msg("gnark profiling enabled") } // add the session to active sessions - chCommands <- command{p: &prof} + chCommands <- command{p: &p} atomic.AddUint32(&activeSessions, 1) - return &prof + return &p } // Stop removes the profile from active session and may write the pprof file to disk. See ProfilePath option. @@ -122,6 +124,21 @@ func (p *Profile) Stop() { } +// Top return a similar output than pprof top command +func (p *Profile) Top() string { + r := report.NewDefault(&p.pprof, report.Options{ + OutputFormat: report.Text, + CompactLabels: true, + NodeFraction: 0.005, + EdgeFraction: 0.001, + SampleValue: func(v []int64) int64 { return v[0] }, + SampleUnit: "count", + }) + var buf bytes.Buffer + report.Generate(&buf, r) + return buf.String() +} + // RecordConstraint add a sample (with count == 1) to all the active profiling sessions. func RecordConstraint() { if n := atomic.LoadUint32(&activeSessions); n == 0 { @@ -139,9 +156,6 @@ func RecordConstraint() { } func (p *Profile) getLocation(frame *runtime.Frame) *profile.Location { - - // location - // locationID := frame.File + strconv.Itoa(frame.Line) l, ok := p.locations[uint64(frame.PC)] if !ok { // first let's see if we have the function. From adb0d8e7f352724ebc82c3ed82b9088952bb9a34 Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Tue, 2 Aug 2022 14:23:29 -0500 Subject: [PATCH 04/11] feat: added profile.NbConstraints() --- profile/internal/report/synth.go | 39 -------------------------------- profile/profile.go | 5 ++++ 2 files changed, 5 insertions(+), 39 deletions(-) delete mode 100644 profile/internal/report/synth.go diff --git a/profile/internal/report/synth.go b/profile/internal/report/synth.go deleted file mode 100644 index 7a35bbcda8..0000000000 --- a/profile/internal/report/synth.go +++ /dev/null @@ -1,39 +0,0 @@ -package report - -import ( - "github.com/google/pprof/profile" -) - -// synthCode assigns addresses to locations without an address. -type synthCode struct { - next uint64 - addr map[*profile.Location]uint64 // Synthesized address assigned to a location -} - -func newSynthCode(mappings []*profile.Mapping) *synthCode { - // Find a larger address than any mapping. - s := &synthCode{next: 1} - for _, m := range mappings { - if s.next < m.Limit { - s.next = m.Limit - } - } - return s -} - -// address returns the synthetic address for loc, creating one if needed. -func (s *synthCode) address(loc *profile.Location) uint64 { - if loc.Address != 0 { - panic("can only synthesize addresses for locations without an address") - } - if addr, ok := s.addr[loc]; ok { - return addr - } - if s.addr == nil { - s.addr = map[*profile.Location]uint64{} - } - addr := s.next - s.next++ - s.addr[loc] = addr - return addr -} diff --git a/profile/profile.go b/profile/profile.go index e8d6ec6233..d631a25e00 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -124,6 +124,11 @@ func (p *Profile) Stop() { } +// NbConstraints return number of collected samples (constraints) by the profile session +func (p *Profile) NbConstraints() int { + return len(p.pprof.Sample) +} + // Top return a similar output than pprof top command func (p *Profile) Top() string { r := report.NewDefault(&p.pprof, report.Options{ From 389521c73f51a08c58dec0192962b7c906513029 Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Tue, 2 Aug 2022 14:38:08 -0500 Subject: [PATCH 05/11] docs: added package level profile example --- profile/profile_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 profile/profile_test.go diff --git a/profile/profile_test.go b/profile/profile_test.go new file mode 100644 index 0000000000..7bd1f2f777 --- /dev/null +++ b/profile/profile_test.go @@ -0,0 +1,38 @@ +package profile_test + +import ( + "fmt" + + "github.com/consensys/gnark-crypto/ecc" + "github.com/consensys/gnark/frontend" + "github.com/consensys/gnark/frontend/cs/r1cs" + "github.com/consensys/gnark/profile" +) + +type Circuit struct { + A frontend.Variable +} + +func (circuit *Circuit) Define(api frontend.API) error { + api.AssertIsEqual(api.Mul(circuit.A, circuit.A), circuit.A) + return nil +} + +func Example() { + // default options generate gnark.pprof in current dir + // use pprof as usual (go tool pprof -http=:8080 gnark.pprof) to read the profile file + // overlapping profiles are allowed (define profiles inside Define or subfunction to profile + // part of the circuit only) + p := profile.Start() + _, _ = frontend.Compile(ecc.BN254.ScalarField(), r1cs.NewBuilder, &Circuit{}) + p.Stop() + + fmt.Println(p.NbConstraints()) + fmt.Println(p.Top()) + // Output: + // 2 + // Showing nodes accounting for 2, 100% of 2 total + // flat flat% sum% cum cum% + // 1 50.00% 50.00% 2 100% profile_test.(*Circuit).Define profile/profile_test.go:17 + // 1 50.00% 100% 1 50.00% r1cs.(*r1cs).AssertIsEqual frontend/cs/r1cs/api_assertions.go:37 +} From 2ed19e602fecaf2abf1852dcbd8e0eb1b8413154 Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Tue, 2 Aug 2022 14:42:29 -0500 Subject: [PATCH 06/11] style: code cleaning --- examples/rollup/circuit_test.go | 3 --- profile/profile.go | 9 +++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/examples/rollup/circuit_test.go b/examples/rollup/circuit_test.go index 4b51fc907c..0fc65549c7 100644 --- a/examples/rollup/circuit_test.go +++ b/examples/rollup/circuit_test.go @@ -22,7 +22,6 @@ import ( "github.com/consensys/gnark-crypto/ecc" "github.com/consensys/gnark/backend" "github.com/consensys/gnark/frontend" - "github.com/consensys/gnark/profile" "github.com/consensys/gnark/std/hash/mimc" "github.com/consensys/gnark/test" ) @@ -267,12 +266,10 @@ func TestCircuitFull(t *testing.T) { } // TODO full circuit has some unconstrained inputs, that's odd. - p := profile.Start() assert.ProverSucceeded( &rollupCircuit, &operator.witnesses, test.WithCurves(ecc.BN254), test.WithBackends(backend.GROTH16)) - p.Stop() } diff --git a/profile/profile.go b/profile/profile.go index d631a25e00..6ac2042e81 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -50,6 +50,15 @@ func ProfilePath(path string) func(*Profile) { } } +// ProfileNoOutput indicates that the profile is not going to be written to disk. +// +// This is equivalent to ProfilePath("") +func ProfileNoOutput() func(*Profile) { + return func(p *Profile) { + p.filePath = "" + } +} + // Start creates a new active profiling session. When Stop() is called, this session is removed from // active profiling sessions and may be serialized to disk as a pprof compatible file (see ProfilePath option). // From b35cf71966273c8ce0cbb20b494db1a280fd1cfd Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Tue, 2 Aug 2022 15:22:42 -0500 Subject: [PATCH 07/11] fix: fix CI trim path for test purposes --- profile/internal/report/report.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/profile/internal/report/report.go b/profile/internal/report/report.go index 2207d5f78d..01b03e349d 100644 --- a/profile/internal/report/report.go +++ b/profile/internal/report/report.go @@ -1056,6 +1056,7 @@ func abs64(i int64) int64 { func trimPath(path, trimPath, searchPath string) string { const gnarkRoot = "github.com/consensys/gnark/" + const gnarkRootCI = "home/runner/work/gnark/" // Keep path variable intact as it's used below to form the return value. sPath, searchPath := filepath.ToSlash(path), filepath.ToSlash(searchPath) @@ -1065,6 +1066,11 @@ func trimPath(path, trimPath, searchPath string) string { return sPath } + if idx := strings.Index(sPath, gnarkRootCI); idx != -1 { + sPath = sPath[idx+len(gnarkRootCI):] + return sPath + } + if trimPath == "" { // If the trim path is not configured, try to guess it heuristically: // search for basename of each search path in the original path and, if From d588bf18ca3df893d955ec993db2d113818f7b7d Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Tue, 2 Aug 2022 15:30:22 -0500 Subject: [PATCH 08/11] fix: fix previous commit --- profile/internal/report/report.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profile/internal/report/report.go b/profile/internal/report/report.go index 01b03e349d..83bc264cbd 100644 --- a/profile/internal/report/report.go +++ b/profile/internal/report/report.go @@ -1056,7 +1056,7 @@ func abs64(i int64) int64 { func trimPath(path, trimPath, searchPath string) string { const gnarkRoot = "github.com/consensys/gnark/" - const gnarkRootCI = "home/runner/work/gnark/" + const gnarkRootCI = "home/runner/work/gnark/gnark/" // Keep path variable intact as it's used below to form the return value. sPath, searchPath := filepath.ToSlash(path), filepath.ToSlash(searchPath) From 23fb5d64ed1870b07c78beb0bdf396730e3deb5d Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Tue, 2 Aug 2022 16:30:11 -0500 Subject: [PATCH 09/11] clean: revert part of #347 with redundant debug info --- debug/debug.go | 25 +----- debug/debug_test.go | 46 ----------- frontend/ccs.go | 6 -- frontend/compiled/cs.go | 77 +------------------ frontend/compiled/log.go | 5 -- internal/backend/bls12-377/cs/r1cs_test.go | 4 +- internal/backend/bls12-381/cs/r1cs_test.go | 4 +- internal/backend/bls24-315/cs/r1cs_test.go | 4 +- internal/backend/bls24-317/cs/r1cs_test.go | 4 +- internal/backend/bn254/cs/r1cs_test.go | 4 +- internal/backend/bw6-633/cs/r1cs_test.go | 4 +- internal/backend/bw6-761/cs/r1cs_test.go | 4 +- .../representations/tests/r1cs.go.tmpl | 4 +- internal/tinyfield/cs/r1cs_test.go | 4 +- 14 files changed, 12 insertions(+), 183 deletions(-) delete mode 100644 debug/debug_test.go diff --git a/debug/debug.go b/debug/debug.go index 0714cd6fd3..fd6d2ccd4e 100644 --- a/debug/debug.go +++ b/debug/debug.go @@ -1,35 +1,12 @@ package debug import ( - "errors" "path/filepath" "runtime" "strconv" "strings" ) -type StackLine struct { - Line uint32 - File string -} - -// ParseStack parses a stack as stored in a log entry and return readable data -func ParseStack(stack []uint64, stackPaths map[uint32]string) ([]StackLine, error) { - r := make([]StackLine, len(stack)) - - for i, s := range stack { - pID := uint32(s >> 32) - line := uint32(s) - path, ok := stackPaths[pID] - if !ok { - return nil, errors.New("missing stack path in stackPaths map") - } - r[i] = StackLine{Line: line, File: path} - } - - return r, nil -} - func Stack() string { var sbb strings.Builder WriteStack(&sbb) @@ -41,7 +18,7 @@ func WriteStack(sbb *strings.Builder, forceClean ...bool) { // we stop when func name == Define as it is where the gnark circuit code should start // Ask runtime.Callers for up to 10 pcs - pc := make([]uintptr, 10) + pc := make([]uintptr, 20) n := runtime.Callers(3, pc) if n == 0 { // No pcs available. Stop now. diff --git a/debug/debug_test.go b/debug/debug_test.go deleted file mode 100644 index 55d6226f2e..0000000000 --- a/debug/debug_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package debug - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestParseStack(t *testing.T) { - assert := require.New(t) - - const ( - f1 = "/usr/local/file1.go" - f2 = "/usr/local/file2.go" - f3 = "/usr/local/lib/file3.go" - ) - - stackPaths := make(map[uint32]string) - stackPaths[0] = f1 - stackPaths[1] = f2 - stackPaths[2] = f3 - - stack := []uint64{ - uint64(0<<32) | 27, - uint64(1<<32) | 42, - uint64(2<<32) | 2, - } - - parsed, err := ParseStack(stack, stackPaths) - assert.NoError(err) - - assert.True(len(parsed) == 3) - assert.Equal(f1, parsed[0].File) - assert.Equal(uint32(27), parsed[0].Line) - - assert.Equal(f2, parsed[1].File) - assert.Equal(uint32(42), parsed[1].Line) - - assert.Equal(f3, parsed[2].File) - assert.Equal(uint32(2), parsed[2].Line) - - stack = append(stack, uint64(8000<<32)|2) - _, err = ParseStack(stack, stackPaths) - assert.Error(err) - -} diff --git a/frontend/ccs.go b/frontend/ccs.go index a46a86bab3..3e8ac594c7 100644 --- a/frontend/ccs.go +++ b/frontend/ccs.go @@ -47,10 +47,4 @@ type CompiledConstraintSystem interface { // GetConstraints return a human readable representation of the constraints GetConstraints() [][]string - - // GetDebugInfo return the list of debug info per constraints and the map to restore file path - // from the debug info. When compiled with -tags=debug, each constraint has an associated - // stack encoded as a []uint64. The uint64 pack uint32(fileID) | uint32(lineNumber). - // The file string can be retrieved from associated map. - GetDebugInfo() ([][]uint64, map[uint32]string) } diff --git a/frontend/compiled/cs.go b/frontend/compiled/cs.go index b00d7704e2..561175d193 100644 --- a/frontend/compiled/cs.go +++ b/frontend/compiled/cs.go @@ -3,7 +3,6 @@ package compiled import ( "fmt" "math/big" - "runtime" "strings" "github.com/blang/semver/v4" @@ -41,11 +40,6 @@ type ConstraintSystem struct { // debug info contains stack trace (including line number) of a call to a cs.API that // results in an unsolved constraint DebugInfo []LogEntry - // maps unique id to a path (optimized for reading debug info stacks) - DebugStackPaths map[uint32]string - // maps a path to an id (optimized for storing debug info stacks) - debugPathsIds map[string]uint32 `cbor:"-"` - debugPathId uint32 `cbor:"-"` // maps constraint id to debugInfo id // several constraints may point to the same debug info @@ -74,8 +68,6 @@ func NewConstraintSystem(scalarField *big.Int) ConstraintSystem { MDebug: make(map[int]int), MHints: make(map[int]*Hint), MHintsDependencies: make(map[hint.ID]string), - DebugStackPaths: make(map[uint32]string), - debugPathsIds: make(map[string]uint32), q: new(big.Int).Set(scalarField), bitLen: scalarField.BitLen(), } @@ -144,15 +136,6 @@ func (cs *ConstraintSystem) AddDebugInfo(errName string, i ...interface{}) int { var l LogEntry - // add the stack info - // TODO @gbotrel duplicate with Debug.stack below - l.Stack = cs.stack() - - if errName == "" { - cs.DebugInfo = append(cs.DebugInfo, l) - return len(cs.DebugInfo) - 1 - } - const minLogSize = 500 var sbb strings.Builder sbb.Grow(minLogSize) @@ -180,6 +163,8 @@ func (cs *ConstraintSystem) AddDebugInfo(errName string, i ...interface{}) int { } } sbb.WriteByte('\n') + // TODO this stack should not be stored as string, but as a slice of locations + // to avoid overloading with lots of str duplicate the serialized constraint system debug.WriteStack(&sbb) l.Format = sbb.String() @@ -192,61 +177,3 @@ func (cs *ConstraintSystem) AddDebugInfo(errName string, i ...interface{}) int { func (cs *ConstraintSystem) FieldBitLen() int { return cs.bitLen } - -func (cs *ConstraintSystem) GetDebugInfo() ([][]uint64, map[uint32]string) { - r := make([][]uint64, len(cs.DebugInfo)) - for _, l := range cs.DebugInfo { - r = append(r, l.Stack) - } - return r, cs.DebugStackPaths -} - -func (cs *ConstraintSystem) stack() (r []uint64) { - if !debug.Debug { - return - } - // derived from: https://golang.org/pkg/runtime/#example_Frames - // we stop when func name == Define as it is where the gnark circuit code should start - - // Ask runtime.Callers for up to 10 pcs - pc := make([]uintptr, 10) - n := runtime.Callers(3, pc) - if n == 0 { - // No pcs available. Stop now. - // This can happen if the first argument to runtime.Callers is large. - return - } - pc = pc[:n] // pass only valid pcs to runtime.CallersFrames - frames := runtime.CallersFrames(pc) - // Loop to get frames. - // A fixed number of pcs can expand to an indefinite number of Frames. - for { - frame, more := frames.Next() - fe := strings.Split(frame.Function, "/") - function := fe[len(fe)-1] - file := frame.File - - if strings.Contains(frame.File, "gnark/frontend") { - continue - } - - // TODO @gbotrel this stores an absolute path, so will work only locally - id, ok := cs.debugPathsIds[file] - if !ok { - id = cs.debugPathId - cs.debugPathId++ - cs.debugPathsIds[file] = id - cs.DebugStackPaths[id] = file - } - r = append(r, ((uint64(id) << 32) | uint64(frame.Line))) - if !more { - break - } - if strings.HasSuffix(function, "Define") { - break - } - } - - return - -} diff --git a/frontend/compiled/log.go b/frontend/compiled/log.go index 407866ebf4..23370cea9a 100644 --- a/frontend/compiled/log.go +++ b/frontend/compiled/log.go @@ -27,11 +27,6 @@ type LogEntry struct { Caller string Format string ToResolve []Term - - // When compiled with -tags=debug, each constraint has an associated - // stack encoded as a []uint64. The uint64 pack uint32(fileID) | uint32(lineNumber). - // The actual string describing the file is stored in a map in the compiled.ConstraintSystem. - Stack []uint64 } func (l *LogEntry) WriteVariable(le LinearExpression, sbb *strings.Builder) { diff --git a/internal/backend/bls12-377/cs/r1cs_test.go b/internal/backend/bls12-377/cs/r1cs_test.go index a40df67d6f..8139a9a293 100644 --- a/internal/backend/bls12-377/cs/r1cs_test.go +++ b/internal/backend/bls12-377/cs/r1cs_test.go @@ -76,9 +76,7 @@ func TestSerialization(t *testing.T) { if diff := cmp.Diff(r1cs1, &reconstructed, cmpopts.IgnoreFields(cs.R1CS{}, "ConstraintSystem.q", - "ConstraintSystem.bitLen", - "ConstraintSystem.debugPathsIds", - "ConstraintSystem.debugPathId")); diff != "" { + "ConstraintSystem.bitLen")); diff != "" { t.Fatalf("round trip mismatch (-want +got):\n%s", diff) } } diff --git a/internal/backend/bls12-381/cs/r1cs_test.go b/internal/backend/bls12-381/cs/r1cs_test.go index 369fcbb22a..36a3046e3f 100644 --- a/internal/backend/bls12-381/cs/r1cs_test.go +++ b/internal/backend/bls12-381/cs/r1cs_test.go @@ -76,9 +76,7 @@ func TestSerialization(t *testing.T) { if diff := cmp.Diff(r1cs1, &reconstructed, cmpopts.IgnoreFields(cs.R1CS{}, "ConstraintSystem.q", - "ConstraintSystem.bitLen", - "ConstraintSystem.debugPathsIds", - "ConstraintSystem.debugPathId")); diff != "" { + "ConstraintSystem.bitLen")); diff != "" { t.Fatalf("round trip mismatch (-want +got):\n%s", diff) } } diff --git a/internal/backend/bls24-315/cs/r1cs_test.go b/internal/backend/bls24-315/cs/r1cs_test.go index 9237c704b2..2e4662d704 100644 --- a/internal/backend/bls24-315/cs/r1cs_test.go +++ b/internal/backend/bls24-315/cs/r1cs_test.go @@ -76,9 +76,7 @@ func TestSerialization(t *testing.T) { if diff := cmp.Diff(r1cs1, &reconstructed, cmpopts.IgnoreFields(cs.R1CS{}, "ConstraintSystem.q", - "ConstraintSystem.bitLen", - "ConstraintSystem.debugPathsIds", - "ConstraintSystem.debugPathId")); diff != "" { + "ConstraintSystem.bitLen")); diff != "" { t.Fatalf("round trip mismatch (-want +got):\n%s", diff) } } diff --git a/internal/backend/bls24-317/cs/r1cs_test.go b/internal/backend/bls24-317/cs/r1cs_test.go index 79be7c1def..0b0f8efd27 100644 --- a/internal/backend/bls24-317/cs/r1cs_test.go +++ b/internal/backend/bls24-317/cs/r1cs_test.go @@ -76,9 +76,7 @@ func TestSerialization(t *testing.T) { if diff := cmp.Diff(r1cs1, &reconstructed, cmpopts.IgnoreFields(cs.R1CS{}, "ConstraintSystem.q", - "ConstraintSystem.bitLen", - "ConstraintSystem.debugPathsIds", - "ConstraintSystem.debugPathId")); diff != "" { + "ConstraintSystem.bitLen")); diff != "" { t.Fatalf("round trip mismatch (-want +got):\n%s", diff) } } diff --git a/internal/backend/bn254/cs/r1cs_test.go b/internal/backend/bn254/cs/r1cs_test.go index b8c61d3e37..b2c1946435 100644 --- a/internal/backend/bn254/cs/r1cs_test.go +++ b/internal/backend/bn254/cs/r1cs_test.go @@ -76,9 +76,7 @@ func TestSerialization(t *testing.T) { if diff := cmp.Diff(r1cs1, &reconstructed, cmpopts.IgnoreFields(cs.R1CS{}, "ConstraintSystem.q", - "ConstraintSystem.bitLen", - "ConstraintSystem.debugPathsIds", - "ConstraintSystem.debugPathId")); diff != "" { + "ConstraintSystem.bitLen")); diff != "" { t.Fatalf("round trip mismatch (-want +got):\n%s", diff) } } diff --git a/internal/backend/bw6-633/cs/r1cs_test.go b/internal/backend/bw6-633/cs/r1cs_test.go index c65d74da87..f889f63ed2 100644 --- a/internal/backend/bw6-633/cs/r1cs_test.go +++ b/internal/backend/bw6-633/cs/r1cs_test.go @@ -76,9 +76,7 @@ func TestSerialization(t *testing.T) { if diff := cmp.Diff(r1cs1, &reconstructed, cmpopts.IgnoreFields(cs.R1CS{}, "ConstraintSystem.q", - "ConstraintSystem.bitLen", - "ConstraintSystem.debugPathsIds", - "ConstraintSystem.debugPathId")); diff != "" { + "ConstraintSystem.bitLen")); diff != "" { t.Fatalf("round trip mismatch (-want +got):\n%s", diff) } } diff --git a/internal/backend/bw6-761/cs/r1cs_test.go b/internal/backend/bw6-761/cs/r1cs_test.go index 909f98c478..556bd43466 100644 --- a/internal/backend/bw6-761/cs/r1cs_test.go +++ b/internal/backend/bw6-761/cs/r1cs_test.go @@ -79,9 +79,7 @@ func TestSerialization(t *testing.T) { if diff := cmp.Diff(r1cs1, &reconstructed, cmpopts.IgnoreFields(cs.R1CS{}, "ConstraintSystem.q", - "ConstraintSystem.bitLen", - "ConstraintSystem.debugPathsIds", - "ConstraintSystem.debugPathId")); diff != "" { + "ConstraintSystem.bitLen")); diff != "" { t.Fatalf("round trip mismatch (-want +got):\n%s", diff) } } diff --git a/internal/generator/backend/template/representations/tests/r1cs.go.tmpl b/internal/generator/backend/template/representations/tests/r1cs.go.tmpl index eb1d484abe..797b781a19 100644 --- a/internal/generator/backend/template/representations/tests/r1cs.go.tmpl +++ b/internal/generator/backend/template/representations/tests/r1cs.go.tmpl @@ -67,9 +67,7 @@ func TestSerialization(t *testing.T) { if diff := cmp.Diff(r1cs1, &reconstructed, cmpopts.IgnoreFields(cs.R1CS{}, "ConstraintSystem.q", - "ConstraintSystem.bitLen", - "ConstraintSystem.debugPathsIds", - "ConstraintSystem.debugPathId")); diff != "" { + "ConstraintSystem.bitLen")); diff != "" { t.Fatalf("round trip mismatch (-want +got):\n%s", diff) } } diff --git a/internal/tinyfield/cs/r1cs_test.go b/internal/tinyfield/cs/r1cs_test.go index b0ebfcaf5b..707452bd32 100644 --- a/internal/tinyfield/cs/r1cs_test.go +++ b/internal/tinyfield/cs/r1cs_test.go @@ -79,9 +79,7 @@ func TestSerialization(t *testing.T) { if diff := cmp.Diff(r1cs1, &reconstructed, cmpopts.IgnoreFields(cs.R1CS{}, "ConstraintSystem.q", - "ConstraintSystem.bitLen", - "ConstraintSystem.debugPathsIds", - "ConstraintSystem.debugPathId")); diff != "" { + "ConstraintSystem.bitLen")); diff != "" { t.Fatalf("round trip mismatch (-want +got):\n%s", diff) } } From 57bf774b935ec74567758b8043955fa86655de82 Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Tue, 2 Aug 2022 16:36:43 -0500 Subject: [PATCH 10/11] clean: remove deadcode and kill api.Tag and api.Counter --- frontend/api.go | 8 ------ frontend/builder.go | 9 ------ frontend/ccs.go | 4 --- frontend/compiled/cs.go | 18 ------------ frontend/counter.go | 30 ------------------- frontend/cs/r1cs/builder.go | 26 ----------------- frontend/cs/scs/builder.go | 28 ------------------ profile/internal/report/report.go | 40 -------------------------- std/algebra/fields_bls12377/e2_test.go | 3 -- std/math/emulated/field.go | 8 ------ test/assert.go | 23 --------------- test/engine.go | 9 ------ 12 files changed, 206 deletions(-) delete mode 100644 frontend/counter.go diff --git a/frontend/api.go b/frontend/api.go index 6272b965b3..c339a8b693 100644 --- a/frontend/api.go +++ b/frontend/api.go @@ -119,14 +119,6 @@ type API interface { // Deprecated: use api.Compiler().NewHint() instead NewHint(f hint.Function, nbOutputs int, inputs ...Variable) ([]Variable, error) - // Tag is a shorcut to api.Compiler().Tag() - // Deprecated: use api.Compiler().Tag() instead - Tag(name string) Tag - - // AddCounter is a shorcut to api.Compiler().AddCounter() - // Deprecated: use api.Compiler().AddCounter() instead - AddCounter(from, to Tag) - // ConstantValue is a shorcut to api.Compiler().ConstantValue() // Deprecated: use api.Compiler().ConstantValue() instead ConstantValue(v Variable) (*big.Int, bool) diff --git a/frontend/builder.go b/frontend/builder.go index 64dceca527..38011b8f7e 100644 --- a/frontend/builder.go +++ b/frontend/builder.go @@ -38,15 +38,6 @@ type Compiler interface { // If nbOutputs is specified, it must be >= 1 and <= f.NbOutputs NewHint(f hint.Function, nbOutputs int, inputs ...Variable) ([]Variable, error) - // Tag creates a tag at a given place in a circuit. The state of the tag may contain informations needed to - // measure constraints, variables and coefficients creations through AddCounter - Tag(name string) Tag - - // AddCounter measures the number of constraints, variables and coefficients created between two tags - // note that the PlonK statistics are contextual since there is a post-compile phase where linear expressions - // are factorized. That is, measuring 2 times the "repeating" piece of circuit may give less constraints the second time - AddCounter(from, to Tag) - // ConstantValue returns the big.Int value of v and true if op is a success. // nil and false if failure. This API returns a boolean to allow for future refactoring // replacing *big.Int with fr.Element diff --git a/frontend/ccs.go b/frontend/ccs.go index 3e8ac594c7..2fac57ac9d 100644 --- a/frontend/ccs.go +++ b/frontend/ccs.go @@ -20,7 +20,6 @@ import ( "github.com/consensys/gnark-crypto/ecc" "github.com/consensys/gnark/backend" "github.com/consensys/gnark/backend/witness" - "github.com/consensys/gnark/frontend/compiled" "github.com/consensys/gnark/frontend/schema" ) @@ -40,9 +39,6 @@ type CompiledConstraintSystem interface { CurveID() ecc.ID - // GetCounters return the collected constraint counters, if any - GetCounters() []compiled.Counter - GetSchema() *schema.Schema // GetConstraints return a human readable representation of the constraints diff --git a/frontend/compiled/cs.go b/frontend/compiled/cs.go index 561175d193..ea5616d470 100644 --- a/frontend/compiled/cs.go +++ b/frontend/compiled/cs.go @@ -8,7 +8,6 @@ import ( "github.com/blang/semver/v4" "github.com/consensys/gnark" "github.com/consensys/gnark-crypto/ecc" - "github.com/consensys/gnark/backend" "github.com/consensys/gnark/backend/hint" "github.com/consensys/gnark/debug" "github.com/consensys/gnark/frontend/schema" @@ -45,8 +44,6 @@ type ConstraintSystem struct { // several constraints may point to the same debug info MDebug map[int]int - Counters []Counter `cbor:"-"` - MHints map[int]*Hint // maps wireID to hint MHintsDependencies map[hint.ID]string // maps hintID to hint string identifier @@ -115,23 +112,8 @@ func (cs *ConstraintSystem) Field() *big.Int { return new(big.Int).Set(cs.q) } -// GetCounters return the collected constraint counters, if any -func (cs *ConstraintSystem) GetCounters() []Counter { return cs.Counters } - func (cs *ConstraintSystem) GetSchema() *schema.Schema { return cs.Schema } -// Counter contains measurements of useful statistics between two Tag -type Counter struct { - From, To string - NbVariables int - NbConstraints int - BackendID backend.ID -} - -func (c Counter) String() string { - return fmt.Sprintf("%s %s - %s: %d variables, %d constraints", c.BackendID, c.From, c.To, c.NbVariables, c.NbConstraints) -} - func (cs *ConstraintSystem) AddDebugInfo(errName string, i ...interface{}) int { var l LogEntry diff --git a/frontend/counter.go b/frontend/counter.go deleted file mode 100644 index 4285a3dd25..0000000000 --- a/frontend/counter.go +++ /dev/null @@ -1,30 +0,0 @@ -/* -Copyright © 2020 ConsenSys - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package frontend - -// Tag contains informations needed to measure and display statistics of a delimited piece of circuit -type Tag struct { - Name string - VID, CID int -} - -// Counter contains measurements of useful statistics between two Tag -type Counter struct { - From, To Tag - NbVariables int - NbConstraints int -} diff --git a/frontend/cs/r1cs/builder.go b/frontend/cs/r1cs/builder.go index 0b4d2ffeeb..534b816983 100644 --- a/frontend/cs/r1cs/builder.go +++ b/frontend/cs/r1cs/builder.go @@ -20,15 +20,12 @@ import ( "errors" "fmt" "math/big" - "path/filepath" "reflect" - "runtime" "sort" "strconv" "strings" "github.com/consensys/gnark-crypto/ecc" - "github.com/consensys/gnark/backend" "github.com/consensys/gnark/backend/hint" "github.com/consensys/gnark/debug" "github.com/consensys/gnark/frontend" @@ -572,29 +569,6 @@ func (system *r1cs) toVariables(in ...frontend.Variable) ([]compiled.LinearExpre return r, s } -// Tag creates a tag at a given place in a circuit. The state of the tag may contain informations needed to -// measure constraints, variables and coefficients creations through AddCounter -func (system *r1cs) Tag(name string) frontend.Tag { - _, file, line, _ := runtime.Caller(1) - - return frontend.Tag{ - Name: fmt.Sprintf("%s[%s:%d]", name, filepath.Base(file), line), - VID: system.NbInternalVariables, - CID: len(system.Constraints), - } -} - -// AddCounter measures the number of constraints, variables and coefficients created between two tags -func (system *r1cs) AddCounter(from, to frontend.Tag) { - system.Counters = append(system.Counters, compiled.Counter{ - From: from.Name, - To: to.Name, - NbVariables: to.VID - from.VID, - NbConstraints: to.CID - from.CID, - BackendID: backend.GROTH16, - }) -} - // NewHint initializes internal variables whose value will be evaluated using // the provided hint function at run time from the inputs. Inputs must be either // variables or convertible to *big.Int. The function returns an error if the diff --git a/frontend/cs/scs/builder.go b/frontend/cs/scs/builder.go index 08d15886ec..0392cf6ce8 100644 --- a/frontend/cs/scs/builder.go +++ b/frontend/cs/scs/builder.go @@ -20,15 +20,12 @@ import ( "errors" "fmt" "math/big" - "path/filepath" "reflect" - "runtime" "sort" "strconv" "strings" "github.com/consensys/gnark-crypto/ecc" - "github.com/consensys/gnark/backend" "github.com/consensys/gnark/backend/hint" "github.com/consensys/gnark/debug" "github.com/consensys/gnark/frontend" @@ -476,31 +473,6 @@ func (system *scs) ConstantValue(v frontend.Variable) (*big.Int, bool) { } } -// Tag creates a tag at a given place in a circuit. The state of the tag may contain informations needed to -// measure constraints, variables and coefficients creations through AddCounter -func (system *scs) Tag(name string) frontend.Tag { - _, file, line, _ := runtime.Caller(1) - - return frontend.Tag{ - Name: fmt.Sprintf("%s[%s:%d]", name, filepath.Base(file), line), - VID: system.NbInternalVariables, - CID: len(system.Constraints), - } -} - -// AddCounter measures the number of constraints, variables and coefficients created between two tags -// note that the PlonK statistics are contextual since there is a post-compile phase where linear expressions -// are factorized. That is, measuring 2 times the "repeating" piece of circuit may give less constraints the second time -func (system *scs) AddCounter(from, to frontend.Tag) { - system.Counters = append(system.Counters, compiled.Counter{ - From: from.Name, - To: to.Name, - NbVariables: to.VID - from.VID, - NbConstraints: to.CID - from.CID, - BackendID: backend.PLONK, - }) -} - // NewHint initializes internal variables whose value will be evaluated using // the provided hint function at run time from the inputs. Inputs must be either // variables or convertible to *big.Int. The function returns an error if the diff --git a/profile/internal/report/report.go b/profile/internal/report/report.go index 83bc264cbd..2bb48c34b6 100644 --- a/profile/internal/report/report.go +++ b/profile/internal/report/report.go @@ -373,46 +373,6 @@ func (fm functionMap) findOrAdd(ni graph.NodeInfo) (*profile.Function, bool) { return f, true } -type assemblyInstruction struct { - address uint64 - instruction string - function string - file string - line int - flat, cum int64 - flatDiv, cumDiv int64 - startsBlock bool - inlineCalls []callID -} - -type callID struct { - file string - line int -} - -func (a *assemblyInstruction) flatValue() int64 { - if a.flatDiv != 0 { - return a.flat / a.flatDiv - } - return a.flat -} - -func (a *assemblyInstruction) cumValue() int64 { - if a.cumDiv != 0 { - return a.cum / a.cumDiv - } - return a.cum -} - -// valueOrDot formats a value according to a report, intercepting zero -// values. -func valueOrDot(value int64, rpt *Report) string { - if value == 0 { - return "." - } - return rpt.formatValue(value) -} - // printTags collects all tags referenced in the profile and prints // them in a sorted table. func printTags(w io.Writer, rpt *Report) error { diff --git a/std/algebra/fields_bls12377/e2_test.go b/std/algebra/fields_bls12377/e2_test.go index ab65964f25..2d069a20df 100644 --- a/std/algebra/fields_bls12377/e2_test.go +++ b/std/algebra/fields_bls12377/e2_test.go @@ -264,9 +264,6 @@ func TestMulByNonResidueFp2(t *testing.T) { // fp2c := NewFp2Elmt(&cs, nil, nil) // fp2c.MulByNonResidue(&cs, &fp2a) - // api.Tag(fp2c.X, "c0") - // api.Tag(fp2c.Y, "c1") - // // witness.A.A0 = (a.A0) // witness.A.A1 = (a.A1) diff --git a/std/math/emulated/field.go b/std/math/emulated/field.go index 3cec8af812..588991da0c 100644 --- a/std/math/emulated/field.go +++ b/std/math/emulated/field.go @@ -513,14 +513,6 @@ func (f *field[T]) NewHint(hf hint.Function, nbOutputs int, inputs ...frontend.V return ret, nil } -func (f *field[T]) Tag(name string) frontend.Tag { - return f.api.Compiler().Tag(name) -} - -func (f *field[T]) AddCounter(from frontend.Tag, to frontend.Tag) { - f.api.Compiler().AddCounter(from, to) -} - func (f *field[T]) ConstantValue(v frontend.Variable) (*big.Int, bool) { var limbs []frontend.Variable // emulated limbs switch vv := v.(type) { diff --git a/test/assert.go b/test/assert.go index e667aaa613..1bd0f2de5e 100644 --- a/test/assert.go +++ b/test/assert.go @@ -31,7 +31,6 @@ import ( "github.com/consensys/gnark/backend/plonkfri" "github.com/consensys/gnark/backend/witness" "github.com/consensys/gnark/frontend" - "github.com/consensys/gnark/frontend/compiled" "github.com/consensys/gnark/frontend/cs/r1cs" "github.com/consensys/gnark/frontend/cs/scs" "github.com/stretchr/testify/require" @@ -338,28 +337,6 @@ func (assert *Assert) solvingFailed(circuit frontend.Circuit, invalidAssignment } -// GetCounters compiles (or fetch from the compiled circuit cache) the circuit with set backends and curves -// and returns measured counters -func (assert *Assert) GetCounters(circuit frontend.Circuit, opts ...TestingOption) []compiled.Counter { - opt := assert.options(opts...) - - var r []compiled.Counter - - for _, curve := range opt.curves { - for _, b := range opt.backends { - curve := curve - b := b - assert.Run(func(assert *Assert) { - ccs, err := assert.compile(circuit, curve, b, opt.compileOpts) - assert.NoError(err) - r = append(r, ccs.GetCounters()...) - }, curve.String(), b.String()) - } - } - - return r -} - // Fuzz fuzzes the given circuit by instantiating "randomized" witnesses and cross checking // execution result between constraint system solver and big.Int test execution engine // diff --git a/test/engine.go b/test/engine.go index d823230608..434199b68b 100644 --- a/test/engine.go +++ b/test/engine.go @@ -465,15 +465,6 @@ func (e *engine) MarkBoolean(v frontend.Variable) { } } -func (e *engine) Tag(name string) frontend.Tag { - // do nothing, we don't measure constraints with the test engine - return frontend.Tag{Name: name} -} - -func (e *engine) AddCounter(from, to frontend.Tag) { - // do nothing, we don't measure constraints with the test engine -} - func (e *engine) toBigInt(i1 frontend.Variable) *big.Int { switch vv := i1.(type) { case *big.Int: From 511ea9905ca29dbfde652ca6bfbea090b853b602 Mon Sep 17 00:00:00 2001 From: Gautam Botrel Date: Wed, 3 Aug 2022 08:27:41 -0500 Subject: [PATCH 11/11] fix: saving files helps --- profile/profile.go | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/profile/profile.go b/profile/profile.go index 6dfee2a607..3d3778d995 100644 --- a/profile/profile.go +++ b/profile/profile.go @@ -41,12 +41,6 @@ type Profile struct { chDone chan struct{} } -<<<<<<< HEAD -// ProfilePath controls the profile destination file. If blank, profile is not written. -// -// Defaults to ./gnark.pprof. -func ProfilePath(path string) func(*Profile) { -======= // Option defines configuration Options for Profile. type Option func(*Profile) @@ -54,23 +48,15 @@ type Option func(*Profile) // // Defaults to ./gnark.pprof. func WithPath(path string) Option { ->>>>>>> develop return func(p *Profile) { p.filePath = path } } -<<<<<<< HEAD -// ProfileNoOutput indicates that the profile is not going to be written to disk. -// -// This is equivalent to ProfilePath("") -func ProfileNoOutput() func(*Profile) { -======= // WithNoOutput indicates that the profile is not going to be written to disk. // // This is equivalent to WithPath("") func WithNoOutput() Option { ->>>>>>> develop return func(p *Profile) { p.filePath = "" } @@ -82,11 +68,7 @@ func WithNoOutput() Option { // All calls to profile.Start() and Stop() are meant to be executed in the same go routine (frontend.Compile). // // It is allowed to create multiple overlapping profiling sessions in one circuit. -<<<<<<< HEAD -func Start(options ...func(*Profile)) *Profile { -======= func Start(options ...Option) *Profile { ->>>>>>> develop // start the worker first time a profiling session starts. onceInit.Do(func() {