Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support to show metrics #4033

Merged
merged 13 commits into from
Nov 7, 2019
Merged

Add support to show metrics #4033

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
e25e61a
add metrics for attributes
Sep 20, 2019
382e964
call recursively
Sep 20, 2019
0a8c19d
change destuid to src uid
poonai Oct 10, 2019
fefda14
moved to execution result
Oct 14, 2019
c7c8608
Merge branch 'master' of https://github.com/dgraph-io/dgraph into bal…
Oct 22, 2019
b4a5cff
add api
Oct 22, 2019
24eea69
test added
Oct 22, 2019
9e43fb3
don't return
poonai Oct 22, 2019
2550922
typo fixed
poonai Oct 22, 2019
d1a0444
Merge branch 'master' of https://github.com/dgraph-io/dgraph into bal…
poonai Nov 1, 2019
ffa28fc
fix sys test
Nov 5, 2019
071e164
increment one for the systest index test. beacuse, num_uid will give …
Nov 5, 2019
c5c10a8
Merge branch 'master' of https://github.com/dgraph-io/dgraph into bal…
Nov 6, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions contrib/scripts/goldendata-queries.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,15 +41,15 @@ function run_index_test {
}

echo -e "Running some queries and checking count of results returned."
run_index_test basic name 138676
run_index_test allof_the name 25431
run_index_test allof_the_a name 367
run_index_test allof_the_first name 4383
run_index_test releasedate release_date 137858
run_index_test releasedate_sort release_date 137858
run_index_test releasedate_sort_first_offset release_date 2315
run_index_test releasedate_geq release_date 60991
run_index_test gen_anyof_good_bad name 1103
run_index_test basic name 138677
run_index_test allof_the name 25432
run_index_test allof_the_a name 368
run_index_test allof_the_first name 4384
run_index_test releasedate release_date 137859
run_index_test releasedate_sort release_date 137859
run_index_test releasedate_sort_first_offset release_date 2316
run_index_test releasedate_geq release_date 60992
run_index_test gen_anyof_good_bad name 1104

popd &> /dev/null

1 change: 1 addition & 0 deletions dgraph/cmd/alpha/http.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,7 @@ func queryHandler(w http.ResponseWriter, r *http.Request) {
e := query.Extensions{
Txn: resp.Txn,
Latency: resp.Latency,
Metrics: resp.Metrics,
}
js, err := json.Marshal(e)
if err != nil {
Expand Down
3 changes: 3 additions & 0 deletions edgraph/server.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1064,6 +1064,9 @@ func (s *Server) doQuery(ctx context.Context, req *api.Request, authorize int) (
}

resp.Latency = gl
resp.Metrics = &api.Metrics{
NumUids: er.Metrics,
}
return resp, err
}

Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,15 +15,15 @@ require (
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgraph-io/badger v0.0.0-20190917133922-cbdef65095c7
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191011032519-062f5605f6da
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191021171913-3efa60e5593b
github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534 // indirect
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2
github.com/dgryski/go-groupvarint v0.0.0-20190318181831-5ce5df8ca4e1
github.com/dustin/go-humanize v1.0.0
github.com/go-ini/ini v1.39.0 // indirect
github.com/go-sql-driver/mysql v0.0.0-20190330032241-c0f6b444ad8f
github.com/gogo/protobuf v1.2.0
github.com/gogo/protobuf v1.3.1
github.com/golang/geo v0.0.0-20170810003146-31fb0106dc4a
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
github.com/golang/protobuf v1.3.2
Expand Down
6 changes: 6 additions & 0 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,8 @@ github.com/dgraph-io/badger v0.0.0-20190917133922-cbdef65095c7 h1:sXNo2Xtte2hOGe
github.com/dgraph-io/badger v0.0.0-20190917133922-cbdef65095c7/go.mod h1:V7oadAIwiqodXo5jT04lP+8ysOAuMFZO5x20VSI94Uc=
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191011032519-062f5605f6da h1:JAadBE19xm15ZsBZ9HOSuQvMiaCKu+NsV5O5Mfb/62M=
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191011032519-062f5605f6da/go.mod h1:R/MTZMGhTo60XSziuKpLXzI1OnVWQCZS5oJjxA8Q1bo=
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191021171913-3efa60e5593b h1:p3gcEqQqQBp4jPIYO5x05sZbY0n8WVp+V1F/EnbJ1oo=
github.com/dgraph-io/dgo/v2 v2.1.1-0.20191021171913-3efa60e5593b/go.mod h1:LJCkLxm5fUMcU+yb8gHFjHt7ChgNuz3YnQQ6MQkmscI=
github.com/dgraph-io/ristretto v0.0.0-20190903064322-eb48d2f7ca30 h1:FkdGlqxPjfHKdVUzS5dOSMeOkGjEG7PnR1RLbcEqw88=
github.com/dgraph-io/ristretto v0.0.0-20190903064322-eb48d2f7ca30/go.mod h1:UvZmzj8odp3S1nli6yEb1vLME8iJFBrRcw8rAJEiu9Q=
github.com/dgraph-io/ristretto v0.0.0-20191010170704-2ba187ef9534 h1:9G6fVccQriMJu4nXwpwLDoy9y31t/KUSLAbPcoBgv+4=
Expand Down Expand Up @@ -90,6 +92,8 @@ github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
github.com/golang/geo v0.0.0-20170810003146-31fb0106dc4a h1:DG/Rx1VnnaqyPhKoPFuU61p4N7lkF5//weoP7QwddNs=
github.com/golang/geo v0.0.0-20170810003146-31fb0106dc4a/go.mod h1:vgWZ7cu0fq0KY3PpEHsocXOWJpRtkcbKemU4IUw0M60=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
Expand Down Expand Up @@ -131,6 +135,7 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
Expand Down Expand Up @@ -302,6 +307,7 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
Expand Down
10 changes: 10 additions & 0 deletions query/common_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,16 @@ func processQueryNoErr(t *testing.T, query string) string {
return res
}

// processQueryForMetrics works like processQuery but returns metrics instead of response.
func processQueryForMetrics(t *testing.T, query string) *api.Metrics {
txn := client.NewTxn()
defer txn.Discard(context.Background())

res, err := txn.Query(context.Background(), query)
require.NoError(t, err)
return res.Metrics
}

func processQueryWithVars(t *testing.T, query string,
vars map[string]string) (string, error) {
txn := client.NewTxn()
Expand Down
1 change: 1 addition & 0 deletions query/outputnode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,7 @@ func processNodeUids(fj *fastJsonNode, sg *SubGraph) error {
type Extensions struct {
Latency *api.Latency `json:"server_latency,omitempty"`
Txn *api.TxnContext `json:"txn,omitempty"`
Metrics *api.Metrics `json:"metrics,omitempty"`
}

func (sg *SubGraph) toFastJSON(l *Latency) ([]byte, error) {
Expand Down
27 changes: 27 additions & 0 deletions query/query.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2699,6 +2699,7 @@ type ExecutionResult struct {
Subgraphs []*SubGraph
SchemaNode []*pb.SchemaNode
Types []*pb.TypeUpdate
Metrics map[string]uint64
}

// Process handles a query request.
Expand All @@ -2708,6 +2709,12 @@ func (req *Request) Process(ctx context.Context) (er ExecutionResult, err error)
return er, err
}
er.Subgraphs = req.Subgraphs
// calculate metrics.
metrics := make(map[string]uint64)
for _, sg := range er.Subgraphs {
calculateMetrics(sg, metrics)
}
er.Metrics = metrics

schemaProcessingStart := time.Now()
if req.GqlQuery.Schema != nil {
Expand All @@ -2733,3 +2740,23 @@ func StripBlankNode(mp map[string]uint64) map[string]uint64 {
}
return temp
}

// calculateMetrics populates the given map with the number of uids are gathered for each
// attributes.
func calculateMetrics(sg *SubGraph, metrics map[string]uint64) {

// skip internal nodes.
if !sg.IsInternal() {
// we'll calculate srcUid of the each attribute. because, these are number of uids
// processed by this attribute.
metrics[sg.Attr] = metrics[sg.Attr] + uint64(len(sg.SrcUIDs.GetUids()))
}
// add all the uids gathered by filters
for _, filter := range sg.Filters {
calculateMetrics(filter, metrics)
}
// calculate metrics for the children as well.
for _, child := range sg.Children {
calculateMetrics(child, metrics)
}
}
14 changes: 14 additions & 0 deletions query/query4_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1158,3 +1158,17 @@ func TestCountUIDNestedMultiple(t *testing.T) {
}
}`, js)
}

func TestNumUids(t *testing.T) {
query := `{
me(func:has(name), first:10){
name
friend{
name
}
}
}`
metrics := processQueryForMetrics(t, query)
require.Equal(t, metrics.NumUids["friend"], uint64(10))
require.Equal(t, metrics.NumUids["name"], uint64(16))
}