Track a few anonymous metrics to measure Dgraph adoption

dgraph/cmd/zero/raft.go

@@ -22,6 +22,8 @@ import (
 	"github.com/dgraph-io/dgraph/conn"
 	"github.com/dgraph-io/dgraph/protos/intern"
 	"github.com/dgraph-io/dgraph/x"
+	"github.com/golang/glog"
+	"github.com/google/uuid"
 	"golang.org/x/net/context"
 	"golang.org/x/net/trace"
 )
@@ -178,6 +180,12 @@ func (n *node) applyProposal(e raftpb.Entry) (string, error) {
 
 	state := n.server.state
 	state.Counter = e.Index
+	if len(p.Cid) > 0 {
+		if len(state.Cid) > 0 {
+			return p.Key, errInvalidProposal
+		}
+		state.Cid = p.Cid
+	}
 	if p.MaxRaftId > 0 {
 		if p.MaxRaftId <= state.MaxRaftId {
 			return p.Key, errInvalidProposal
@@ -411,6 +419,23 @@ func (n *node) initAndStartNode() error {
 		x.Check(err)
 		peers := []raft.Peer{{ID: n.Id, Context: data}}
 		n.SetRaft(raft.StartNode(n.Cfg, peers))
+
+		go func() {
+			// This is a new cluster. So, propose a new ID for the cluster.
+			for {
+				id := uuid.New().String()
+				err := n.proposeAndWait(context.Background(), &intern.ZeroProposal{Cid: id})
+				if err == nil {
+					glog.Infof("CID set for cluster: %v", id)
+					return
+				}
+				if err == errInvalidProposal {
+					return
+				}
+				glog.Errorf("While proposing CID: %v. Retrying...", err)
+				time.Sleep(3 * time.Second)
+			}
+		}()
 	}
 
 	go n.Run()

dgraph/cmd/zero/run.go

@@ -72,6 +72,7 @@ instances to achieve high-availability.
 	flag.String("peer", "", "Address of another dgraphzero server.")
 	flag.StringP("wal", "w", "zw", "Directory storing WAL.")
 	flag.Duration("rebalance_interval", 8*time.Minute, "Interval for trying a predicate move.")
+	flag.Bool("telemetry", true, "Send anonymous telemetry data to Dgraph devs.")
 }
 
 func setupListener(addr string, port int, kind string) (listener net.Listener, err error) {
@@ -196,6 +197,10 @@ func run() {
 	// This must be here. It does not work if placed before Grpc init.
 	x.Check(st.node.initAndStartNode())
 
+	if Zero.Conf.GetBool("telemetry") {
+		go st.zero.periodicallyPostTelemetry()
+	}
+
 	sdCh := make(chan os.Signal, 1)
 	signal.Notify(sdCh, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
 

dgraph/cmd/zero/telemetry.go

@@ -0,0 +1,94 @@
+/*
+ * Copyright 2018 Dgraph Labs, Inc.
+ *
+ * This file is available under the Apache License, Version 2.0,
+ * with the Commons Clause restriction.
+ */
+
+package zero
+
+import (
+	"bytes"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"runtime"
+
+	"github.com/dgraph-io/dgraph/protos/intern"
+	"github.com/dgraph-io/dgraph/x"
+	"github.com/golang/glog"
+)
+
+type Telemetry struct {
+	Arch        string
+	Cid         string
+	ClusterSize int
+	DiskUsageMB int64
+	NumAlphas   int
+	NumGroups   int
+	NumTablets  int
+	NumZeros    int
+	OS          string
+	SinceHours  int
+	Version     string
+}
+
+var keenUrl = "https://api.keen.io/3.0/projects/5b809dfac9e77c0001783ad0/events"
+
+func newTelemetry(ms *intern.MembershipState) *Telemetry {
+	if len(ms.Cid) == 0 {
+		glog.V(2).Infoln("No CID found yet")
+		return nil
+	}
+	t := &Telemetry{
+		Cid:       ms.Cid,
+		NumGroups: len(ms.GetGroups()),
+		NumZeros:  len(ms.GetZeros()),
+		Version:   x.Version(),
+		OS:        runtime.GOOS,
+		Arch:      runtime.GOARCH,
+	}
+	for _, g := range ms.GetGroups() {
+		t.NumAlphas += len(g.GetMembers())
+		for _, tablet := range g.GetTablets() {
+			t.NumTablets++
+			t.DiskUsageMB += tablet.GetSpace()
+		}
+	}
+	t.DiskUsageMB /= (1 << 20)
+	t.ClusterSize = t.NumAlphas + t.NumZeros
+	glog.V(2).Infof("Posting Telemetry data: %+v", t)
+	return t
+}
+
+func (t *Telemetry) post() error {
+	data, err := json.Marshal(t)
+	if err != nil {
+		return err
+	}
+	url := keenUrl + "/dev"
+	if len(t.Version) > 0 {
+		url = keenUrl + "/pings"
+	}
+	req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Authorization", "D0398E8C83BB30F67C519FDA6175975F680921890C35B36C34BE109544597497CA758881BD7D56CC2355A2F36B4560102CBC3279AC7B27E5391372C36A31167EB0D06BF3764894AD20A0554BAFF14C292A40BC252BB9FF008736A0FD1D44E085")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	glog.V(2).Infof("Telemetry response status: %v", resp.Status)
+	glog.V(2).Infof("Telemetry response body: %s", body)
+	return nil
+}

dgraph/cmd/zero/zero.go

@@ -19,6 +19,7 @@ import (
 	"github.com/dgraph-io/dgraph/protos/intern"
 	"github.com/dgraph-io/dgraph/x"
 	"github.com/gogo/protobuf/proto"
+	"github.com/golang/glog"
 )
 
 var (
@@ -74,6 +75,37 @@ func (s *Server) Init() {
 	go s.purgeOracle()
 }
 
+func (s *Server) periodicallyPostTelemetry() {
+	glog.V(2).Infof("Starting telemetry data collection...")
+	start := time.Now()
+
+	ticker := time.NewTicker(time.Minute)
+	defer ticker.Stop()
+
+	var lastPostedAt time.Time
+	for range ticker.C {
+		if !s.Node.AmLeader() {
+			continue
+		}
+		if time.Since(lastPostedAt) < time.Hour {
+			continue
+		}
+		ms := s.membershipState()
+		t := newTelemetry(ms)
+		if t == nil {
+			continue
+		}
+		t.SinceHours = int(time.Since(start).Hours())
+		glog.V(2).Infof("Posting Telemetry data: %+v", t)
+
+		err := t.post()
+		glog.V(2).Infof("Telemetry data posted with error: %v", err)
+		if err == nil {
+			lastPostedAt = time.Now()
+		}
+	}
+}
+
 func (s *Server) triggerLeaderChange() {
 	s.Lock()
 	defer s.Unlock()

dgraph/docker-compose.yml

@@ -5,13 +5,13 @@
 version: "3.5"
 services:
   zero1:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg0.1
     working_dir: /data/dg0.1
     ports:
       - 5080:5080
       - 6080:6080
-    command: /gobin/dgraph zero --my=zero1:5080 --replicas 3 --idx 1 --bindall --expose_trace --profile_mode block --block_rate 10
+    command: /gobin/dgraph zero --my=zero1:5080 --replicas 3 --idx 1 --bindall --expose_trace --profile_mode block --block_rate 10 --logtostderr -v=2
     volumes:
       - type: bind
         source: $GOPATH/bin
@@ -22,7 +22,7 @@ services:
       #   target: /data
 
   zero2:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg0.2
     command: /gobin/dgraph zero -o 2 --my=zero2:5082 --replicas 3 --peer=zero1:5080 --idx 2
     ports:
@@ -35,7 +35,7 @@ services:
         read_only: true
 
   zero3:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg0.3
     command: /gobin/dgraph zero -o 3 --my=zero3:5083 --replicas 3 --peer=zero1:5080 --idx 3
     ports:
@@ -49,6 +49,7 @@ services:
 
   # zero4:
   #   image: debian:latest
+    # image: dgraph/dgraph:latest
   #   container_name: bank-dg0.4
   #   command: /gobin/dgraph zero --my=zero4:5080 --replicas 3 --peer=zero1:5080 --idx 4
   #   volumes:
@@ -59,6 +60,7 @@ services:
 
   # zero5:
   #   image: debian:latest
+    # image: dgraph/dgraph:latest
   #   container_name: bank-dg0.5
   #   command: /gobin/dgraph zero --my=zero5:5080 --replicas 3 --peer=zero1:5080 --idx 5
   #   volumes:
@@ -68,7 +70,7 @@ services:
   #       read_only: true
 
   dg1:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg1
     working_dir: /data/dg1
     volumes:
@@ -85,7 +87,7 @@ services:
     command: /gobin/dgraph server --my=dg1:7180 --lru_mb=1024 --zero=zero1:5080 -o 100 --expose_trace  --trace 1.0 --profile_mode block --block_rate 10
 
   dg2:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg2
     working_dir: /data/dg2
     volumes:
@@ -102,7 +104,7 @@ services:
     command: /gobin/dgraph server --my=dg2:7182 --lru_mb=1024 --zero=zero1:5080 -o 102 --expose_trace  --trace 1.0 --profile_mode block --block_rate 10
 
   dg3:
-    image: debian:latest
+    image: dgraph/dgraph:latest
     container_name: bank-dg3
     working_dir: /data/dg3
     volumes:
@@ -120,6 +122,7 @@ services:
 
   # dg4:
   #   image: debian:latest
+    # image: dgraph/dgraph:latest
   #   container_name: bank-dg4
   #   working_dir: /data/dg4
   #   volumes:
@@ -137,6 +140,7 @@ services:
 
   # dg5:
   #   image: debian:latest
+    # image: dgraph/dgraph:latest
   #   container_name: bank-dg5
   #   working_dir: /data/dg5
   #   volumes: