diff --git a/.gitignore b/.gitignore index 15cbc64529..191bb70307 100644 --- a/.gitignore +++ b/.gitignore @@ -48,4 +48,3 @@ experimental/example/goapp/app *sublime-project *sublime-workspace *npm-debug.log - diff --git a/app/api_report.go b/app/api_report.go deleted file mode 100644 index f4bfa329c9..0000000000 --- a/app/api_report.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - "net/http" -) - -// Raw report handler -func makeRawReportHandler(rep Reporter) func(http.ResponseWriter, *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - // r.ParseForm() - respondWith(w, http.StatusOK, rep.Report()) - } -} diff --git a/app/api_report_test.go b/app/api_report_test.go deleted file mode 100644 index 737a9173c2..0000000000 --- a/app/api_report_test.go +++ /dev/null @@ -1,24 +0,0 @@ -package main - -import ( - "encoding/json" - "net/http/httptest" - "testing" - - "github.com/weaveworks/scope/report" -) - -func TestAPIReport(t *testing.T) { - ts := httptest.NewServer(Router(StaticReport{})) - defer ts.Close() - - is404(t, ts, "/api/report/foobar") - - var body = getRawJSON(t, ts, "/api/report") - // fmt.Printf("Body: %v\n", string(body)) - var r report.Report - err := json.Unmarshal(body, &r) - if err != nil { - t.Fatalf("JSON parse error: %s", err) - } -} diff --git a/app/api_topologies.go b/app/api_topologies.go deleted file mode 100644 index 8fef79c2ec..0000000000 --- a/app/api_topologies.go +++ /dev/null @@ -1,72 +0,0 @@ -package main - -import ( - "net/http" - "strings" - - "github.com/weaveworks/scope/report" -) - -// APITopologyDesc is returned in a list by the /api/topology handler. -type APITopologyDesc struct { - Name string `json:"name"` - URL string `json:"url"` - GroupedURL string `json:"grouped_url,omitempty"` - Stats topologyStats `json:"stats"` -} - -type topologyStats struct { - NodeCount int `json:"node_count"` - NonpseudoNodeCount int `json:"nonpseudo_node_count"` - EdgeCount int `json:"edge_count"` -} - -// makeTopologyList returns a handler that yields an APITopologyList. -func makeTopologyList(rep Reporter) func(w http.ResponseWriter, r *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - rpt := rep.Report() - - var a []APITopologyDesc - for name, def := range topologyRegistry { - if strings.HasSuffix(name, "grouped") { - continue - } - - url := "/api/topology/" + name - var groupedURL string - if def.groupedTopology != "" { - groupedURL = "/api/topology/" + def.groupedTopology - } - - a = append(a, APITopologyDesc{ - Name: def.human, - URL: url, - GroupedURL: groupedURL, - Stats: stats(def.selector(rpt).RenderBy(def.mapper, def.pseudo)), - }) - } - respondWith(w, http.StatusOK, a) - } -} - -func stats(r map[string]report.RenderableNode) topologyStats { - var ( - nodes int - realNodes int - edges int - ) - - for _, n := range r { - nodes++ - if !n.Pseudo { - realNodes++ - } - edges += len(n.Adjacency) - } - - return topologyStats{ - NodeCount: nodes, - NonpseudoNodeCount: realNodes, - EdgeCount: edges, - } -} diff --git a/app/api_topologies_test.go b/app/api_topologies_test.go deleted file mode 100644 index 0c9b67da4e..0000000000 --- a/app/api_topologies_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package main - -import ( - "encoding/json" - "net/http/httptest" - "testing" -) - -func TestAPITopology(t *testing.T) { - ts := httptest.NewServer(Router(StaticReport{})) - defer ts.Close() - - body := getRawJSON(t, ts, "/api/topology") - var topos []APITopologyDesc - if err := json.Unmarshal(body, &topos); err != nil { - t.Fatalf("JSON parse error: %s", err) - } - equals(t, 3, len(topos)) - for _, topo := range topos { - is200(t, ts, topo.URL) - if topo.GroupedURL != "" { - is200(t, ts, topo.GroupedURL) - } - if have := topo.Stats.EdgeCount; have <= 0 { - t.Errorf("EdgeCount isn't positive: %d", have) - } - if have := topo.Stats.NodeCount; have <= 0 { - t.Errorf("NodeCount isn't positive: %d", have) - } - if have := topo.Stats.NonpseudoNodeCount; have <= 0 { - t.Errorf("NonpseudoNodeCount isn't positive: %d", have) - } - } -} diff --git a/app/api_topology.go b/app/api_topology.go deleted file mode 100644 index 3fb1d1a38a..0000000000 --- a/app/api_topology.go +++ /dev/null @@ -1,147 +0,0 @@ -package main - -import ( - "net/http" - "time" - - "github.com/gorilla/mux" - "github.com/gorilla/websocket" - - "github.com/weaveworks/scope/report" -) - -const ( - websocketLoop = 1 * time.Second - websocketTimeout = 10 * time.Second -) - -// APITopology is returned by the /api/topology/{name} handler. -type APITopology struct { - Nodes map[string]report.RenderableNode `json:"nodes"` -} - -// APINode is returned by the /api/topology/{name}/{id} handler. -type APINode struct { - Node report.DetailedNode `json:"node"` -} - -// APIEdge is returned by the /api/topology/*/*/* handlers. -type APIEdge struct { - Metadata report.AggregateMetadata `json:"metadata"` -} - -// topologySelecter selects a single topology from a report. -type topologySelecter func(r report.Report) report.Topology - -func selectProcess(r report.Report) report.Topology { - return r.Process -} - -func selectNetwork(r report.Report) report.Topology { - return r.Network -} - -// Full topology. -func handleTopology(rep Reporter, t topologyView, w http.ResponseWriter, r *http.Request) { - respondWith(w, http.StatusOK, APITopology{ - Nodes: t.selector(rep.Report()).RenderBy(t.mapper, t.pseudo), - }) -} - -// Websocket for the full topology. This route overlaps with the next. -func handleWs(rep Reporter, t topologyView, w http.ResponseWriter, r *http.Request) { - if err := r.ParseForm(); err != nil { - respondWith(w, http.StatusInternalServerError, err.Error()) - return - } - loop := websocketLoop - if t := r.Form.Get("t"); t != "" { - var err error - if loop, err = time.ParseDuration(t); err != nil { - respondWith(w, http.StatusBadRequest, t) - return - } - } - handleWebsocket(w, r, rep, t, loop) -} - -// Individual nodes. -func handleNode(rep Reporter, t topologyView, w http.ResponseWriter, r *http.Request) { - var ( - vars = mux.Vars(r) - nodeID = vars["id"] - rpt = rep.Report() - node, ok = t.selector(rpt).RenderBy(t.mapper, t.pseudo)[nodeID] - ) - if !ok { - http.NotFound(w, r) - return - } - originHostFunc := func(id string) (OriginHost, bool) { return getOriginHost(rpt.HostMetadatas, id) } - originNodeFunc := func(id string) (OriginNode, bool) { return getOriginNode(t.selector(rpt), id) } - respondWith(w, http.StatusOK, APINode{Node: makeDetailed(node, originHostFunc, originNodeFunc)}) -} - -// Individual edges. -func handleEdge(rep Reporter, t topologyView, w http.ResponseWriter, r *http.Request) { - var ( - vars = mux.Vars(r) - localID = vars["local"] - remoteID = vars["remote"] - rpt = rep.Report() - metadata = t.selector(rpt).EdgeMetadata(t.mapper, localID, remoteID).Transform() - ) - respondWith(w, http.StatusOK, APIEdge{Metadata: metadata}) -} - -var upgrader = websocket.Upgrader{ - CheckOrigin: func(r *http.Request) bool { return true }, -} - -func handleWebsocket( - w http.ResponseWriter, - r *http.Request, - rep Reporter, - t topologyView, - loop time.Duration, -) { - conn, err := upgrader.Upgrade(w, r, nil) - if err != nil { - // log.Println("Upgrade:", err) - return - } - defer conn.Close() - - quit := make(chan struct{}) - go func(c *websocket.Conn) { - for { // just discard everything the browser sends - if _, _, err := c.NextReader(); err != nil { - close(quit) - break - } - } - }(conn) - - var ( - previousTopo map[string]report.RenderableNode - tick = time.Tick(loop) - ) - for { - newTopo := t.selector(rep.Report()).RenderBy(t.mapper, t.pseudo) - diff := report.TopoDiff(previousTopo, newTopo) - previousTopo = newTopo - - if err := conn.SetWriteDeadline(time.Now().Add(websocketTimeout)); err != nil { - return - } - if err := conn.WriteJSON(diff); err != nil { - return - } - - select { - case <-quit: - return - case <-tick: - } - } -} diff --git a/app/api_topology_test.go b/app/api_topology_test.go deleted file mode 100644 index 152d722bfd..0000000000 --- a/app/api_topology_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package main - -import ( - "encoding/json" - "net/http/httptest" - "reflect" - "testing" - - "github.com/gorilla/websocket" - - "github.com/weaveworks/scope/report" -) - -func TestAPITopologyApplications(t *testing.T) { - ts := httptest.NewServer(Router(StaticReport{})) - defer ts.Close() - is404(t, ts, "/api/topology/applications/foobar") - { - body := getRawJSON(t, ts, "/api/topology/applications") - var topo APITopology - if err := json.Unmarshal(body, &topo); err != nil { - t.Fatal(err) - } - equals(t, 4, len(topo.Nodes)) - node, ok := topo.Nodes["pid:node-a.local:23128"] - if !ok { - t.Errorf("missing curl node") - } - equals(t, 1, len(node.Adjacency)) - equals(t, report.NewIDList("pid:node-b.local:215"), node.Adjacency) - equals(t, report.NewIDList("hostA"), node.OriginHosts) - equals(t, "curl", node.LabelMajor) - equals(t, "node-a.local (23128)", node.LabelMinor) - equals(t, "23128", node.Rank) - equals(t, false, node.Pseudo) - } - { - body := getRawJSON(t, ts, "/api/topology/applications/pid:node-a.local:23128") - var node APINode - if err := json.Unmarshal(body, &node); err != nil { - t.Fatal(err) - } - equals(t, "pid:node-a.local:23128", node.Node.ID) - equals(t, "curl", node.Node.LabelMajor) - equals(t, "node-a.local (23128)", node.Node.LabelMinor) - equals(t, false, node.Node.Pseudo) - // Let's not unit-test the specific content of the detail tables - } - { - body := getRawJSON(t, ts, "/api/topology/applications/pid:node-a.local:23128/pid:node-b.local:215") - var edge APIEdge - if err := json.Unmarshal(body, &edge); err != nil { - t.Fatalf("JSON parse error: %s", err) - } - want := report.AggregateMetadata{ - "egress_bytes": 24, - "ingress_bytes": 0, - "max_conn_count_tcp": 401, - } - if !reflect.DeepEqual(want, edge.Metadata) { - t.Errorf("Edge metadata error. Want %v, have %v", want, edge) - } - } -} - -func TestAPITopologyHosts(t *testing.T) { - ts := httptest.NewServer(Router(StaticReport{})) - defer ts.Close() - is404(t, ts, "/api/topology/hosts/foobar") - { - body := getRawJSON(t, ts, "/api/topology/hosts") - var topo APITopology - if err := json.Unmarshal(body, &topo); err != nil { - t.Fatal(err) - } - equals(t, 3, len(topo.Nodes)) - node, ok := topo.Nodes["host:host-b"] - if !ok { - t.Errorf("missing host:host-b node") - } - equals(t, report.NewIDList("host:host-a"), node.Adjacency) - equals(t, report.NewIDList("hostB"), node.OriginHosts) - equals(t, "host-b", node.LabelMajor) - equals(t, "", node.LabelMinor) - equals(t, "host-b", node.Rank) - equals(t, false, node.Pseudo) - } - { - body := getRawJSON(t, ts, "/api/topology/hosts/host:host-b") - var node APINode - if err := json.Unmarshal(body, &node); err != nil { - t.Fatal(err) - } - equals(t, "host:host-b", node.Node.ID) - equals(t, "host-b", node.Node.LabelMajor) - equals(t, "", node.Node.LabelMinor) - equals(t, false, node.Node.Pseudo) - // Let's not unit-test the specific content of the detail tables - } - { - body := getRawJSON(t, ts, "/api/topology/hosts/host:host-b/host:host-a") - var edge APIEdge - if err := json.Unmarshal(body, &edge); err != nil { - t.Fatalf("JSON parse error: %s", err) - } - want := report.AggregateMetadata{ - "egress_bytes": 0, - "ingress_bytes": 12, - "max_conn_count_tcp": 16, - } - if !reflect.DeepEqual(want, edge.Metadata) { - t.Errorf("Edge metadata error. Want %v, have %v", want, edge) - } - } -} - -// Basic websocket test -func TestAPITopologyWebsocket(t *testing.T) { - ts := httptest.NewServer(Router(StaticReport{})) - defer ts.Close() - url := "/api/topology/applications/ws" - - // Not a websocket request - res, _ := checkGet(t, ts, url) - if have := res.StatusCode; have != 400 { - t.Fatalf("Expected status %d, got %d.", 400, have) - } - - // Proper websocket request - ts.URL = "ws" + ts.URL[len("http"):] - dialer := &websocket.Dialer{} - ws, res, err := dialer.Dial(ts.URL+url, nil) - ok(t, err) - defer ws.Close() - - if want, have := 101, res.StatusCode; want != have { - t.Fatalf("want %d, have %d", want, have) - } - - _, p, err := ws.ReadMessage() - ok(t, err) - var d report.Diff - if err := json.Unmarshal(p, &d); err != nil { - t.Fatalf("JSON parse error: %s", err) - } - equals(t, 4, len(d.Add)) - equals(t, 0, len(d.Update)) - equals(t, 0, len(d.Remove)) -} diff --git a/app/detail_pane.go b/app/detail_pane.go deleted file mode 100644 index 2f607f2f27..0000000000 --- a/app/detail_pane.go +++ /dev/null @@ -1,97 +0,0 @@ -package main - -import ( - "fmt" - "reflect" - "strconv" - - "github.com/weaveworks/scope/report" -) - -func makeDetailed( - n report.RenderableNode, - originHostLookup func(string) (OriginHost, bool), - originNodeLookup func(string) (OriginNode, bool), -) report.DetailedNode { - tables := []report.Table{{ - Title: "Connections", - Numeric: true, - Rows: []report.Row{ - // TODO omit these rows if there's no data? - {"TCP connections", strconv.FormatInt(int64(n.Metadata[report.KeyMaxConnCountTCP]), 10), ""}, - {"Bytes ingress", strconv.FormatInt(int64(n.Metadata[report.KeyBytesIngress]), 10), ""}, - {"Bytes egress", strconv.FormatInt(int64(n.Metadata[report.KeyBytesEgress]), 10), ""}, - }, - }} - - // Note that a RenderableNode may be the result of merge operation(s), and - // so may have multiple origin hosts and nodes. - -outer: - for _, id := range n.OriginNodes { - // Origin node IDs in e.g. the process topology are actually network - // n-tuples. (The process topology is actually more like a network - // n-tuple topology.) So we can have multiple IDs mapping to the same - // process. There are several ways to dedupe that, but here we take - // the lazy way and do simple equivalence of the resulting table. - node, ok := originNodeLookup(id) - if !ok { - node = unknownOriginNode(id) - } - for _, table := range tables { - if reflect.DeepEqual(table, node.Table) { - continue outer - } - } - tables = append(tables, node.Table) - } - - for _, id := range n.OriginHosts { - host, ok := originHostLookup(id) - if !ok { - host = unknownOriginHost(id) - } - tables = append(tables, report.Table{ - Title: "Origin Host", - Numeric: false, - Rows: []report.Row{ - {"Hostname", host.Hostname, ""}, - {"Load", fmt.Sprintf("%.2f %.2f %.2f", host.LoadOne, host.LoadFive, host.LoadFifteen), ""}, - {"OS", host.OS, ""}, - //{"Addresses", strings.Join(host.Addresses, ", "), ""}, - {"ID", id, ""}, - }, - }) - } - - return report.DetailedNode{ - ID: n.ID, - LabelMajor: n.LabelMajor, - LabelMinor: n.LabelMinor, - Pseudo: n.Pseudo, - Tables: tables, - } -} - -func unknownOriginHost(id string) OriginHost { - return OriginHost{ - Hostname: fmt.Sprintf("[%s]", id), - OS: "unknown", - Addresses: []string{}, - LoadOne: 0.0, - LoadFive: 0.0, - LoadFifteen: 0.0, - } -} - -func unknownOriginNode(id string) OriginNode { - return OriginNode{ - Table: report.Table{ - Title: "Origin Node", - Numeric: false, - Rows: []report.Row{ - {"ID", id, ""}, - }, - }, - } -} diff --git a/app/main.go b/app/main.go index e3c4ca3ab9..5852f06b9a 100644 --- a/app/main.go +++ b/app/main.go @@ -4,7 +4,6 @@ import ( "flag" "fmt" "log" - "log/syslog" "net/http" _ "net/http/pprof" "os" @@ -16,13 +15,11 @@ import ( "github.com/weaveworks/scope/xfer" ) -// Set during buildtime. -var version = "unknown" +var version = "dev" // set at build time func main() { var ( defaultProbes = []string{fmt.Sprintf("localhost:%d", xfer.ProbePort), fmt.Sprintf("scope.weave.local:%d", xfer.ProbePort)} - logfile = flag.String("log", "stderr", "stderr, syslog, or filename") batch = flag.Duration("batch", 1*time.Second, "batch interval") window = flag.Duration("window", 15*time.Second, "window") listen = flag.String("http.address", ":"+strconv.Itoa(xfer.AppPort), "webserver listen address") @@ -36,56 +33,32 @@ func main() { return } - switch *logfile { - case "stderr": - break // by default + log.Printf("app version %s", version) - case "syslog": - w, err := syslog.New(syslog.LOG_INFO, "scope-app") - if err != nil { - log.Print(err) - return - } - defer w.Close() - log.SetFlags(0) - log.SetOutput(w) - - default: // file - f, err := os.OpenFile(*logfile, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644) - if err != nil { - log.Print(err) - return - } - defer f.Close() - log.SetOutput(f) - } - - log.Printf("app starting, version %s", version) - - // Collector deals with the probes, and generates merged reports. xfer.MaxBackoff = 10 * time.Second c := xfer.NewCollector(*batch) defer c.Stop() - r := NewResolver(probes, c.AddAddress) + r := newStaticResolver(probes, c.Add) defer r.Stop() - lifo := NewReportLIFO(c, *window) - defer lifo.Stop() + reporter := newLIFOReporter(c.Reports(), *window) + defer reporter.Stop() - http.Handle("/", Router(lifo)) - irq := interrupt() + errc := make(chan error) go func() { + http.Handle("/", newRouter(reporter)) log.Printf("listening on %s", *listen) - log.Print(http.ListenAndServe(*listen, nil)) - irq <- syscall.SIGINT + errc <- http.ListenAndServe(*listen, nil) + }() + go func() { + errc <- interrupt() }() - <-irq - log.Printf("shutting down") + log.Print(<-errc) } -func interrupt() chan os.Signal { +func interrupt() error { c := make(chan os.Signal) signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) - return c + return fmt.Errorf("%s", <-c) } diff --git a/app/mock_reporter_test.go b/app/mock_reporter_test.go deleted file mode 100644 index c3740d0e5a..0000000000 --- a/app/mock_reporter_test.go +++ /dev/null @@ -1,135 +0,0 @@ -package main - -import ( - "net" - - "github.com/weaveworks/scope/report" -) - -// StaticReport is used as know test data in api tests. -type StaticReport struct{} - -func (s StaticReport) Report() report.Report { - _, localNet, err := net.ParseCIDR("192.168.1.1/24") - if err != nil { - panic(err.Error()) - } - - var testReport = report.Report{ - Process: report.Topology{ - Adjacency: report.Adjacency{ - "hostA|;192.168.1.1;12345": []string{";192.168.1.2;80"}, - "hostA|;192.168.1.1;12346": []string{";192.168.1.2;80"}, - "hostA|;192.168.1.1;8888": []string{";1.2.3.4;22"}, - "hostB|;192.168.1.2;80": []string{";192.168.1.1;12345"}, - }, - EdgeMetadatas: report.EdgeMetadatas{ - ";192.168.1.1;12345|;192.168.1.2;80": report.EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - WithConnCountTCP: true, - MaxConnCountTCP: 200, - }, - ";192.168.1.1;12346|;192.168.1.2;80": report.EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - WithConnCountTCP: true, - MaxConnCountTCP: 201, - }, - ";192.168.1.1;8888|;1.2.3.4;80": report.EdgeMetadata{ - WithBytes: true, - BytesEgress: 200, - BytesIngress: 0, - WithConnCountTCP: true, - MaxConnCountTCP: 202, - }, - ";192.168.1.2;80|;192.168.1.1;12345": report.EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - WithConnCountTCP: true, - MaxConnCountTCP: 203, - }, - }, - NodeMetadatas: report.NodeMetadatas{ - ";192.168.1.1;12345": report.NodeMetadata{ - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - ";192.168.1.1;12346": report.NodeMetadata{ // <-- same as :12345 - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - ";192.168.1.1;8888": report.NodeMetadata{ - "pid": "55100", - "name": "ssh", - "domain": "node-a.local", - }, - ";192.168.1.2;80": report.NodeMetadata{ - "pid": "215", - "name": "apache", - "domain": "node-b.local", - }, - }, - }, - - Network: report.Topology{ - Adjacency: report.Adjacency{ - "hostA|;192.168.1.1": []string{";192.168.1.2", ";1.2.3.4"}, - "hostB|;192.168.1.2": []string{";192.168.1.1"}, - }, - EdgeMetadatas: report.EdgeMetadatas{ - ";192.168.1.1|;192.168.1.2": report.EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - WithConnCountTCP: true, - MaxConnCountTCP: 14, - }, - ";192.168.1.1|;1.2.3.4": report.EdgeMetadata{ - WithBytes: true, - BytesEgress: 200, - BytesIngress: 0, - WithConnCountTCP: true, - MaxConnCountTCP: 15, - }, - ";192.168.1.2|;192.168.1.1": report.EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - WithConnCountTCP: true, - MaxConnCountTCP: 16, - }, - }, - NodeMetadatas: report.NodeMetadatas{ - ";192.168.1.1": report.NodeMetadata{ - "name": "host-a", - }, - ";192.168.1.2": report.NodeMetadata{ - "name": "host-b", - }, - }, - }, - - HostMetadatas: report.HostMetadatas{ - "hostA": report.HostMetadata{ - Hostname: "node-a.local", - LocalNets: []*net.IPNet{localNet}, - OS: "Linux", - LoadOne: 3.1415, - LoadFive: 2.7182, - LoadFifteen: 1.6180, - }, - "hostB": report.HostMetadata{ - Hostname: "node-b.local", - LocalNets: []*net.IPNet{localNet}, - OS: "Linux", - }, - }, - } - return testReport.SquashRemote() -} diff --git a/app/origin_host.go b/app/origin_host.go deleted file mode 100644 index a7b722ee60..0000000000 --- a/app/origin_host.go +++ /dev/null @@ -1,58 +0,0 @@ -package main - -import ( - "net/http" - - "github.com/gorilla/mux" - - "github.com/weaveworks/scope/report" -) - -// OriginHost represents a host that runs a probe, i.e. the origin host of -// some data in the system. The struct is returned by the /api/origin/{id} -// handler. -type OriginHost struct { - Hostname string `json:"hostname"` - OS string `json:"os"` - Addresses []string `json:"addresses"` - LoadOne float64 `json:"load_one"` - LoadFive float64 `json:"load_five"` - LoadFifteen float64 `json:"load_fifteen"` -} - -func getOriginHost(mds report.HostMetadatas, nodeID string) (OriginHost, bool) { - host, ok := mds[nodeID] - if !ok { - return OriginHost{}, false - } - - var addrs []string - for _, l := range host.LocalNets { - addrs = append(addrs, l.String()) - } - - return OriginHost{ - Hostname: host.Hostname, - OS: host.OS, - Addresses: addrs, - LoadOne: host.LoadOne, - LoadFive: host.LoadFive, - LoadFifteen: host.LoadFifteen, - }, true -} - -// makeOriginHostHandler makes the /api/origin/* handler. -func makeOriginHostHandler(rep Reporter) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - var ( - vars = mux.Vars(r) - nodeID = vars["id"] - ) - origin, ok := getOriginHost(rep.Report().HostMetadatas, nodeID) - if !ok { - http.NotFound(w, r) - return - } - respondWith(w, http.StatusOK, origin) - } -} diff --git a/app/origin_host_test.go b/app/origin_host_test.go deleted file mode 100644 index 6ff9830413..0000000000 --- a/app/origin_host_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package main - -import ( - "encoding/json" - "net/http/httptest" - "testing" -) - -func TestAPIOriginHost(t *testing.T) { - ts := httptest.NewServer(Router(StaticReport{})) - defer ts.Close() - - is404(t, ts, "/api/origin/foobar") - is404(t, ts, "/api/origin/host/foobar") - - { - // Origin - body := getRawJSON(t, ts, "/api/origin/host/hostA") - var o OriginHost - if err := json.Unmarshal(body, &o); err != nil { - t.Fatalf("JSON parse error: %s", err) - } - if want, have := "Linux", o.OS; want != have { - t.Errorf("Origin error. Want %v, have %v", want, have) - } - if want, have := 3.1415, o.LoadOne; want != have { - t.Errorf("Origin error. Want %v, have %v", want, have) - } - if want, have := 2.7182, o.LoadFive; want != have { - t.Errorf("Origin error. Want %v, have %v", want, have) - } - if want, have := 1.6180, o.LoadFifteen; want != have { - t.Errorf("Origin error. Want %v, have %v", want, have) - } - } -} diff --git a/app/origin_node.go b/app/origin_node.go deleted file mode 100644 index 0cd96fcf79..0000000000 --- a/app/origin_node.go +++ /dev/null @@ -1,67 +0,0 @@ -package main - -import "github.com/weaveworks/scope/report" - -// OriginNode is a node in the originating report topology. It's a process ID -// or network host. It's used by the /api/topology/{topology}/{nodeID} handler -// to generate detailed information. One node from a rendered topology may -// have multiple origin nodes. -type OriginNode struct { - Table report.Table -} - -func getOriginNode(t report.Topology, id string) (OriginNode, bool) { - node, ok := t.NodeMetadatas[id] - if !ok { - return OriginNode{}, false - } - - // The node represents different actual things depending on the topology. - // So we deduce what it is, based on the metadata. - if _, ok := node["pid"]; ok { - return originNodeForProcess(node), true - } - - // Assume network host. Could strengthen this guess by adding a - // special key in the probe spying procedure. - return originNodeForNetworkHost(node), true -} - -func originNodeForProcess(node report.NodeMetadata) OriginNode { - rows := []report.Row{ - {Key: "Host", ValueMajor: node["domain"], ValueMinor: ""}, - {Key: "PID", ValueMajor: node["pid"], ValueMinor: ""}, - {Key: "Process name", ValueMajor: node["name"], ValueMinor: ""}, - } - for _, tuple := range []struct{ key, human string }{ - {"docker_id", "Container ID"}, - {"docker_name", "Container name"}, - {"docker_image_id", "Container image ID"}, - {"docker_image_name", "Container image name"}, - {"cgroup", "cgroup"}, - } { - if val, ok := node[tuple.key]; ok { - rows = append(rows, report.Row{Key: tuple.human, ValueMajor: val, ValueMinor: ""}) - } - } - return OriginNode{ - Table: report.Table{ - Title: "Origin Process", - Numeric: false, - Rows: rows, - }, - } -} - -func originNodeForNetworkHost(node report.NodeMetadata) OriginNode { - rows := []report.Row{ - {"Hostname", node["name"], ""}, - } - return OriginNode{ - Table: report.Table{ - Title: "Origin Host", - Numeric: false, - Rows: rows, - }, - } -} diff --git a/app/report_lifo.go b/app/report_lifo.go deleted file mode 100644 index 9bdc9b7c3c..0000000000 --- a/app/report_lifo.go +++ /dev/null @@ -1,92 +0,0 @@ -package main - -import ( - "time" - - "github.com/weaveworks/scope/report" -) - -// Reporter is something which generates a single 'current' report over a -// stream of incoming reports. -type Reporter interface { - Report() report.Report -} - -type timedReport struct { - report.Report - Timestamp time.Time -} - -// ReportLIFO keeps a short-term history of reports. -type ReportLIFO struct { - reports []timedReport - requests chan chan report.Report - quit chan chan struct{} -} - -type reporter interface { - Reports() <-chan report.Report -} - -// NewReportLIFO collects reports up to a certain age. -func NewReportLIFO(r reporter, maxAge time.Duration) *ReportLIFO { - l := ReportLIFO{ - reports: []timedReport{}, - requests: make(chan chan report.Report), - quit: make(chan chan struct{}), - } - - go func() { - for { - select { - case report := <-r.Reports(): - // Incoming report from the collecter. - report = report.SquashRemote() // TODO?: make this a CLI argument. - tr := timedReport{ - Timestamp: time.Now(), - Report: report, - } - l.reports = append(l.reports, tr) - l.reports = cleanOld(l.reports, time.Now().Add(-maxAge)) - - case req := <-l.requests: - // Request for the current report. - report := report.NewReport() - for _, r := range l.reports { - report.Merge(r.Report) - } - req <- report - - case q := <-l.quit: - close(q) - return - } - } - }() - return &l -} - -// Stop shuts down the monitor. -func (r *ReportLIFO) Stop() { - q := make(chan struct{}) - r.quit <- q - <-q -} - -// Report returns the latest report. -func (r *ReportLIFO) Report() report.Report { - req := make(chan report.Report) - r.requests <- req - return <-req -} - -func cleanOld(reports []timedReport, threshold time.Time) []timedReport { - res := make([]timedReport, 0, len(reports)) - for _, tr := range reports { - if tr.Timestamp.Before(threshold) { - continue - } - res = append(res, tr) - } - return res -} diff --git a/app/reporter.go b/app/reporter.go new file mode 100644 index 0000000000..e761144def --- /dev/null +++ b/app/reporter.go @@ -0,0 +1,78 @@ +package main + +import ( + "time" + + "github.com/weaveworks/scope/report" +) + +// Reporter represents something that can yield reports on-demand. +type Reporter interface { + Report() report.Report + Stop() +} + +var ( + now = time.Now +) + +type lifoReporter struct { + requests chan chan report.Report + quit chan struct{} +} + +func newLIFOReporter(incoming <-chan report.Report, maxAge time.Duration) Reporter { + r := &lifoReporter{ + requests: make(chan chan report.Report), + quit: make(chan struct{}), + } + go r.loop(incoming, maxAge) + return r +} + +func (r *lifoReporter) Report() report.Report { + c := make(chan report.Report) + r.requests <- c + return <-c +} + +func (r *lifoReporter) Stop() { + close(r.quit) +} + +func (r *lifoReporter) loop(incoming <-chan report.Report, maxAge time.Duration) { + reports := []timedReport{} + for { + select { + case report := <-incoming: + reports = append(reports, timedReport{now(), report.Squash()}) + reports = trim(reports, now().Add(-maxAge)) + + case c := <-r.requests: + r := report.MakeReport() + for _, tr := range reports { + r = r.Merge(tr.Report) + } + c <- r + + case <-r.quit: + return + } + } +} + +type timedReport struct { + time.Time + report.Report +} + +func trim(in []timedReport, oldest time.Time) []timedReport { + out := make([]timedReport, 0, len(in)) + for _, tr := range in { + if tr.Time.Before(oldest) { + continue + } + out = append(out, tr) + } + return out +} diff --git a/app/reporter_test.go b/app/reporter_test.go new file mode 100644 index 0000000000..8ac0e45029 --- /dev/null +++ b/app/reporter_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "reflect" + "testing" + "time" + + "github.com/weaveworks/scope/report" +) + +func TestLIFOReporter(t *testing.T) { + oldNow := now + defer func() { now = oldNow }() + ts := time.Now() + now = func() time.Time { return ts } + + var ( + incoming = make(chan report.Report) + maxAge = time.Second + ) + r := newLIFOReporter(incoming, maxAge) + defer r.Stop() + + // By default, we want an empty report. + want := report.MakeReport() + if have := r.Report(); !reflect.DeepEqual(want, have) { + t.Errorf("want\n\t%+v, have\n\t%+v", want, have) + } + + // Seed the initial report. + incoming <- report.Report{Endpoint: report.Topology{NodeMetadatas: report.NodeMetadatas{"a": report.NodeMetadata{"foo": "bar"}}}} + want.Endpoint.NodeMetadatas["a"] = report.NodeMetadata{"foo": "bar"} + if have := r.Report(); !reflect.DeepEqual(want, have) { + t.Errorf("want\n\t%+v, have\n\t%+v", want, have) + } + + // Add some time, and merge more reports. + ts = ts.Add(maxAge / 2) + incoming <- report.Report{Endpoint: report.Topology{NodeMetadatas: report.NodeMetadatas{"a": report.NodeMetadata{"1": "1"}}}} + incoming <- report.Report{Endpoint: report.Topology{NodeMetadatas: report.NodeMetadatas{"a": report.NodeMetadata{"1": "1"}}}} // dupe! + want.Endpoint.NodeMetadatas["a"] = report.NodeMetadata{"foo": "bar", "1": "1"} + if have := r.Report(); !reflect.DeepEqual(want, have) { + t.Errorf("want\n\t%+v, have\n\t%+v", want, have) + } + + // Add enough time that the initial report should go away. + ts = ts.Add(9 * (maxAge / 10)) + incoming <- report.MakeReport() + want.Endpoint.NodeMetadatas["a"] = report.NodeMetadata{"1": "1"} // we should lose the initial data + if have := r.Report(); !reflect.DeepEqual(want, have) { + t.Errorf("want\n\t%+v, have\n\t%+v", want, have) + } +} diff --git a/app/resolver.go b/app/resolver.go index 48953fe882..9cb4654356 100644 --- a/app/resolver.go +++ b/app/resolver.go @@ -15,9 +15,11 @@ var ( lookupIP = net.LookupIP ) -// Resolver periodically tries to resolve the IP addresses for a given -// set of hostnames. -type Resolver struct { +type resolver interface { + Stop() +} + +type staticResolver struct { quit chan struct{} add func(string) peers []peer @@ -33,8 +35,8 @@ type peer struct { // resolved IPs. It explictiy supports hostnames which // resolve to multiple IPs; it will repeatedly call // add with the same IP, expecting the target to dedupe. -func NewResolver(peers []string, add func(string)) Resolver { - r := Resolver{ +func newStaticResolver(peers []string, add func(string)) resolver { + r := staticResolver{ quit: make(chan struct{}), add: add, peers: prepareNames(peers), @@ -67,20 +69,21 @@ func prepareNames(strs []string) []peer { return results } -func (r Resolver) loop() { +func (r staticResolver) loop() { r.resolveHosts() t := tick(time.Minute) for { select { case <-t: r.resolveHosts() + case <-r.quit: return } } } -func (r Resolver) resolveHosts() { +func (r staticResolver) resolveHosts() { for _, peer := range r.peers { var addrs []net.IP if addr := net.ParseIP(peer.hostname); addr != nil { @@ -103,7 +106,6 @@ func (r Resolver) resolveHosts() { } } -// Stop this Resolver. -func (r Resolver) Stop() { +func (r staticResolver) Stop() { close(r.quit) } diff --git a/app/resolver_test.go b/app/resolver_test.go index a40d248ec9..6e47fe6b2f 100644 --- a/app/resolver_test.go +++ b/app/resolver_test.go @@ -32,17 +32,18 @@ func TestResolver(t *testing.T) { ip2 := "192.168.0.10" adds := make(chan string) add := func(s string) { adds <- s } - r := NewResolver([]string{"symbolic.name" + port, "namewithnoport", ip1 + port, ip2}, add) + + r := newStaticResolver([]string{"symbolic.name" + port, "namewithnoport", ip1 + port, ip2}, add) assertAdd := func(want string) { + _, _, line, _ := runtime.Caller(1) select { case have := <-adds: if want != have { - _, _, line, _ := runtime.Caller(1) t.Errorf("line %d: want %q, have %q", line, want, have) } case <-time.After(time.Millisecond): - t.Fatal("didn't get add in time") + t.Fatalf("line %d: didn't get add in time", line) } } diff --git a/app/router.go b/app/router.go index 931f7c2bc9..14e6a6b75b 100644 --- a/app/router.go +++ b/app/router.go @@ -1,62 +1,209 @@ package main import ( + "encoding/json" + "fmt" + "log" "net/http" + "strings" + "time" "github.com/gorilla/mux" "github.com/weaveworks/scope/report" ) -// Router gives of the HTTP dispatcher. It will always use the embedded HTML -// resources. -func Router(c Reporter) *mux.Router { +var topologyRegistry = map[string]topologyDefinition{ + "applications": {"Applications", report.SelectEndpoint, report.ProcessPID, report.BasicPseudoNode, "applications-grouped"}, + "applications-grouped": {"Applications", report.SelectEndpoint, report.ProcessName, report.GroupedPseudoNode, ""}, + "containers": {"Containers", report.SelectEndpoint, report.ProcessContainer, report.NoPseudoNode, "containers-grouped"}, + "containers-grouped": {"Containers", report.SelectEndpoint, report.ProcessContainerImage, report.NoPseudoNode, ""}, + "hosts": {"Hosts", report.SelectAddress, report.AddressHostname, report.BasicPseudoNode, ""}, +} + +func newRouter(r Reporter) http.Handler { router := mux.NewRouter() - get := router.Methods("GET").Subrouter() - get.HandleFunc("/api", apiHandler) - get.HandleFunc("/api/topology", makeTopologyList(c)) - get.HandleFunc("/api/topology/{topology}", captureTopology(c, handleTopology)) - get.HandleFunc("/api/topology/{topology}/ws", captureTopology(c, handleWs)) - get.HandleFunc("/api/topology/{topology}/{id}", captureTopology(c, handleNode)) - get.HandleFunc("/api/topology/{topology}/{local}/{remote}", captureTopology(c, handleEdge)) - get.HandleFunc("/api/origin/host/{id}", makeOriginHostHandler(c)) - get.HandleFunc("/api/report", makeRawReportHandler(c)) - get.PathPrefix("/").Handler(http.FileServer(FS(false))) // everything else is static + getRouter := router.Methods("GET").Subrouter() + getRouter.HandleFunc("/api", handleAPI) + getRouter.HandleFunc("/api/topology", handleListTopologies(r)) + getRouter.HandleFunc("/api/topology/{topology}", captureTopology(r, handleTopology)) + getRouter.HandleFunc("/api/topology/{topology}/ws", captureTopology(r, handleTopologyWebsocket)) + getRouter.HandleFunc("/api/topology/{topology}/{id}", captureTopology(r, handleTopologyNode)) + getRouter.HandleFunc("/api/topology/{topology}/{src}/{dst}", captureTopology(r, handleTopologyEdge)) + getRouter.HandleFunc("/api/report", handleReport(r)) + getRouter.PathPrefix("/").Handler(http.FileServer(FS(false))) // static return router } -func captureTopology(rep Reporter, f func(Reporter, topologyView, http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) { +func handleAPI(w http.ResponseWriter, r *http.Request) { + respondWith(w, http.StatusOK, APIDetails{Version: version}) +} + +func handleListTopologies(reporter Reporter) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + rpt := reporter.Report() + var descriptors []APITopologyDescriptor + for name, definition := range topologyRegistry { + if strings.HasSuffix(name, "-grouped") { + continue + } + var ( + path = "/api/topology/" + name + groupedPath = "" + ) + if definition.grouped != "" { + groupedPath = "/api/topology/" + definition.grouped + } + descriptors = append(descriptors, APITopologyDescriptor{ + Name: definition.human, + Path: path, + GroupedPath: groupedPath, + Stats: makeTopologyStats(report.Render(rpt, definition.selector, definition.mapper, definition.pseudo)), + }) + } + respondWith(w, http.StatusOK, descriptors) + } +} + +func captureTopology(reporter Reporter, f func(Reporter, topologyDefinition, http.ResponseWriter, *http.Request)) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { - topology, ok := topologyRegistry[mux.Vars(r)["topology"]] + topology, ok := mux.Vars(r)["topology"] if !ok { - http.NotFound(w, r) + respondWith(w, http.StatusBadRequest, errorResponse("topology not provided")) return } - f(rep, topology, w, r) + definition, ok := topologyRegistry[topology] + if !ok { + respondWith(w, http.StatusNotFound, errorResponse(fmt.Sprintf("%q not found", topology))) + return + } + f(reporter, definition, w, r) + } +} + +func handleTopology(reporter Reporter, definition topologyDefinition, w http.ResponseWriter, r *http.Request) { + respondWith(w, http.StatusOK, APITopology{ + Nodes: report.Render(reporter.Report(), definition.selector, definition.mapper, definition.pseudo), + }) +} + +func handleTopologyWebsocket(reporter Reporter, definition topologyDefinition, w http.ResponseWriter, r *http.Request) { + if err := r.ParseForm(); err != nil { + respondWith(w, http.StatusInternalServerError, err.Error()) + return + } + interval := websocketInterval + if t := r.Form.Get("t"); t != "" { + var err error + if interval, err = time.ParseDuration(t); err != nil { + respondWith(w, http.StatusBadRequest, errorResponse(err.Error())) + return + } + } + handleWebsocket(w, r, reporter, definition, interval) +} + +func handleTopologyNode(reporter Reporter, definition topologyDefinition, w http.ResponseWriter, r *http.Request) { + var ( + nodeID = mux.Vars(r)["id"] + rpt = reporter.Report() + node, ok = report.Render(rpt, definition.selector, definition.mapper, definition.pseudo)[nodeID] + ) + if !ok { + respondWith(w, http.StatusNotFound, errorResponse(fmt.Sprintf("%q not found", nodeID))) + return + } + respondWith(w, http.StatusOK, APINode{Node: report.MakeDetailedNode(rpt, node)}) +} + +func handleTopologyEdge(reporter Reporter, definition topologyDefinition, w http.ResponseWriter, r *http.Request) { + var ( + vars = mux.Vars(r) + srcMappedID = vars["src"] + dstMappedID = vars["dst"] + rpt = reporter.Report() + metadata = rpt.EdgeMetadata(definition.selector, definition.mapper, srcMappedID, dstMappedID).Export() + ) + respondWith(w, http.StatusOK, APIEdge{Metadata: metadata}) +} + +func handleReport(reporter Reporter) http.HandlerFunc { + return func(w http.ResponseWriter, _ *http.Request) { + respondWith(w, http.StatusOK, reporter.Report()) + } +} + +func makeTopologyStats(m map[string]report.RenderableNode) topologyStats { + var ( + nodes int + realNodes int + edges int + ) + for _, node := range m { + nodes++ + if !node.Pseudo { + realNodes++ + } + edges += len(node.Adjacency) + } + return topologyStats{ + NodeCount: nodes, + NonPseudoNodeCount: realNodes, + EdgeCount: edges, } } -// APIDetails are some generic details that can be fetched from /api +func respondWith(w http.ResponseWriter, code int, response interface{}) { + w.Header().Set("Content-Type", "application/json") + w.Header().Add("Cache-Control", "no-cache") + w.WriteHeader(code) + if err := json.NewEncoder(w).Encode(response); err != nil { + log.Print(err) + } +} + +func errorResponse(err string) interface{} { + return map[string]string{"err": err} +} + +type topologyDefinition struct { + human string + selector report.TopologySelector + mapper report.MapFunc + pseudo report.PseudoFunc + grouped string +} + +// APIDetails are some generic details that can be fetched from /api. type APIDetails struct { Version string `json:"version"` } -func apiHandler(w http.ResponseWriter, r *http.Request) { - respondWith(w, http.StatusOK, APIDetails{Version: version}) +// APITopologyDescriptor is returned by the /api/topology handler. +type APITopologyDescriptor struct { + Name string `json:"name"` + Path string `json:"url"` // TODO rename to path + GroupedPath string `json:"grouped_url,omitempty"` // TODO rename to grouped_path + Stats topologyStats `json:"stats"` +} + +// APITopology is returned by the /api/topology/{topology} handler. +type APITopology struct { + Nodes map[string]report.RenderableNode `json:"nodes"` +} + +// APINode is returned by the /api/topology/{topology}/{id} handler. +type APINode struct { + Node report.DetailedNode `json:"node"` } -type topologyView struct { - human string - selector topologySelecter - mapper report.MapFunc - pseudo report.PseudoFunc - groupedTopology string +// APIEdge is returned by the /api/topology/{topology}/{src}/{dst} handler. +type APIEdge struct { + Metadata report.AggregateMetadata `json:"metadata"` } -var topologyRegistry = map[string]topologyView{ - "applications": {"Applications", selectProcess, report.ProcessPID, report.GenericPseudoNode, "applications-grouped"}, - "applications-grouped": {"Applications", selectProcess, report.ProcessName, report.GenericGroupedPseudoNode, ""}, - "containers": {"Containers", selectProcess, report.ProcessContainer, report.InternetOnlyPseudoNode, "containers-grouped"}, - "containers-grouped": {"Containers", selectProcess, report.ProcessContainerImage, report.InternetOnlyPseudoNode, ""}, - "hosts": {"Hosts", selectNetwork, report.NetworkHostname, report.GenericPseudoNode, ""}, +type topologyStats struct { + NodeCount int `json:"node_count"` + NonPseudoNodeCount int `json:"non_pseudo_node_count"` + EdgeCount int `json:"edge_count"` } diff --git a/app/scope_test.go b/app/scope_test.go deleted file mode 100644 index 68d350ae18..0000000000 --- a/app/scope_test.go +++ /dev/null @@ -1,116 +0,0 @@ -package main - -import ( - "bytes" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "path/filepath" - "reflect" - "runtime" - "testing" -) - -// assert fails the test if the condition is false. -func assert(tb testing.TB, condition bool, msg string, v ...interface{}) { - if !condition { - _, file, line, _ := runtime.Caller(1) - tb.Fatalf("%s:%d: "+msg, append([]interface{}{filepath.Base(file), line}, v...)...) - } -} - -// ok errors the test if an err is not nil. -func ok(tb testing.TB, err error) { - if err != nil { - _, file, line, _ := runtime.Caller(1) - tb.Errorf("%s:%d: unexpected error: %v", filepath.Base(file), line, err) - } -} - -// equals errors the test if want is not equal to have. -func equals(tb testing.TB, want, have interface{}) { - if !reflect.DeepEqual(want, have) { - _, file, line, _ := runtime.Caller(1) - tb.Errorf("%s:%d: want %#v, have %#v", filepath.Base(file), line, want, have) - } -} - -// checkGet does a GET and returns the response and the body -func checkGet(t *testing.T, ts *httptest.Server, path string) (*http.Response, []byte) { - return checkRequest(t, ts, "GET", path, nil) -} - -// checkRequest does a 'method'-request (e.g. 'GET') and returns the response and the body -func checkRequest(t *testing.T, ts *httptest.Server, method, path string, body []byte) (*http.Response, []byte) { - fullPath := ts.URL + path - var bodyReader io.Reader - if len(body) > 0 { - bodyReader = bytes.NewReader(body) - } - req, err := http.NewRequest(method, fullPath, bodyReader) - if err != nil { - t.Fatalf("Error getting %s: %s", path, err) - } - - client := &http.Client{} - res, err := client.Do(req) - if err != nil { - t.Fatalf("Error getting %s: %s", path, err) - } - - body, err = ioutil.ReadAll(res.Body) - res.Body.Close() - if err != nil { - t.Fatalf("%s body read error: %s", path, err) - } - return res, body -} - -// getRawJSON GETs a file, checks it is JSON, and returns the non-parsed body -func getRawJSON(t *testing.T, ts *httptest.Server, path string) []byte { - res, body := checkGet(t, ts, path) - - if res.StatusCode != 200 { - t.Fatalf("Expected status %d, got %d. Path: %s", 200, res.StatusCode, path) - } - - foundCtype := res.Header.Get("content-type") - if foundCtype != "application/json" { - t.Errorf("Wrong Content-type for JSON: %s", foundCtype) - } - - if len(body) == 0 { - t.Errorf("No response body") - } - // fmt.Printf("Body: %s", body) - - return body -} - -// is200 GETs path and verifies the status code. Returns the body -func is200(t *testing.T, ts *httptest.Server, path string) []byte { - res, body := checkGet(t, ts, path) - if res.StatusCode != 200 { - t.Fatalf("Expected status %d, got %d. Path: %s", 200, res.StatusCode, path) - } - return body -} - -// is404 GETs path and verifies it returns a 404 status code. Returns the body -func is404(t *testing.T, ts *httptest.Server, path string) []byte { - res, body := checkGet(t, ts, path) - if res.StatusCode != 404 { - t.Fatalf("Expected status %d, got %d", 404, res.StatusCode) - } - return body -} - -// is400 GETs path and verifies it returns a 400 status code. Returns the body -func is400(t *testing.T, ts *httptest.Server, path string) []byte { - res, body := checkGet(t, ts, path) - if res.StatusCode != 400 { - t.Fatalf("Expected status %d, got %d", 400, res.StatusCode) - } - return body -} diff --git a/app/server_helpers.go b/app/server_helpers.go deleted file mode 100644 index 8aa1fe8285..0000000000 --- a/app/server_helpers.go +++ /dev/null @@ -1,16 +0,0 @@ -package main - -import ( - "encoding/json" - "log" - "net/http" -) - -func respondWith(w http.ResponseWriter, code int, response interface{}) { - w.Header().Set("Content-Type", "application/json") - w.Header().Add("Cache-Control", "no-cache") - w.WriteHeader(code) - if err := json.NewEncoder(w).Encode(response); err != nil { - log.Print(err) - } -} diff --git a/app/site_test.go b/app/site_test.go deleted file mode 100644 index 1bbc67012f..0000000000 --- a/app/site_test.go +++ /dev/null @@ -1,17 +0,0 @@ -// Basic site layout tests. -package main - -import ( - "net/http/httptest" - "testing" -) - -// Test site -func TestSite(t *testing.T) { - ts := httptest.NewServer(Router(StaticReport{})) - defer ts.Close() - - is200(t, ts, "/") - is200(t, ts, "/index.html") - is404(t, ts, "/index.html/foobar") -} diff --git a/app/websocket.go b/app/websocket.go new file mode 100644 index 0000000000..c19e6ff348 --- /dev/null +++ b/app/websocket.go @@ -0,0 +1,95 @@ +package main + +import ( + "net/http" + "reflect" + "time" + + "github.com/gorilla/websocket" + + "github.com/weaveworks/scope/report" +) + +const ( + websocketInterval = 1 * time.Second + websocketTimeout = 10 * time.Second +) + +var upgrader = websocket.Upgrader{ + CheckOrigin: func(r *http.Request) bool { return true }, +} + +func handleWebsocket(w http.ResponseWriter, r *http.Request, reporter Reporter, definition topologyDefinition, d time.Duration) { + conn, err := upgrader.Upgrade(w, r, nil) + if err != nil { + // log.Println("Upgrade:", err) + return + } + defer conn.Close() + + quit := make(chan struct{}) + go func(c *websocket.Conn) { + for { + // just discard everything the browser sends + if _, _, err := c.NextReader(); err != nil { + close(quit) + break + } + } + }(conn) + + var ( + prevTopology map[string]report.RenderableNode + tick = time.Tick(d) + ) + for { + currTopology := report.Render(reporter.Report(), definition.selector, definition.mapper, definition.pseudo) + diff := topologyDiff(prevTopology, currTopology) + prevTopology = currTopology + + if err := conn.SetWriteDeadline(time.Now().Add(websocketTimeout)); err != nil { + return + } + if err := conn.WriteJSON(diff); err != nil { + return + } + + select { + case <-tick: + case <-quit: + return + } + } +} + +// diff is returned by topologyDiff. It represents the changes between two +// renderable topologies. +type diff struct { + Add []report.RenderableNode `json:"add"` + Update []report.RenderableNode `json:"update"` + Remove []string `json:"remove"` +} + +// topologyDiff gives you the diff to get from A to B. +func topologyDiff(a, b map[string]report.RenderableNode) diff { + d := diff{} + notSeen := map[string]struct{}{} + for k := range a { + notSeen[k] = struct{}{} + } + + for k, node := range b { + if _, ok := a[k]; !ok { + d.Add = append(d.Add, node) + } else if !reflect.DeepEqual(node, a[k]) { + d.Update = append(d.Update, node) + } + delete(notSeen, k) + } + + for k := range notSeen { + d.Remove = append(d.Remove, k) + } + + return d +} diff --git a/app/websocket_test.go b/app/websocket_test.go new file mode 100644 index 0000000000..08c014ac94 --- /dev/null +++ b/app/websocket_test.go @@ -0,0 +1,49 @@ +package main + +import ( + "reflect" + "testing" + + "github.com/weaveworks/scope/report" +) + +func TestTopologyDiff(t *testing.T) { + for i, tuple := range []struct { + a, b map[string]report.RenderableNode + want diff + }{ + { + map[string]report.RenderableNode{}, + map[string]report.RenderableNode{}, + diff{}, + }, + { + map[string]report.RenderableNode{}, + map[string]report.RenderableNode{"a": {ID: "a"}}, + diff{Add: []report.RenderableNode{{ID: "a"}}}, + }, + { + map[string]report.RenderableNode{"a": {ID: "a"}}, + map[string]report.RenderableNode{}, + diff{Remove: []string{"a"}}, + }, + { + map[string]report.RenderableNode{"a": {ID: "a"}}, + map[string]report.RenderableNode{"a": {ID: "a", LabelMajor: "different"}}, + diff{Update: []report.RenderableNode{{ID: "a", LabelMajor: "different"}}}, + }, + { + map[string]report.RenderableNode{"c": {ID: "c"}, "b": {ID: "b"}}, + map[string]report.RenderableNode{"a": {ID: "a"}, "b": {ID: "b", LabelMajor: "different"}}, + diff{ + Add: []report.RenderableNode{{ID: "a"}}, + Update: []report.RenderableNode{{ID: "b", LabelMajor: "different"}}, + Remove: []string{"c"}, + }, + }, + } { + if want, have := tuple.want, topologyDiff(tuple.a, tuple.b); !reflect.DeepEqual(want, have) { + t.Errorf("%d: want\n\t%#+v, have\n\t%#+v", i, want, have) + } + } +} diff --git a/experimental/demoprobe/Makefile b/experimental/_demoprobe/Makefile similarity index 100% rename from experimental/demoprobe/Makefile rename to experimental/_demoprobe/Makefile diff --git a/experimental/demoprobe/generate.go b/experimental/_demoprobe/generate.go similarity index 68% rename from experimental/demoprobe/generate.go rename to experimental/_demoprobe/generate.go index 1a44c9e109..99b84e83c8 100644 --- a/experimental/demoprobe/generate.go +++ b/experimental/_demoprobe/generate.go @@ -66,56 +66,56 @@ func DemoReport(nodeCount int) report.Report { nodeDstAddressID = "hostX" + report.IDDelim + dstAddressID ) - // Process topology - if _, ok := r.Process.NodeMetadatas[srcPortID]; !ok { - r.Process.NodeMetadatas[srcPortID] = report.NodeMetadata{ + // Endpoint topology + if _, ok := r.Endpoint.NodeMetadatas[srcPortID]; !ok { + r.Endpoint.NodeMetadatas[srcPortID] = report.NodeMetadata{ "pid": "4000", "name": c.srcProc, "domain": "node-" + src, } } - r.Process.Adjacency[srcID] = r.Process.Adjacency[srcID].Add(dstPortID) - if _, ok := r.Process.NodeMetadatas[dstPortID]; !ok { - r.Process.NodeMetadatas[dstPortID] = report.NodeMetadata{ + r.Endpoint.Adjacency[srcID] = r.Endpoint.Adjacency[srcID].Add(dstPortID) + if _, ok := r.Endpoint.NodeMetadatas[dstPortID]; !ok { + r.Endpoint.NodeMetadatas[dstPortID] = report.NodeMetadata{ "pid": "4000", "name": c.dstProc, "domain": "node-" + dst, } } - r.Process.Adjacency[dstID] = r.Process.Adjacency[dstID].Add(srcPortID) + r.Endpoint.Adjacency[dstID] = r.Endpoint.Adjacency[dstID].Add(srcPortID) var ( edgeKeyEgress = srcPortID + report.IDDelim + dstPortID edgeKeyIngress = dstPortID + report.IDDelim + srcPortID ) - r.Process.EdgeMetadatas[edgeKeyEgress] = report.EdgeMetadata{ + r.Endpoint.EdgeMetadatas[edgeKeyEgress] = report.EdgeMetadata{ WithConnCountTCP: true, MaxConnCountTCP: uint(rand.Intn(100) + 10), } - r.Process.EdgeMetadatas[edgeKeyIngress] = report.EdgeMetadata{ + r.Endpoint.EdgeMetadatas[edgeKeyIngress] = report.EdgeMetadata{ WithConnCountTCP: true, MaxConnCountTCP: uint(rand.Intn(100) + 10), } - // Network topology - if _, ok := r.Network.NodeMetadatas[srcAddressID]; !ok { - r.Network.NodeMetadatas[srcAddressID] = report.NodeMetadata{ + // Address topology + if _, ok := r.Address.NodeMetadatas[srcAddressID]; !ok { + r.Address.NodeMetadatas[srcAddressID] = report.NodeMetadata{ "name": src, } } - r.Network.Adjacency[nodeSrcAddressID] = r.Network.Adjacency[nodeSrcAddressID].Add(dstAddressID) - if _, ok := r.Network.NodeMetadatas[dstAddressID]; !ok { - r.Network.NodeMetadatas[dstAddressID] = report.NodeMetadata{ + r.Address.Adjacency[nodeSrcAddressID] = r.Address.Adjacency[nodeSrcAddressID].Add(dstAddressID) + if _, ok := r.Address.NodeMetadatas[dstAddressID]; !ok { + r.Address.NodeMetadatas[dstAddressID] = report.NodeMetadata{ "name": dst, } } - r.Network.Adjacency[nodeDstAddressID] = r.Network.Adjacency[nodeDstAddressID].Add(srcAddressID) + r.Address.Adjacency[nodeDstAddressID] = r.Address.Adjacency[nodeDstAddressID].Add(srcAddressID) - // Host data - r.HostMetadatas["hostX"] = report.HostMetadata{ - Timestamp: time.Now().UTC(), - Hostname: "host-x", - LocalNets: []*net.IPNet{localNet}, - OS: "linux", + // Host topology + r.Host.NodeMetadatas["hostX"] = report.NodeMetadata{ + report.HostTimestampKey: time.Now().UTC().Format(time.RFC3339Nano), + report.HostHostnameKey: "host-x", + report.HostLocalNetsKey: localNet.String(), + report.HostOSKey: "linux", } } diff --git a/experimental/demoprobe/main.go b/experimental/_demoprobe/main.go similarity index 100% rename from experimental/demoprobe/main.go rename to experimental/_demoprobe/main.go diff --git a/experimental/genreport/Makefile b/experimental/_genreport/Makefile similarity index 100% rename from experimental/genreport/Makefile rename to experimental/_genreport/Makefile diff --git a/experimental/genreport/generate.go b/experimental/_genreport/generate.go similarity index 68% rename from experimental/genreport/generate.go rename to experimental/_genreport/generate.go index 0de3008f30..a71c7d4b36 100644 --- a/experimental/genreport/generate.go +++ b/experimental/_genreport/generate.go @@ -66,56 +66,56 @@ func DemoReport(nodeCount int) report.Report { nodeDstAddressID = "hostX" + report.IDDelim + dstAddressID ) - // Process topology - if _, ok := r.Process.NodeMetadatas[srcPortID]; !ok { - r.Process.NodeMetadatas[srcPortID] = report.NodeMetadata{ + // Endpoint topology + if _, ok := r.Endpoint.NodeMetadatas[srcPortID]; !ok { + r.Endpoint.NodeMetadatas[srcPortID] = report.NodeMetadata{ "pid": "4000", "name": c.srcProc, "domain": "node-" + src, } } - r.Process.Adjacency[srcID] = r.Process.Adjacency[srcID].Add(dstPortID) - if _, ok := r.Process.NodeMetadatas[dstPortID]; !ok { - r.Process.NodeMetadatas[dstPortID] = report.NodeMetadata{ + r.Endpoint.Adjacency[srcID] = r.Endpoint.Adjacency[srcID].Add(dstPortID) + if _, ok := r.Endpoint.NodeMetadatas[dstPortID]; !ok { + r.Endpoint.NodeMetadatas[dstPortID] = report.NodeMetadata{ "pid": "4000", "name": c.dstProc, "domain": "node-" + dst, } } - r.Process.Adjacency[dstID] = r.Process.Adjacency[dstID].Add(srcPortID) + r.Endpoint.Adjacency[dstID] = r.Endpoint.Adjacency[dstID].Add(srcPortID) var ( edgeKeyEgress = srcPortID + report.IDDelim + dstPortID edgeKeyIngress = dstPortID + report.IDDelim + srcPortID ) - r.Process.EdgeMetadatas[edgeKeyEgress] = report.EdgeMetadata{ + r.Endpoint.EdgeMetadatas[edgeKeyEgress] = report.EdgeMetadata{ WithConnCountTCP: true, MaxConnCountTCP: uint(rand.Intn(100) + 10), } - r.Process.EdgeMetadatas[edgeKeyIngress] = report.EdgeMetadata{ + r.Endpoint.EdgeMetadatas[edgeKeyIngress] = report.EdgeMetadata{ WithConnCountTCP: true, MaxConnCountTCP: uint(rand.Intn(100) + 10), } - // Network topology - if _, ok := r.Network.NodeMetadatas[srcAddressID]; !ok { - r.Network.NodeMetadatas[srcAddressID] = report.NodeMetadata{ + // Address topology + if _, ok := r.Address.NodeMetadatas[srcAddressID]; !ok { + r.Address.NodeMetadatas[srcAddressID] = report.NodeMetadata{ "name": src, } } - r.Network.Adjacency[nodeSrcAddressID] = r.Network.Adjacency[nodeSrcAddressID].Add(dstAddressID) - if _, ok := r.Network.NodeMetadatas[dstAddressID]; !ok { - r.Network.NodeMetadatas[dstAddressID] = report.NodeMetadata{ + r.Address.Adjacency[nodeSrcAddressID] = r.Address.Adjacency[nodeSrcAddressID].Add(dstAddressID) + if _, ok := r.Address.NodeMetadatas[dstAddressID]; !ok { + r.Address.NodeMetadatas[dstAddressID] = report.NodeMetadata{ "name": dst, } } - r.Network.Adjacency[nodeDstAddressID] = r.Network.Adjacency[nodeDstAddressID].Add(srcAddressID) + r.Address.Adjacency[nodeDstAddressID] = r.Address.Adjacency[nodeDstAddressID].Add(srcAddressID) - // Host data - r.HostMetadatas["hostX"] = report.HostMetadata{ - Timestamp: time.Now().UTC(), - Hostname: "host-x", - LocalNets: []*net.IPNet{localNet}, - OS: "linux", + // Host topology + r.Host.NodeMetadatas["hostX"] = report.NodeMetadata{ + report.HostTimestampKey: time.Now().UTC().Format(time.RFC3339Nano), + report.HostHostnameKey: "host-x", + report.HostLocalNetsKey: localNet.String(), + report.HostOSKey: "linux", } } diff --git a/experimental/genreport/main.go b/experimental/_genreport/main.go similarity index 100% rename from experimental/genreport/main.go rename to experimental/_genreport/main.go diff --git a/experimental/bridge/main.go b/experimental/bridge/main.go index 93277fb947..f2ad37720a 100644 --- a/experimental/bridge/main.go +++ b/experimental/bridge/main.go @@ -50,7 +50,9 @@ func main() { // Collector deals with the probes, and generates a single merged report // every second. c := xfer.NewCollector(*batch) - c.AddAddresses(fixedAddresses) + for _, addr := range fixedAddresses { + c.Add(addr) + } defer c.Stop() publisher, err := xfer.NewTCPPublisher(*listen) @@ -81,8 +83,8 @@ func interrupt() chan os.Signal { type collector interface { Reports() <-chan report.Report - RemoveAddress(string) - AddAddress(string) + Remove(string) + Add(string) } type publisher xfer.Publisher @@ -99,8 +101,8 @@ func makeAvoid(fixed []string) map[string]struct{} { } // Don't go Ouroboros. - if localNets, err := net.InterfaceAddrs(); err == nil { - for _, n := range localNets { + if localNetworks, err := net.InterfaceAddrs(); err == nil { + for _, n := range localNetworks { if net, ok := n.(*net.IPNet); ok { avoid[net.IP.String()] = struct{}{} } @@ -124,13 +126,13 @@ func discover(c collector, p publisher, fixed []string) { p.Publish(r) var ( - now = time.Now() - localNets = r.LocalNets() + now = time.Now() + localNetworks = r.LocalNetworks() ) - for _, adjacent := range r.Network.Adjacency { + for _, adjacent := range r.Address.Adjacency { for _, a := range adjacent { - ip := report.AddressIP(a) // address id -> IP + ip := report.AddressIDAddresser(a) // address node ID -> IP if ip == nil { continue } @@ -141,9 +143,9 @@ func discover(c collector, p publisher, fixed []string) { } // log.Printf("potential address: %v (via %s)", addr, src) if _, ok := lastSeen[addr]; !ok { - if interestingAddress(localNets, addr) { + if interestingAddress(localNetworks, addr) { log.Printf("discovery %v: potential probe address", addr) - c.AddAddress(addressToDial(addr)) + c.Add(addressToDial(addr)) } else { log.Printf("discovery %v: non-probe address", addr) } @@ -164,7 +166,7 @@ func discover(c collector, p publisher, fixed []string) { // anything. log.Printf("discovery %v: traffic timeout", addr) delete(lastSeen, addr) - c.RemoveAddress(addressToDial(addr)) + c.Remove(addressToDial(addr)) } } } @@ -172,7 +174,7 @@ func discover(c collector, p publisher, fixed []string) { // interestingAddress tells whether the address is a local and normal address, // which we want to try to connect to. -func interestingAddress(localNets []*net.IPNet, addr string) bool { +func interestingAddress(localNetworks []*net.IPNet, addr string) bool { if addr == "" { return false } @@ -189,7 +191,7 @@ func interestingAddress(localNets []*net.IPNet, addr string) bool { } // Only connect to addresses we know are localnet. - for _, n := range localNets { + for _, n := range localNetworks { if n.Contains(ip) { return true } diff --git a/experimental/graphviz/handle.go b/experimental/graphviz/handle.go index c30cee2ea7..c61ef77bfc 100644 --- a/experimental/graphviz/handle.go +++ b/experimental/graphviz/handle.go @@ -14,7 +14,7 @@ import ( func handleTXT(r Reporter) http.HandlerFunc { return func(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "text/plain") - dot(w, r.Report().Process.RenderBy(mapFunc(req), classView(req))) + dot(w, report.Render(r.Report(), report.SelectEndpoint, mapFunc(req), report.NoPseudoNode)) } } @@ -30,7 +30,7 @@ func handleSVG(r Reporter) http.HandlerFunc { cmd.Stdout = w - dot(wc, r.Report().Process.RenderBy(mapFunc(req), classView(req))) + dot(wc, report.Render(r.Report(), report.SelectEndpoint, mapFunc(req), report.NoPseudoNode)) wc.Close() w.Header().Set("Content-Type", "image/svg+xml") @@ -98,12 +98,15 @@ func engine(r *http.Request) string { func mapFunc(r *http.Request) report.MapFunc { switch strings.ToLower(r.FormValue("map_func")) { - case "hosts", "networkhost", "networkhostname": - return report.NetworkHostname + case "addresshostname": + return report.AddressHostname + case "processname": + return report.ProcessName + case "processcontainer": + return report.ProcessContainer + case "processcontainerimage": + return report.ProcessContainerImage + default: + return report.ProcessPID } - return report.ProcessPID -} - -func classView(r *http.Request) bool { - return r.FormValue("class_view") == "true" } diff --git a/experimental/graphviz/main.go b/experimental/graphviz/main.go index 67ac849465..7aa189a7d8 100644 --- a/experimental/graphviz/main.go +++ b/experimental/graphviz/main.go @@ -27,7 +27,9 @@ func main() { xfer.MaxBackoff = 10 * time.Second c := xfer.NewCollector(*batch) - c.AddAddresses(strings.Split(*probes, ",")) + for _, addr := range strings.Split(*probes, ",") { + c.Add(addr) + } defer c.Stop() lifo := NewReportLIFO(c, *window) defer lifo.Stop() diff --git a/experimental/graphviz/report_lifo.go b/experimental/graphviz/report_lifo.go index c785e24d7d..d28572e457 100644 --- a/experimental/graphviz/report_lifo.go +++ b/experimental/graphviz/report_lifo.go @@ -41,7 +41,7 @@ func NewReportLIFO(r reporter, maxAge time.Duration) *ReportLIFO { for { select { case report := <-r.Reports(): - report = report.SquashRemote() + report = report.Squash() tr := timedReport{ Timestamp: time.Now(), Report: report, @@ -50,7 +50,7 @@ func NewReportLIFO(r reporter, maxAge time.Duration) *ReportLIFO { l.reports = cleanOld(l.reports, time.Now().Add(-maxAge)) case req := <-l.requests: - report := report.NewReport() + report := report.MakeReport() for _, r := range l.reports { report.Merge(r.Report) } diff --git a/experimental/oneshot/main.go b/experimental/oneshot/main.go index 01c645a4df..8c7ef5e65a 100644 --- a/experimental/oneshot/main.go +++ b/experimental/oneshot/main.go @@ -29,10 +29,12 @@ func main() { // Collector deals with the probes, and generates merged reports. xfer.MaxBackoff = 1 * time.Second c := xfer.NewCollector(1 * time.Second) - c.AddAddresses(strings.Split(*probes, ",")) + for _, addr := range strings.Split(*probes, ",") { + c.Add(addr) + } defer c.Stop() - report := report.NewReport() + report := report.MakeReport() irq := interrupt() OUTER: for { diff --git a/probe/docker_process_mapper.go b/probe/docker_process_mapper.go deleted file mode 100644 index 6687932323..0000000000 --- a/probe/docker_process_mapper.go +++ /dev/null @@ -1,304 +0,0 @@ -package main - -import ( - "fmt" - "log" - "strings" - "sync" - "time" - - docker "github.com/fsouza/go-dockerclient" -) - -const ( - stop = "stop" - start = "start" -) - -type dockerMapper struct { - sync.RWMutex - quit chan struct{} - interval time.Duration - - containers map[string]*docker.Container - containersByPID map[int]*docker.Container - images map[string]*docker.APIImages - - procRoot string - pidTree *pidTree -} - -func newDockerMapper(procRoot string, interval time.Duration) (*dockerMapper, error) { - pidTree, err := newPIDTreeStub(procRoot) - if err != nil { - return nil, err - } - - m := dockerMapper{ - containers: map[string]*docker.Container{}, - containersByPID: map[int]*docker.Container{}, - images: map[string]*docker.APIImages{}, - - procRoot: procRoot, - pidTree: pidTree, - - interval: interval, - quit: make(chan struct{}), - } - - go m.loop() - return &m, nil -} - -func (m *dockerMapper) Stop() { - close(m.quit) -} - -func (m *dockerMapper) loop() { - if !m.update() { - return - } - - ticker := time.Tick(m.interval) - for { - select { - case <-ticker: - if !m.update() { - return - } - - case <-m.quit: - return - } - } -} - -// for mocking -type dockerClient interface { - ListContainers(docker.ListContainersOptions) ([]docker.APIContainers, error) - InspectContainer(string) (*docker.Container, error) - ListImages(docker.ListImagesOptions) ([]docker.APIImages, error) - AddEventListener(chan<- *docker.APIEvents) error - RemoveEventListener(chan *docker.APIEvents) error -} - -func newRealDockerClient(endpoint string) (dockerClient, error) { - return docker.NewClient(endpoint) -} - -var ( - newDockerClient = newRealDockerClient - newPIDTreeStub = newPIDTree -) - -// returns false when stopping. -func (m *dockerMapper) update() bool { - endpoint := "unix:///var/run/docker.sock" - client, err := newDockerClient(endpoint) - if err != nil { - log.Printf("docker mapper: %s", err) - return true - } - - events := make(chan *docker.APIEvents) - if err := client.AddEventListener(events); err != nil { - log.Printf("docker mapper: %s", err) - return true - } - defer func() { - if err := client.RemoveEventListener(events); err != nil { - log.Printf("docker mapper: %s", err) - } - }() - - if err := m.updateContainers(client); err != nil { - log.Printf("docker mapper: %s", err) - return true - } - - if err := m.updateImages(client); err != nil { - log.Printf("docker mapper: %s", err) - return true - } - - otherUpdates := time.Tick(m.interval) - for { - select { - case event := <-events: - m.handleEvent(event, client) - - case <-otherUpdates: - if err := m.updatePIDTree(); err != nil { - log.Printf("docker mapper: %s", err) - continue - } - - if err := m.updateImages(client); err != nil { - log.Printf("docker mapper: %s", err) - continue - } - - case <-m.quit: - return false - } - } -} - -func (m *dockerMapper) updateContainers(client dockerClient) error { - apiContainers, err := client.ListContainers(docker.ListContainersOptions{All: true}) - if err != nil { - return err - } - - containers := []*docker.Container{} - for _, apiContainer := range apiContainers { - container, err := client.InspectContainer(apiContainer.ID) - if err != nil { - log.Printf("docker mapper: %s", err) - continue - } - - if !container.State.Running { - continue - } - - containers = append(containers, container) - } - - m.Lock() - for _, container := range containers { - m.containers[container.ID] = container - m.containersByPID[container.State.Pid] = container - } - m.Unlock() - - return nil -} - -func (m *dockerMapper) updateImages(client dockerClient) error { - images, err := client.ListImages(docker.ListImagesOptions{}) - if err != nil { - return err - } - - m.Lock() - for i := range images { - image := &images[i] - m.images[image.ID] = image - } - m.Unlock() - - return nil -} - -func (m *dockerMapper) handleEvent(event *docker.APIEvents, client dockerClient) { - switch event.Status { - case stop: - containerID := event.ID - m.Lock() - if container, ok := m.containers[containerID]; ok { - delete(m.containers, containerID) - delete(m.containersByPID, container.State.Pid) - } else { - log.Printf("docker mapper: container %s not found", containerID) - } - m.Unlock() - - case start: - containerID := event.ID - container, err := client.InspectContainer(containerID) - if err != nil { - log.Printf("docker mapper: %s", err) - return - } - - if !container.State.Running { - log.Printf("docker mapper: container %s not running", containerID) - return - } - - m.Lock() - m.containers[containerID] = container - m.containersByPID[container.State.Pid] = container - m.Unlock() - } -} - -func (m *dockerMapper) updatePIDTree() error { - pidTree, err := newPIDTreeStub(m.procRoot) - if err != nil { - return err - } - - m.Lock() - m.pidTree = pidTree - m.Unlock() - return nil -} - -type dockerProcessMapper struct { - *dockerMapper - key string - f func(*docker.Container) string -} - -func (m *dockerProcessMapper) Key() string { return m.key } -func (m *dockerProcessMapper) Map(pid uint) (string, error) { - var ( - container *docker.Container - ok bool - err error - candidate = int(pid) - ) - - m.RLock() - for { - container, ok = m.containersByPID[candidate] - if ok { - break - } - candidate, err = m.pidTree.getParent(candidate) - if err != nil { - break - } - } - m.RUnlock() - - if err != nil { - return "", fmt.Errorf("no container found for PID %d", pid) - } - - return m.f(container), nil -} - -func (m *dockerMapper) idMapper() processMapper { - return &dockerProcessMapper{m, "docker_id", func(c *docker.Container) string { - return c.ID - }} -} - -func (m *dockerMapper) nameMapper() processMapper { - return &dockerProcessMapper{m, "docker_name", func(c *docker.Container) string { - return strings.TrimPrefix(c.Name, "/") - }} -} - -func (m *dockerMapper) imageIDMapper() processMapper { - return &dockerProcessMapper{m, "docker_image_id", func(c *docker.Container) string { - return c.Image - }} -} - -func (m *dockerMapper) imageNameMapper() processMapper { - return &dockerProcessMapper{m, "docker_image_name", func(c *docker.Container) string { - m.RLock() - image, ok := m.images[c.Image] - m.RUnlock() - - if !ok || len(image.RepoTags) == 0 { - return "" - } - - return image.RepoTags[0] - }} -} diff --git a/probe/docker_process_mapper_test.go b/probe/docker_process_mapper_test.go deleted file mode 100644 index aaba6f8d61..0000000000 --- a/probe/docker_process_mapper_test.go +++ /dev/null @@ -1,100 +0,0 @@ -package main - -import ( - "runtime" - "testing" - "time" - - docker "github.com/fsouza/go-dockerclient" -) - -type mockDockerClient struct { - apiContainers []docker.APIContainers - containers map[string]*docker.Container - apiImages []docker.APIImages -} - -func (m mockDockerClient) ListContainers(options docker.ListContainersOptions) ([]docker.APIContainers, error) { - return m.apiContainers, nil -} - -func (m mockDockerClient) InspectContainer(id string) (*docker.Container, error) { - return m.containers[id], nil -} - -func (m mockDockerClient) ListImages(options docker.ListImagesOptions) ([]docker.APIImages, error) { - return m.apiImages, nil -} - -func (m mockDockerClient) AddEventListener(events chan<- *docker.APIEvents) error { - return nil -} - -func (m mockDockerClient) RemoveEventListener(events chan *docker.APIEvents) error { - return nil -} - -func TestDockerProcessMapper(t *testing.T) { - oldPIDTreeStub, oldDockerClientStub := newPIDTreeStub, newDockerClient - defer func() { - newPIDTreeStub = oldPIDTreeStub - newDockerClient = oldDockerClientStub - }() - - newPIDTreeStub = func(procRoot string) (*pidTree, error) { - pid1 := &process{pid: 1} - pid2 := &process{pid: 2, ppid: 1, parent: pid1} - pid1.children = []*process{pid2} - - return &pidTree{ - processes: map[int]*process{ - 1: pid1, 2: pid2, - }, - }, nil - } - - newDockerClient = func(endpoint string) (dockerClient, error) { - return mockDockerClient{ - apiContainers: []docker.APIContainers{{ID: "foo"}}, - containers: map[string]*docker.Container{ - "foo": { - ID: "foo", - Name: "bar", - Image: "baz", - State: docker.State{Pid: 1, Running: true}, - }, - }, - apiImages: []docker.APIImages{{ID: "baz", RepoTags: []string{"tag"}}}, - }, nil - } - - dockerMapper, _ := newDockerMapper("/proc", 10*time.Second) - dockerIDMapper := dockerMapper.idMapper() - dockerNameMapper := dockerMapper.nameMapper() - dockerImageIDMapper := dockerMapper.imageIDMapper() - dockerImageNameMapper := dockerMapper.imageNameMapper() - - runtime.Gosched() - - for pid, want := range map[uint]struct{ id, name, imageID, imageName string }{ - 1: {"foo", "bar", "baz", "tag"}, - 2: {"foo", "bar", "baz", "tag"}, - } { - haveID, err := dockerIDMapper.Map(pid) - if err != nil || want.id != haveID { - t.Errorf("%d: want %q, have %q (%v)", pid, want.id, haveID, err) - } - haveName, err := dockerNameMapper.Map(pid) - if err != nil || want.name != haveName { - t.Errorf("%d: want %q, have %q (%v)", pid, want.name, haveName, err) - } - haveImageID, err := dockerImageIDMapper.Map(pid) - if err != nil || want.imageID != haveImageID { - t.Errorf("%d: want %q, have %q (%v)", pid, want.imageID, haveImageID, err) - } - haveImageName, err := dockerImageNameMapper.Map(pid) - if err != nil || want.imageName != haveImageName { - t.Errorf("%d: want %q, have %q (%v)", pid, want.imageName, haveImageName, err) - } - } -} diff --git a/probe/hostname.go b/probe/hostname.go deleted file mode 100644 index 1bf88e2f78..0000000000 --- a/probe/hostname.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -import "os" - -func hostname() string { - if hostname := os.Getenv("SCOPE_HOSTNAME"); hostname != "" { - return hostname - } - hostname, err := os.Hostname() - if err != nil { - return "(unknown)" - } - return hostname -} diff --git a/probe/main.go b/probe/main.go index ce7fb27331..dd3c6434f6 100644 --- a/probe/main.go +++ b/probe/main.go @@ -5,21 +5,21 @@ import ( "log" "net" "net/http" - _ "net/http/pprof" "os" "os/signal" "runtime" "strconv" + "strings" "syscall" "time" "github.com/weaveworks/procspy" + "github.com/weaveworks/scope/probe/tag" "github.com/weaveworks/scope/report" "github.com/weaveworks/scope/xfer" ) -// Set during buildtime. -var version = "unknown" +var version = "dev" // set at build time func main() { var ( @@ -29,9 +29,7 @@ func main() { listen = flag.String("listen", ":"+strconv.Itoa(xfer.ProbePort), "listen address") prometheusEndpoint = flag.String("prometheus.endpoint", "/metrics", "Prometheus metrics exposition endpoint (requires -http.listen)") spyProcs = flag.Bool("processes", true, "report processes (needs root)") - cgroupsRoot = flag.String("cgroups.root", "", "if provided, enrich -processes with cgroup names from this root (e.g. /mnt/cgroups)") - cgroupsInterval = flag.Duration("cgroups.interval", 10*time.Second, "how often to update cgroup names") - dockerMapper = flag.Bool("docker", true, "collect Docker-related attributes for processes") + dockerTagger = flag.Bool("docker", true, "collect Docker-related attributes for processes") dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes") procRoot = flag.String("proc.root", "/proc", "location of the proc filesystem") ) @@ -42,7 +40,7 @@ func main() { os.Exit(1) } - log.Printf("probe starting, version %s", version) + log.Printf("probe version %s", version) procspy.SetProcRoot(*procRoot) @@ -57,7 +55,7 @@ func main() { } if *spyProcs && os.Getegid() != 0 { - log.Printf("warning: process reporting enabled, but that requires root to find everything") + log.Printf("warning: process reporting enabled, but not running as root: will miss some things") } publisher, err := xfer.NewTCPPublisher(*listen) @@ -66,57 +64,38 @@ func main() { } defer publisher.Close() - pms := []processMapper{identityMapper{}} - - if *cgroupsRoot != "" { - if fi, err := os.Stat(*cgroupsRoot); err == nil && fi.IsDir() { - log.Printf("enriching -processes with cgroup names from %s", *cgroupsRoot) - cgroupMapper := newCgroupMapper(*cgroupsRoot, *cgroupsInterval) - defer cgroupMapper.Stop() - pms = append(pms, cgroupMapper) - } else { - log.Printf("-cgroups.root=%s: %v", *cgroupsRoot, err) - } - } - - if *dockerMapper { - docker, err := newDockerMapper(*procRoot, *dockerInterval) - if err != nil { - log.Fatal(err) - } - defer docker.Stop() - - pms = append(pms, - docker.idMapper(), - docker.nameMapper(), - docker.imageIDMapper(), - docker.imageNameMapper(), - ) + taggers := []tag.Tagger{tag.NewTopologyTagger()} + if *dockerTagger { + t := tag.NewDockerTagger(*procRoot, *dockerInterval) + defer t.Stop() + taggers = append(taggers, t) } log.Printf("listening on %s", *listen) quit := make(chan struct{}) defer close(quit) + go func() { var ( hostname = hostname() - nodeID = hostname // TODO: we should sanitize the hostname + hostID = hostname // TODO: we should sanitize the hostname pubTick = time.Tick(*publishInterval) spyTick = time.Tick(*spyInterval) - r = report.NewReport() + r = report.MakeReport() ) for { select { case <-pubTick: publishTicks.WithLabelValues().Add(1) - r.HostMetadatas[nodeID] = hostMetadata(hostname) + r.Host = hostTopology(hostID, hostname) + r = tag.Apply(r, taggers) publisher.Publish(r) - r = report.NewReport() + r = report.MakeReport() case <-spyTick: - r.Merge(spy(hostname, hostname, *spyProcs, pms)) + r.Merge(spy(hostname, hostname, *spyProcs)) // log.Printf("merged report:\n%#v\n", r) case <-quit: @@ -128,34 +107,42 @@ func main() { log.Printf("%s", <-interrupt()) } -func interrupt() chan os.Signal { - c := make(chan os.Signal) - signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) - return c -} - -// hostMetadata produces an instantaneous HostMetadata for this host. No need -// to do this more than once per published report. -func hostMetadata(hostname string) report.HostMetadata { - loadOne, loadFive, loadFifteen := getLoads() - - host := report.HostMetadata{ - Timestamp: time.Now().UTC(), - Hostname: hostname, - OS: runtime.GOOS, - LoadOne: loadOne, - LoadFive: loadFive, - LoadFifteen: loadFifteen, +func hostname() string { + if hostname := os.Getenv("SCOPE_HOSTNAME"); hostname != "" { + return hostname + } + hostname, err := os.Hostname() + if err != nil { + return "(unknown)" } + return hostname +} +// hostTopology produces a host topology for this host. No need to do this +// more than once per published report. +func hostTopology(hostID, hostname string) report.Topology { + var localCIDRs []string if localNets, err := net.InterfaceAddrs(); err == nil { // Not all networks are IP networks. for _, localNet := range localNets { - if net, ok := localNet.(*net.IPNet); ok { - host.LocalNets = append(host.LocalNets, net) + if ipNet, ok := localNet.(*net.IPNet); ok { + localCIDRs = append(localCIDRs, ipNet.String()) } } } + t := report.MakeTopology() + t.NodeMetadatas[hostID] = report.NodeMetadata{ + "ts": time.Now().UTC().Format(time.RFC3339Nano), + "host_name": hostname, + "local_networks": strings.Join(localCIDRs, " "), + "os": runtime.GOOS, + "load": getLoad(), + } + return t +} - return host +func interrupt() chan os.Signal { + c := make(chan os.Signal) + signal.Notify(c, syscall.SIGINT, syscall.SIGTERM) + return c } diff --git a/probe/process_mapper.go b/probe/process_mapper.go deleted file mode 100644 index cfb59307d7..0000000000 --- a/probe/process_mapper.go +++ /dev/null @@ -1,143 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "log" - "os" - "path/filepath" - "regexp" - "strconv" - "strings" - "sync" - "time" -) - -type processMapper interface { - Key() string - Map(pid uint) (string, error) -} - -type identityMapper struct{} - -func (m identityMapper) Key() string { return "identity" } -func (m identityMapper) Map(pid uint) (string, error) { return strconv.FormatUint(uint64(pid), 10), nil } - -// cgroupMapper is a cgroup task mapper. -type cgroupMapper struct { - sync.RWMutex - root string - d map[uint]string - quit chan struct{} -} - -func newCgroupMapper(root string, interval time.Duration) *cgroupMapper { - m := cgroupMapper{ - root: root, - d: map[uint]string{}, - quit: make(chan struct{}), - } - m.update() - go m.loop(interval) - return &m -} - -func (m *cgroupMapper) Stop() { - close(m.quit) -} - -func (m *cgroupMapper) Key() string { return "cgroup" } - -// Map uses the cache to find the process name for pid. It is safe for -// concurrent use. -func (m *cgroupMapper) Map(pid uint) (string, error) { - m.RLock() - p, ok := m.d[pid] - m.RUnlock() - - if !ok { - return "", fmt.Errorf("no cgroup for PID %d", pid) - } - - return p, nil -} - -func (m *cgroupMapper) loop(d time.Duration) { - ticker := time.Tick(d) - for { - select { - case <-ticker: - m.update() - case <-m.quit: - return - } - } -} - -func (m *cgroupMapper) update() { - // We want to read "//tasks" files. - fh, err := os.Open(m.root) - if err != nil { - log.Printf("cgroup mapper: %s", err) - return - } - - dirNames, err := fh.Readdirnames(-1) - fh.Close() - if err != nil { - log.Printf("cgroup mapper: %s", err) - return - } - - pmap := map[uint]string{} - for _, d := range dirNames { - cg := normalizeCgroup(d) - dirFilename := filepath.Join(m.root, d) - - s, err := os.Stat(dirFilename) - if err != nil || !s.IsDir() { - continue - } - - taskFilename := filepath.Join(dirFilename, "tasks") - - f, err := os.Open(taskFilename) - if err != nil { - continue - } - - r := bufio.NewReader(f) - for { - line, _, err := r.ReadLine() - if err != nil { - break // we expect an EOF - } - - pid, err := strconv.ParseUint(string(line), 10, 64) - if err != nil { - log.Printf("continue mapper: %s", err) - continue - } - - pmap[uint(pid)] = cg - } - - f.Close() - } - - m.Lock() - m.d = pmap - m.Unlock() -} - -var lxcRe = regexp.MustCompile(`^([^-]+)-([^-]+)-([A-Fa-f0-9]+)-([0-9]+)$`) - -func normalizeCgroup(s string) string { - // Format is currently "primarykey-secondarykey-revision-instance". We - // want to collapse all instances (and maybe all revisions, in the future) - // to the same node. So we remove the instance. - if m := lxcRe.FindStringSubmatch(s); len(m) > 0 { - return strings.Join([]string{m[1], m[2], m[3]}, "-") - } - return s -} diff --git a/probe/process_mapper_test.go b/probe/process_mapper_test.go deleted file mode 100644 index 99421a9fc5..0000000000 --- a/probe/process_mapper_test.go +++ /dev/null @@ -1,63 +0,0 @@ -package main - -import ( - "io/ioutil" - "os" - "path" - "path/filepath" - "testing" - "time" -) - -func TestCgroupMapper(t *testing.T) { - tmp := setupTmpFS(t, map[string]string{ - "/systemd/tasks": "1\n2\n4911\n1000\n25156\n", - "/systemd/notify_on_release": "0\n", - "/netscape/tasks": "666\n4242\n", - "/netscape/notify_on_release": "0\n", - "/weirdfile": "", - }) - defer removeAll(t, tmp) - - m := newCgroupMapper(tmp, 1*time.Second) - for pid, want := range map[uint]string{ - 111: "", - 999: "", - 4911: "systemd", - 1: "systemd", // first one in the file - 25156: "systemd", // last one in the tasks file - 4242: "netscape", - } { - if have, _ := m.Map(pid); want != have { - t.Errorf("%d: want %q, have %q", pid, want, have) - } - } -} - -func setupTmpFS(t *testing.T, fs map[string]string) string { - tmp, err := ioutil.TempDir(os.TempDir(), "scope-probe-test-cgroup-mapper") - if err != nil { - t.Fatal(err) - } - //t.Logf("using TempDir %s", tmp) - - for file, content := range fs { - dir := path.Dir(file) - if err := os.MkdirAll(filepath.Join(tmp, dir), 0777); err != nil { - removeAll(t, tmp) - t.Fatalf("MkdirAll: %v", err) - } - - if err := ioutil.WriteFile(filepath.Join(tmp, file), []byte(content), 0655); err != nil { - removeAll(t, tmp) - t.Fatalf("WriteFile: %v", err) - } - } - return tmp -} - -func removeAll(t *testing.T, path string) { - if err := os.RemoveAll(path); err != nil { - t.Error(err) - } -} diff --git a/probe/spy.go b/probe/spy.go index 422f205de4..47900d3e2e 100644 --- a/probe/spy.go +++ b/probe/spy.go @@ -3,8 +3,6 @@ package main import ( "fmt" "log" - "net" - "strconv" "time" "github.com/weaveworks/procspy" @@ -15,102 +13,95 @@ import ( // every discovered (spied) connection on the host machine, at the granularity // of host and port. It optionally enriches that topology with process (PID) // information. -func spy( - hostID, hostName string, - includeProcesses bool, - pms []processMapper, -) report.Report { +func spy(hostID, hostName string, includeProcesses bool) report.Report { defer func(begin time.Time) { spyDuration.WithLabelValues().Observe(float64(time.Since(begin))) }(time.Now()) - r := report.NewReport() + r := report.MakeReport() conns, err := procspy.Connections(includeProcesses) if err != nil { log.Printf("spy connections: %v", err) return r } - for conn := conns.Next(); conn != nil; conn = conns.Next() { - addConnection(&r, conn, hostID, hostName, pms) + r = addConnection(r, conn, hostID, hostName) } return r } -func addConnection( - r *report.Report, - c *procspy.Connection, - hostID, hostName string, - pms []processMapper, -) { +func addConnection(r report.Report, c *procspy.Connection, hostID, hostName string) report.Report { var ( - scopedLocal = scopedIP(hostID, c.LocalAddress) - scopedRemote = scopedIP(hostID, c.RemoteAddress) - key = hostID + report.IDDelim + scopedLocal - edgeKey = scopedLocal + report.IDDelim + scopedRemote + addressNodeID = report.MakeAddressNodeID(hostID, c.LocalAddress.String()) + endpointNodeID = report.MakeEndpointNodeID(hostID, c.LocalAddress.String(), fmt.Sprint(c.LocalPort)) ) - - r.Network.Adjacency[key] = r.Network.Adjacency[key].Add(scopedRemote) - - if _, ok := r.Network.NodeMetadatas[scopedLocal]; !ok { - r.Network.NodeMetadatas[scopedLocal] = report.NodeMetadata{ - "name": hostName, + { + var ( + remoteNodeID = report.MakeAddressNodeID(hostID, c.RemoteAddress.String()) + adjacencyID = report.MakeAdjacencyID(hostID, addressNodeID) + edgeID = report.MakeEdgeID(addressNodeID, remoteNodeID) + ) + r.Address.Adjacency[adjacencyID] = r.Address.Adjacency[adjacencyID].Add(remoteNodeID) + if _, ok := r.Address.NodeMetadatas[addressNodeID]; !ok { + r.Address.NodeMetadatas[addressNodeID] = report.NodeMetadata{ + "host_id": hostID, + "host_name": hostName, + "local_address": c.LocalAddress.String(), + // Can't put remote information here, meaingfully, as the same + // endpoint may appear in multiple connections with different + // remotes. + } } + md := r.Address.EdgeMetadatas[edgeID] + md.WithConnCountTCP = true + md.MaxConnCountTCP++ + r.Address.EdgeMetadatas[edgeID] = md } - - // Count the TCP connection. - edgeMeta := r.Network.EdgeMetadatas[edgeKey] - edgeMeta.WithConnCountTCP = true - edgeMeta.MaxConnCountTCP++ - r.Network.EdgeMetadatas[edgeKey] = edgeMeta - - if c.Proc.PID > 0 { + { var ( - scopedLocal = scopedIPPort(hostID, c.LocalAddress, c.LocalPort) - scopedRemote = scopedIPPort(hostID, c.RemoteAddress, c.RemotePort) - key = hostID + report.IDDelim + scopedLocal - edgeKey = scopedLocal + report.IDDelim + scopedRemote + remoteNodeID = report.MakeEndpointNodeID(hostID, c.RemoteAddress.String(), fmt.Sprint(c.RemotePort)) + adjacencyID = report.MakeAdjacencyID(hostID, endpointNodeID) + edgeID = report.MakeEdgeID(endpointNodeID, remoteNodeID) ) - - r.Process.Adjacency[key] = r.Process.Adjacency[key].Add(scopedRemote) - - if _, ok := r.Process.NodeMetadatas[scopedLocal]; !ok { - // First hit establishes NodeMetadata for scoped local address + port - md := report.NodeMetadata{ - "pid": fmt.Sprintf("%d", c.Proc.PID), - "name": c.Proc.Name, - "domain": hostID, - } - - for _, pm := range pms { - v, err := pm.Map(c.PID) - if err != nil { - continue - } - md[pm.Key()] = v + r.Endpoint.Adjacency[adjacencyID] = r.Endpoint.Adjacency[adjacencyID].Add(remoteNodeID) + if _, ok := r.Endpoint.NodeMetadatas[endpointNodeID]; !ok { + r.Endpoint.NodeMetadatas[endpointNodeID] = report.NodeMetadata{ + "host_id": hostID, + "host_name": hostName, + "address": c.LocalAddress.String(), + "port": fmt.Sprintf("%d", c.LocalPort), + // Can't put remote information here, meaingfully, as the same + // endpoint may appear in multiple connections with different + // remotes. } - - r.Process.NodeMetadatas[scopedLocal] = md } - // Count the TCP connection. - edgeMeta := r.Process.EdgeMetadatas[edgeKey] - edgeMeta.WithConnCountTCP = true - edgeMeta.MaxConnCountTCP++ - r.Process.EdgeMetadatas[edgeKey] = edgeMeta + md := r.Endpoint.EdgeMetadatas[edgeID] + md.WithConnCountTCP = true + md.MaxConnCountTCP++ + r.Endpoint.EdgeMetadatas[edgeID] = md } -} - -// scopedIP makes an IP unique over multiple networks. -func scopedIP(scope string, ip net.IP) string { - if ip.IsLoopback() { - return scope + report.ScopeDelim + ip.String() + if c.Proc.PID > 0 { + var ( + pidStr = fmt.Sprint(c.Proc.PID) + processNodeID = report.MakeProcessNodeID(hostID, pidStr) + ) + if _, ok := r.Process.NodeMetadatas[processNodeID]; !ok { + r.Process.NodeMetadatas[processNodeID] = report.NodeMetadata{ + "pid": fmt.Sprintf("%d", c.Proc.PID), + "process_name": c.Proc.Name, + "host_id": hostID, + "host_name": hostName, + } + } + // We don't currently have enough info to build process-to-process + // edges. But we do want to make a foreign-key-like association from + // the endpoint to this process... + r.Endpoint.NodeMetadatas[endpointNodeID]["process_node_id"] = processNodeID + // That works as it's one-to-one. We could make a similar many-to-one + // relationship from r.Address to this process, but it's more + // complicated and less useful. } - return report.ScopeDelim + ip.String() -} - -// scopedIPPort makes an IP+port tuple unique over multiple networks. -func scopedIPPort(scope string, ip net.IP, port uint16) string { - return scopedIP(scope, ip) + report.ScopeDelim + strconv.FormatUint(uint64(port), 10) + return r } diff --git a/probe/spy_test.go b/probe/spy_test.go index a7e80df1c1..b295fa11da 100644 --- a/probe/spy_test.go +++ b/probe/spy_test.go @@ -1,27 +1,82 @@ package main import ( + "fmt" "net" - "strconv" "testing" "github.com/weaveworks/procspy" "github.com/weaveworks/scope/report" ) -func TestScopedIP(t *testing.T) { - const scope = "my-scope" +func TestSpyNetwork(t *testing.T) { + procspy.SetFixtures(fixConnections) + const ( + hostID = "heinz-tomato-ketchup" + hostName = "frenchs-since-1904" + ) + r := spy(hostID, hostName, false) + //buf, _ := json.MarshalIndent(r, "", " ") + //t.Logf("\n%s\n", buf) + + // We passed fixConnections without processes. Make sure we don't get + // process nodes. + if want, have := 0, len(r.Process.NodeMetadatas); want != have { + t.Errorf("want %d, have %d", want, have) + } + + var ( + localAddressNodeID = report.MakeAddressNodeID(hostID, fixLocalAddress.String()) + remoteAddressNodeID = report.MakeAddressNodeID(hostID, fixRemoteAddress.String()) + adjacencyID = report.MakeAdjacencyID(hostID, localAddressNodeID) + ) + if want, have := 1, len(r.Address.Adjacency[adjacencyID]); want != have { + t.Fatalf("want %d, have %d", want, have) + } + if want, have := remoteAddressNodeID, r.Address.Adjacency[adjacencyID][0]; want != have { + t.Fatalf("want %q, have %q", want, have) + } + if want, have := hostName, r.Address.NodeMetadatas[localAddressNodeID]["host_name"]; want != have { + t.Fatalf("want %q, have %q", want, have) + } +} - for ip, want := range map[string]string{ - "1.2.3.4": report.ScopeDelim + "1.2.3.4", - "192.168.1.2": report.ScopeDelim + "192.168.1.2", - "127.0.0.1": scope + report.ScopeDelim + "127.0.0.1", // loopback - "::1": scope + report.ScopeDelim + "::1", // loopback - "fd00::451b:b714:85da:489e": report.ScopeDelim + "fd00::451b:b714:85da:489e", // global address - "fe80::82ee:73ff:fe83:588f": report.ScopeDelim + "fe80::82ee:73ff:fe83:588f", // link-local address +func TestSpyProcess(t *testing.T) { + procspy.SetFixtures(fixConnectionsWithProcesses) + const ( + hostID = "nikon" + hostName = "fishermans-friend" + ) + r := spy(hostID, hostName, true) + //buf, _ := json.MarshalIndent(r, "", " ") + //t.Logf("\n%s\n", buf) + + var ( + processNodeID = report.MakeProcessNodeID(hostID, fmt.Sprint(fixProcessPIDB)) + localEndpointNodeID = report.MakeEndpointNodeID(hostID, fixLocalAddress.String(), fmt.Sprint(fixLocalPort)) + remoteEndpointNodeID = report.MakeEndpointNodeID(hostID, fixRemoteAddress.String(), fmt.Sprint(fixRemotePort)) + adjacencyID = report.MakeAdjacencyID(hostID, localEndpointNodeID) + ) + if want, have := 1, len(r.Endpoint.Adjacency[adjacencyID]); want != have { + t.Fatalf("want %d, have %d", want, have) + } + if want, have := remoteEndpointNodeID, r.Endpoint.Adjacency[adjacencyID][0]; want != have { + t.Fatalf("want %q, have %q", want, have) + } + for key, want := range map[string]string{ + "host_id": hostID, + "process_node_id": processNodeID, + } { + if have := r.Endpoint.NodeMetadatas[localEndpointNodeID][key]; want != have { + t.Errorf("Endpoint.NodeMetadatas[%q][%q]: want %q, have %q", localEndpointNodeID, key, want, have) + } + } + for key, want := range map[string]string{ + "pid": fmt.Sprint(fixProcessPIDB), + "process_name": fixProcessName, } { - if have := scopedIP(scope, net.ParseIP(ip)); have != want { - t.Errorf("%q: have %q, want %q", ip, have, want) + if have := r.Process.NodeMetadatas[processNodeID][key]; want != have { + t.Errorf("Process.NodeMetadatas[%q][%q]: want %q, have %q", processNodeID, key, want, have) } } } @@ -78,98 +133,3 @@ var ( }, } ) - -func TestSpyNetwork(t *testing.T) { - procspy.SetFixtures(fixConnections) - - const ( - nodeID = "heinz-tomato-ketchup" - nodeName = "frenchs-since-1904" - ) - - r := spy(nodeID, nodeName, false, []processMapper{}) - //buf, _ := json.MarshalIndent(r, "", " ") - //t.Logf("\n%s\n", buf) - - // No process nodes, please - if want, have := 0, len(r.Process.Adjacency); want != have { - t.Fatalf("want %d, have %d", want, have) - } - - var ( - scopedLocal = scopedIP(nodeID, fixLocalAddress) - scopedRemote = scopedIP(nodeID, fixRemoteAddress) - localKey = nodeID + report.IDDelim + scopedLocal - ) - - if want, have := 1, len(r.Network.Adjacency[localKey]); want != have { - t.Fatalf("want %d, have %d", want, have) - } - - if want, have := scopedRemote, r.Network.Adjacency[localKey][0]; want != have { - t.Fatalf("want %q, have %q", want, have) - } - - if want, have := nodeName, r.Network.NodeMetadatas[scopedLocal]["name"]; want != have { - t.Fatalf("want %q, have %q", want, have) - } -} - -func TestSpyProcess(t *testing.T) { - procspy.SetFixtures(fixConnectionsWithProcesses) - - const ( - nodeID = "nikon" - nodeName = "fishermans-friend" - ) - - r := spy(nodeID, nodeName, true, []processMapper{}) - // buf, _ := json.MarshalIndent(r, "", " ") ; t.Logf("\n%s\n", buf) - - var ( - scopedLocal = scopedIPPort(nodeID, fixLocalAddress, fixLocalPort) - scopedRemote = scopedIPPort(nodeID, fixRemoteAddress, fixRemotePort) - localKey = nodeID + report.IDDelim + scopedLocal - ) - - if want, have := 1, len(r.Process.Adjacency[localKey]); want != have { - t.Fatalf("want %d, have %d", want, have) - } - - if want, have := scopedRemote, r.Process.Adjacency[localKey][0]; want != have { - t.Fatalf("want %q, have %q", want, have) - } - - for key, want := range map[string]string{ - "domain": nodeID, - "name": fixProcessName, - "pid": strconv.FormatUint(uint64(fixProcessPID), 10), - } { - if have := r.Process.NodeMetadatas[scopedLocal][key]; want != have { - t.Errorf("Process.NodeMetadatas[%q][%q]: want %q, have %q", scopedLocal, key, want, have) - } - } -} - -func TestSpyProcessDataSource(t *testing.T) { - procspy.SetFixtures(fixConnectionsWithProcesses) - - const ( - nodeID = "chianti" - nodeName = "harmonisch" - ) - - m := identityMapper{} - r := spy(nodeID, nodeName, true, []processMapper{m}) - scopedLocal := scopedIPPort(nodeID, fixLocalAddress, fixLocalPort) - - k := m.Key() - v, err := m.Map(fixProcessPID) - if err != nil { - t.Fatal(err) - } - - if want, have := v, r.Process.NodeMetadatas[scopedLocal][k]; want != have { - t.Fatalf("%s: want %q, have %q", k, want, have) - } -} diff --git a/probe/system_darwin.go b/probe/system_darwin.go index 33432e0ad9..81874c322a 100644 --- a/probe/system_darwin.go +++ b/probe/system_darwin.go @@ -1,33 +1,21 @@ package main import ( + "fmt" "os/exec" - "strconv" - "strings" + "regexp" ) -func getLoads() (float64, float64, float64) { +var loadRe = regexp.MustCompile(`load average\: ([0-9\.]+), ([0-9\.]+), ([0-9\.]+)`) + +func getLoad() string { out, err := exec.Command("w").CombinedOutput() if err != nil { - return -1, -1, -1 - } - noCommas := strings.NewReplacer(",", "") - firstLine := strings.Split(string(out), "\n")[0] - toks := strings.Fields(firstLine) - if len(toks) < 5 { - return -1, -1, -1 - } - one, err := strconv.ParseFloat(noCommas.Replace(toks[len(toks)-3]), 64) - if err != nil { - return -1, -1, -1 + return "unknown" } - five, err := strconv.ParseFloat(noCommas.Replace(toks[len(toks)-2]), 64) - if err != nil { - return -1, -1, -1 - } - fifteen, err := strconv.ParseFloat(noCommas.Replace(toks[len(toks)-1]), 64) - if err != nil { - return -1, -1, -1 + matches := loadRe.FindAllStringSubmatch(string(out), -1) + if matches == nil || len(matches) < 1 || len(matches[0]) < 4 { + return "unknown" } - return one, five, fifteen + return fmt.Sprintf("%s %s %s", matches[0][1], matches[0][2], matches[0][3]) } diff --git a/probe/system_linux.go b/probe/system_linux.go index 418aebef40..257abf125e 100644 --- a/probe/system_linux.go +++ b/probe/system_linux.go @@ -1,31 +1,32 @@ package main import ( + "fmt" "io/ioutil" "strconv" "strings" ) -func getLoads() (float64, float64, float64) { +func getLoad() string { buf, err := ioutil.ReadFile("/proc/loadavg") if err != nil { - return -1, -1, -1 + return "unknown" } toks := strings.Fields(string(buf)) if len(toks) < 3 { - return -1, -1, -1 + return "unknown" } one, err := strconv.ParseFloat(toks[0], 64) if err != nil { - return -1, -1, -1 + return "unknown" } five, err := strconv.ParseFloat(toks[1], 64) if err != nil { - return -1, -1, -1 + return "unknown" } fifteen, err := strconv.ParseFloat(toks[2], 64) if err != nil { - return -1, -1, -1 + return "unknown" } - return one, five, fifteen + return fmt.Sprintf("%.2f %.2f %.2f", one, five, fifteen) } diff --git a/probe/tag/docker_tagger.go b/probe/tag/docker_tagger.go new file mode 100644 index 0000000000..adee35a8ff --- /dev/null +++ b/probe/tag/docker_tagger.go @@ -0,0 +1,172 @@ +package tag + +import ( + "log" + "strconv" + "sync" + "time" + + docker "github.com/fsouza/go-dockerclient" + "github.com/weaveworks/scope/report" +) + +var ( + newDockerClient = newRealDockerClient + newPIDTree = newRealPIDTree +) + +func newRealDockerClient(endpoint string) (dockerClient, error) { + return docker.NewClient(endpoint) +} + +// Sub-interface for mocking. +type dockerClient interface { + ListContainers(docker.ListContainersOptions) ([]docker.APIContainers, error) + InspectContainer(string) (*docker.Container, error) + ListImages(docker.ListImagesOptions) ([]docker.APIImages, error) +} + +type dockerTagger struct { + sync.RWMutex + procRoot string + containers map[int]*docker.Container + images map[string]*docker.APIImages + quit chan struct{} +} + +// NewDockerTagger returns a tagger that tags Docker container information to +// nodes with a process_node_id. +func NewDockerTagger(procRoot string, interval time.Duration) Tagger { + t := dockerTagger{ + procRoot: procRoot, + containers: map[int]*docker.Container{}, + images: map[string]*docker.APIImages{}, + quit: make(chan struct{}), + } + t.update() + go t.loop(interval) + return &t +} + +func (t *dockerTagger) Tag(r report.Report, ts report.TopologySelector, id string) report.NodeMetadata { + // Cross-reference the process. + myNodeMetadata, ok := ts(r).NodeMetadatas[id] + if !ok { + //log.Printf("dockerTagger: %q: missing", id) + return report.NodeMetadata{} + } + processNodeID, ok := myNodeMetadata["process_node_id"] + if !ok { + //log.Printf("dockerTagger: %q: no process node ID", id) + return report.NodeMetadata{} + } + processNodeMetadata, ok := r.Process.NodeMetadatas[processNodeID] + if !ok { + //log.Printf("dockerTagger: %q: process node ID missing", id) + return report.NodeMetadata{} + } + pidStr, ok := processNodeMetadata["pid"] + if !ok { + //log.Printf("dockerTagger: %q: process node has no PID", id) + return report.NodeMetadata{} + } + pid, err := strconv.ParseUint(pidStr, 10, 64) + if err != nil { + //log.Printf("dockerTagger: %q: bad process node PID (%v)", id, err) + return report.NodeMetadata{} + } + + t.RLock() + container, ok := t.containers[int(pid)] + t.RUnlock() + + if !ok { + return report.NodeMetadata{} + } + + md := report.NodeMetadata{ + "docker_container_id": container.ID, + "docker_container_name": container.Name, + "docker_image_id": container.Image, + } + + t.RLock() + image, ok := t.images[container.Image] + t.RUnlock() + + if ok && len(image.RepoTags) > 0 { + md["docker_image_name"] = image.RepoTags[0] + } + + return md +} + +func (t *dockerTagger) Stop() { + close(t.quit) +} + +func (t *dockerTagger) loop(d time.Duration) { + for range time.Tick(d) { + t.update() + } +} + +func (t *dockerTagger) update() { + pidTree, err := newPIDTree(t.procRoot) + if err != nil { + log.Printf("docker tagger: %s", err) + return + } + + endpoint := "unix:///var/run/docker.sock" + client, err := newDockerClient(endpoint) + if err != nil { + log.Printf("docker tagger: %s", err) + return + } + + containers, err := client.ListContainers(docker.ListContainersOptions{All: true}) + if err != nil { + log.Printf("docker tagger: %s", err) + return + } + + pmap := map[int]*docker.Container{} + for _, container := range containers { + info, err := client.InspectContainer(container.ID) + if err != nil { + log.Printf("docker tagger: %s", err) + continue + } + + if !info.State.Running { + continue + } + + pids, err := pidTree.allChildren(info.State.Pid) + if err != nil { + log.Printf("docker tagger: %s", err) + continue + } + for _, pid := range pids { + pmap[pid] = info + } + } + + imageList, err := client.ListImages(docker.ListImagesOptions{}) + if err != nil { + log.Printf("docker tagger: %s", err) + return + } + + imageMap := map[string]*docker.APIImages{} + for i := range imageList { + image := &imageList[i] + imageMap[image.ID] = image + } + + t.Lock() + t.containers = pmap + t.images = imageMap + t.Unlock() +} diff --git a/probe/tag/docker_tagger_test.go b/probe/tag/docker_tagger_test.go new file mode 100644 index 0000000000..b4635f4bbd --- /dev/null +++ b/probe/tag/docker_tagger_test.go @@ -0,0 +1,87 @@ +package tag + +import ( + "reflect" + "testing" + "time" + + docker "github.com/fsouza/go-dockerclient" + "github.com/weaveworks/scope/report" +) + +func TestDockerTagger(t *testing.T) { + oldPIDTree, oldDockerClient := newPIDTree, newDockerClient + defer func() { newPIDTree, newDockerClient = oldPIDTree, oldDockerClient }() + + newPIDTree = func(procRoot string) (*pidTree, error) { + pid1 := &process{pid: 1} + pid2 := &process{pid: 2, ppid: 1, parent: pid1} + pid1.children = []*process{pid2} + return &pidTree{ + processes: map[int]*process{ + 1: pid1, 2: pid2, + }, + }, nil + } + + newDockerClient = func(endpoint string) (dockerClient, error) { + return mockDockerClient{ + apiContainers: []docker.APIContainers{{ID: "foo"}}, + containers: map[string]*docker.Container{ + "foo": { + ID: "foo", + Name: "bar", + Image: "baz", + State: docker.State{Pid: 1, Running: true}, + }, + }, + apiImages: []docker.APIImages{{ID: "baz", RepoTags: []string{"bang", "not-chosen"}}}, + }, nil + } + + var ( + endpoint1NodeID = "somehost.com;192.168.1.1;12345" + endpoint2NodeID = "somehost.com;192.168.1.1;67890" + process1NodeID = "somehost.com;1" + process2NodeID = "somehost.com;2" + processNodeMetadata = report.NodeMetadata{ + "docker_container_id": "foo", + "docker_container_name": "bar", + "docker_image_id": "baz", + "docker_image_name": "bang", + } + ) + + r := report.MakeReport() + r.Endpoint.NodeMetadatas[endpoint1NodeID] = report.NodeMetadata{"process_node_id": process1NodeID} + r.Endpoint.NodeMetadatas[endpoint2NodeID] = report.NodeMetadata{"process_node_id": process2NodeID} + r.Process.NodeMetadatas[process1NodeID] = processNodeMetadata.Copy().Merge(report.NodeMetadata{"pid": "1"}) + r.Process.NodeMetadatas[process2NodeID] = processNodeMetadata.Copy().Merge(report.NodeMetadata{"pid": "2"}) + + dockerTagger := NewDockerTagger("/irrelevant", 10*time.Second) + for _, endpointNodeID := range []string{endpoint1NodeID, endpoint2NodeID} { + want := processNodeMetadata.Copy() + have := dockerTagger.Tag(r, report.SelectEndpoint, endpointNodeID).Copy() + if !reflect.DeepEqual(want, have) { + t.Errorf("%q: want %+v, have %+v", endpointNodeID, want, have) + } + } +} + +type mockDockerClient struct { + apiContainers []docker.APIContainers + containers map[string]*docker.Container + apiImages []docker.APIImages +} + +func (m mockDockerClient) ListContainers(docker.ListContainersOptions) ([]docker.APIContainers, error) { + return m.apiContainers, nil +} + +func (m mockDockerClient) InspectContainer(id string) (*docker.Container, error) { + return m.containers[id], nil +} + +func (m mockDockerClient) ListImages(docker.ListImagesOptions) ([]docker.APIImages, error) { + return m.apiImages, nil +} diff --git a/probe/pidtree.go b/probe/tag/pidtree.go similarity index 76% rename from probe/pidtree.go rename to probe/tag/pidtree.go index 65d0f2d61e..d3d61fc5fc 100644 --- a/probe/pidtree.go +++ b/probe/tag/pidtree.go @@ -1,4 +1,4 @@ -package main +package tag import ( "fmt" @@ -18,13 +18,12 @@ type process struct { children []*process } -// Hooks for mocking var ( readDir = ioutil.ReadDir readFile = ioutil.ReadFile ) -func newPIDTree(procRoot string) (*pidTree, error) { +func newRealPIDTree(procRoot string) (*pidTree, error) { dirEntries, err := readDir(procRoot) if err != nil { return nil, err @@ -54,8 +53,7 @@ func newPIDTree(procRoot string) (*pidTree, error) { for _, child := range pt.processes { parent, ok := pt.processes[child.ppid] if !ok { - // This can happen as listing proc is not a consistent snapshot - continue + continue // can happen: listing proc is not a consistent snapshot } child.parent = parent parent.children = append(parent.children, child) @@ -64,16 +62,7 @@ func newPIDTree(procRoot string) (*pidTree, error) { return &pt, nil } -func (pt *pidTree) getParent(pid int) (int, error) { - proc, ok := pt.processes[pid] - if !ok { - return -1, fmt.Errorf("PID %d not found", pid) - } - - return proc.ppid, nil -} - -// allChildren returns a flattened list of child pids including the given pid +// allChildren returns a flat list of child PIDs, including the given PID. func (pt *pidTree) allChildren(pid int) ([]int, error) { proc, ok := pt.processes[pid] if !ok { diff --git a/probe/pidtree_test.go b/probe/tag/pidtree_test.go similarity index 93% rename from probe/pidtree_test.go rename to probe/tag/pidtree_test.go index f83fbd4971..c099a59b48 100644 --- a/probe/pidtree_test.go +++ b/probe/tag/pidtree_test.go @@ -1,4 +1,4 @@ -package main +package tag import ( "fmt" @@ -48,9 +48,9 @@ func TestPIDTree(t *testing.T) { return []byte(fmt.Sprintf("%d na R %d", pid, parent)), nil } - pidtree, err := newPIDTree("/proc") + pidtree, err := newRealPIDTree("/proc") if err != nil { - t.Fatalf("newPIDTree error: %v", err) + t.Fatalf("newRealPIDTree error: %v", err) } for pid, want := range map[int][]int{ diff --git a/probe/tag/tagger.go b/probe/tag/tagger.go new file mode 100644 index 0000000000..867e28df92 --- /dev/null +++ b/probe/tag/tagger.go @@ -0,0 +1,27 @@ +package tag + +import "github.com/weaveworks/scope/report" + +// Tagger tags nodes with value-add node metadata. +type Tagger interface { + Tag(r report.Report, ts report.TopologySelector, id string) report.NodeMetadata + Stop() +} + +// Apply tags all nodes in the report with all taggers. +func Apply(r report.Report, taggers []Tagger) report.Report { + for _, tagger := range taggers { + r.Endpoint = tagTopology(r, report.SelectEndpoint, r.Endpoint, tagger) + r.Address = tagTopology(r, report.SelectAddress, r.Address, tagger) + r.Process = tagTopology(r, report.SelectProcess, r.Process, tagger) + r.Host = tagTopology(r, report.SelectHost, r.Host, tagger) + } + return r +} + +func tagTopology(r report.Report, ts report.TopologySelector, t report.Topology, tagger Tagger) report.Topology { + for nodeID := range t.NodeMetadatas { + t.NodeMetadatas[nodeID] = t.NodeMetadatas[nodeID].Merge(tagger.Tag(r, ts, nodeID)) + } + return t +} diff --git a/probe/tag/tagger_test.go b/probe/tag/tagger_test.go new file mode 100644 index 0000000000..0e1b4a64ac --- /dev/null +++ b/probe/tag/tagger_test.go @@ -0,0 +1,52 @@ +package tag_test + +import ( + "reflect" + "testing" + + "github.com/weaveworks/scope/probe/tag" + "github.com/weaveworks/scope/report" +) + +func TestApply(t *testing.T) { + var ( + endpointNodeID = "a" + addressNodeID = "b" + processNodeID = "c" + hostNodeID = "d" + endpointNodeMetadata = report.NodeMetadata{"1": "2"} + addressNodeMetadata = report.NodeMetadata{"3": "4"} + processNodeMetadata = report.NodeMetadata{"5": "6"} + hostNodeMetadata = report.NodeMetadata{"7": "8"} + ) + + r := report.MakeReport() + r.Endpoint.NodeMetadatas[endpointNodeID] = endpointNodeMetadata + r.Address.NodeMetadatas[addressNodeID] = addressNodeMetadata + r.Process.NodeMetadatas[processNodeID] = processNodeMetadata + r.Host.NodeMetadatas[hostNodeID] = hostNodeMetadata + r = tag.Apply(r, []tag.Tagger{tag.NewTopologyTagger()}) + + for _, tuple := range []struct { + want report.NodeMetadata + from report.Topology + via string + }{ + {copy(endpointNodeMetadata).Merge(report.NodeMetadata{"topology": "endpoint"}), r.Endpoint, endpointNodeID}, + {copy(addressNodeMetadata).Merge(report.NodeMetadata{"topology": "address"}), r.Address, addressNodeID}, + {copy(processNodeMetadata).Merge(report.NodeMetadata{"topology": "process"}), r.Process, processNodeID}, + {copy(hostNodeMetadata).Merge(report.NodeMetadata{"topology": "host"}), r.Host, hostNodeID}, + } { + if want, have := tuple.want, tuple.from.NodeMetadatas[tuple.via]; !reflect.DeepEqual(want, have) { + t.Errorf("want %+v, have %+v", want, have) + } + } +} + +func copy(input report.NodeMetadata) report.NodeMetadata { + output := make(report.NodeMetadata, len(input)) + for k, v := range input { + output[k] = v + } + return output +} diff --git a/probe/tag/topology_tagger.go b/probe/tag/topology_tagger.go new file mode 100644 index 0000000000..58933b6ccf --- /dev/null +++ b/probe/tag/topology_tagger.go @@ -0,0 +1,28 @@ +package tag + +import ( + "github.com/weaveworks/scope/report" +) + +type topologyTagger struct{} + +// NewTopologyTagger tags each node with the topology that it comes from. +func NewTopologyTagger() Tagger { + return &topologyTagger{} +} + +func (topologyTagger) Tag(r report.Report, _ report.TopologySelector, id string) report.NodeMetadata { + for val, ts := range map[string]report.TopologySelector{ + "endpoint": report.SelectEndpoint, + "address": report.SelectAddress, + "process": report.SelectProcess, + "host": report.SelectHost, + } { + if _, ok := ts(r).NodeMetadatas[id]; ok { + return report.NodeMetadata{"topology": val} + } + } + return report.NodeMetadata{} +} + +func (topologyTagger) Stop() {} diff --git a/probe/tag/topology_tagger_test.go b/probe/tag/topology_tagger_test.go new file mode 100644 index 0000000000..0ca502e866 --- /dev/null +++ b/probe/tag/topology_tagger_test.go @@ -0,0 +1,19 @@ +package tag_test + +import ( + "reflect" + "testing" + + "github.com/weaveworks/scope/probe/tag" + "github.com/weaveworks/scope/report" +) + +func TestTagMissingID(t *testing.T) { + const nodeID = "not-found" + r := report.MakeReport() + want := report.NodeMetadata{} + have := tag.NewTopologyTagger().Tag(r, report.SelectAddress, nodeID) + if !reflect.DeepEqual(want, have) { + t.Error("TopologyTagger erroneously tagged a missing node ID") + } +} diff --git a/report/detailed_node.go b/report/detailed_node.go new file mode 100644 index 0000000000..e92cbff9a8 --- /dev/null +++ b/report/detailed_node.go @@ -0,0 +1,79 @@ +package report + +import ( + "reflect" + "strconv" +) + +// DetailedNode is given to the UI when a user clicks on a specific node. It +// contains detailed information about a node in the context of a rendered +// topology. +type DetailedNode struct { + ID string `json:"id"` + LabelMajor string `json:"label_major"` + LabelMinor string `json:"label_minor,omitempty"` + Pseudo bool `json:"pseudo,omitempty"` + Tables []Table `json:"tables"` +} + +// Table is part of a detailed node. +type Table struct { + Title string `json:"title"` // e.g. Bandwidth + Numeric bool `json:"numeric"` // should the major column be right-aligned? + Rows []Row `json:"rows"` +} + +// Row is part of a table. +type Row struct { + Key string `json:"key"` // e.g. Ingress + ValueMajor string `json:"value_major"` // e.g. 25 + ValueMinor string `json:"value_minor,omitempty"` // e.g. KB/s +} + +// MakeDetailedNode transforms a renderable node to a detailed node. It uses +// aggregate metadata, plus the set of origin node IDs, to produce tables. +func MakeDetailedNode(r Report, n RenderableNode) DetailedNode { + tables := []Table{} + { + rows := []Row{} + if val, ok := n.Metadata[KeyMaxConnCountTCP]; ok { + rows = append(rows, Row{"TCP connections", strconv.FormatInt(int64(val), 10), ""}) + } + if val, ok := n.Metadata[KeyBytesIngress]; ok { + rows = append(rows, Row{"Bytes ingress", strconv.FormatInt(int64(val), 10), ""}) + } + if val, ok := n.Metadata[KeyBytesEgress]; ok { + rows = append(rows, Row{"Bytes egress", strconv.FormatInt(int64(val), 10), ""}) + } + if len(rows) > 0 { + tables = append(tables, Table{"Connections", true, rows}) + } + } + + // RenderableNode may be the result of merge operation(s), and so may have + // multiple origins. The ultimate goal here is to generate tables to view + // in the UI, so we skip the intermediate representations, but we could + // add them later. +outer: + for _, id := range n.Origins { + table, ok := r.OriginTable(id) + if !ok { + continue + } + // Naïve equivalence-based deduplication. + for _, existing := range tables { + if reflect.DeepEqual(existing, table) { + continue outer + } + } + tables = append(tables, table) + } + + return DetailedNode{ + ID: n.ID, + LabelMajor: n.LabelMajor, + LabelMinor: n.LabelMinor, + Pseudo: n.Pseudo, + Tables: tables, + } +} diff --git a/report/detailed_node_test.go b/report/detailed_node_test.go new file mode 100644 index 0000000000..fcc3fa57ef --- /dev/null +++ b/report/detailed_node_test.go @@ -0,0 +1,47 @@ +package report_test + +import ( + "reflect" + "testing" + + "github.com/weaveworks/scope/report" +) + +func TestMakeDetailedNode(t *testing.T) { + rendered := report.Render(reportFixture, report.SelectEndpoint, report.ProcessPID, report.BasicPseudoNode) + renderableNodeID := report.MakeProcessNodeID(serverHostID, "215") // what ProcessPID does + renderableNode, ok := rendered[renderableNodeID] + if !ok { + t.Fatalf("couldn't find %q", renderableNodeID) + } + have := report.MakeDetailedNode(reportFixture, renderableNode) + want := report.DetailedNode{ + ID: renderableNodeID, + LabelMajor: "apache", + LabelMinor: "(unknown) (215)", // unknown because we don't put a host_name in the process node metadata + Pseudo: false, + Tables: []report.Table{ + { + Title: "Connections", + Numeric: true, + Rows: []report.Row{ + //{"TCP connections", "0", ""}, + {"Bytes ingress", "310", ""}, + {"Bytes egress", "3100", ""}, + }, + }, + { + Title: "Origin Host", + Numeric: false, + Rows: []report.Row{ + {"Host name", "server.host.com", ""}, + {"Load", "0.01 0.01 0.01", ""}, + {"Operating system", "Linux", ""}, + }, + }, + }, + } + if !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } +} diff --git a/report/diff_test.go b/report/diff_test.go new file mode 100644 index 0000000000..88d8640cce --- /dev/null +++ b/report/diff_test.go @@ -0,0 +1,21 @@ +package report_test + +import ( + "github.com/davecgh/go-spew/spew" + "github.com/pmezard/go-difflib/difflib" +) + +func init() { + spew.Config.SortKeys = true // :\ +} + +func diff(want, have interface{}) string { + text, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ + A: difflib.SplitLines(spew.Sdump(want)), + B: difflib.SplitLines(spew.Sdump(have)), + FromFile: "want", + ToFile: "have", + Context: 5, + }) + return "\n" + text +} diff --git a/report/id.go b/report/id.go deleted file mode 100644 index 20f48c2bec..0000000000 --- a/report/id.go +++ /dev/null @@ -1,29 +0,0 @@ -package report - -import ( - "net" - "strings" -) - -// IDAddresser is used to get the IP address from an addressID. Or nil. -type IDAddresser func(string) net.IP - -// AddressIPPort translates "scope;ip;port" to the IP address. These are used -// by Process topologies. -func AddressIPPort(id string) net.IP { - parts := strings.SplitN(id, ScopeDelim, 3) - if len(parts) != 3 { - return nil // hmm - } - return net.ParseIP(parts[1]) -} - -// AddressIP translates "scope;ip" to the IP address. These are used by -// Network topologies. -func AddressIP(id string) net.IP { - parts := strings.SplitN(id, ScopeDelim, 2) - if len(parts) != 2 { - return nil // hmm - } - return net.ParseIP(parts[1]) -} diff --git a/report/idlist.go b/report/id_list.go similarity index 59% rename from report/idlist.go rename to report/id_list.go index 7069095ab8..a7443ce920 100644 --- a/report/idlist.go +++ b/report/id_list.go @@ -1,19 +1,25 @@ package report -import ( - "sort" -) +import "sort" -// IDList is a list of string IDs, which are always sorted and -// without duplicates. +// IDList is a list of string IDs, which are always sorted and unique. type IDList []string -// NewIDList makes a new IDList. -func NewIDList(ids ...string) IDList { +// MakeIDList makes a new IDList. +func MakeIDList(ids ...string) IDList { sort.Strings(ids) return IDList(ids) } +// Copy returns a value copy, useful for tests. +func (a IDList) Copy() IDList { + cp := make(IDList, len(a)) + for i, s := range a { + cp[i] = s + } + return cp +} + // Add is the only correct way to add ids to an IDList. func (a IDList) Add(ids ...string) IDList { for _, s := range ids { diff --git a/report/id_list_test.go b/report/id_list_test.go new file mode 100644 index 0000000000..e0fce067dd --- /dev/null +++ b/report/id_list_test.go @@ -0,0 +1,31 @@ +package report_test + +import ( + "reflect" + "testing" + + "github.com/weaveworks/scope/report" +) + +func TestIDList(t *testing.T) { + have := report.MakeIDList("alpha", "mu", "zeta") + have = have.Add("alpha") + have = have.Add("nu") + have = have.Add("mu") + have = have.Add("alpha") + have = have.Add("alpha") + have = have.Add("epsilon") + have = have.Add("delta") + if want := report.IDList([]string{"alpha", "delta", "epsilon", "mu", "nu", "zeta"}); !reflect.DeepEqual(want, have) { + t.Errorf("want %+v, have %+v", want, have) + } +} + +func TestIDListCopy(t *testing.T) { + one := report.MakeIDList("a", "b", "c") + two := one.Copy() + one.Add("d") + if want, have := report.MakeIDList("a", "b", "c"), two; !reflect.DeepEqual(want, have) { + t.Errorf("want %+v, have %+v", want, have) + } +} diff --git a/report/idlist_test.go b/report/idlist_test.go deleted file mode 100644 index d781d3e2b6..0000000000 --- a/report/idlist_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package report - -import ( - "reflect" - "testing" -) - -func TestidList(t *testing.T) { - var have IDList - have = have.Add("aap") - have = have.Add("noot") - have = have.Add("mies") - have = have.Add("aap") - have = have.Add("aap") - have = have.Add("wim") - have = have.Add("vuur") - - if want := IDList([]string{"aap", "mies", "noot", "vuur", "wim"}); !reflect.DeepEqual(want, have) { - t.Errorf("want %+v, have %+v", want, have) - } -} diff --git a/report/ids.go b/report/ids.go new file mode 100644 index 0000000000..2d284e936a --- /dev/null +++ b/report/ids.go @@ -0,0 +1,117 @@ +package report + +import ( + "fmt" + "net" + "strings" +) + +// TheInternet is used as a node ID to indicate a remote IP. +const TheInternet = "theinternet" + +// Delimiters are used to separate parts of node IDs, to guarantee uniqueness +// in particular contexts. +const ( + // ScopeDelim is a general-purpose delimiter used within node IDs to + // separate different contextual scopes. Different topologies have + // different key structures. + ScopeDelim = ";" + + // EdgeDelim separates two node IDs when they need to exist in the same key. + // Concretely, it separates node IDs in keys that represent edges. + EdgeDelim = "|" +) + +// MakeAdjacencyID produces an adjacency ID from composite parts. +func MakeAdjacencyID(hostID, srcNodeID string) string { + // Here we rely on the fact that every possible source node ID has the + // host ID as the first scope-delimited field, and therefore don't + // duplicate that information. + return ">" + srcNodeID +} + +// ParseAdjacencyID splits an adjacency ID to its composite parts. +func ParseAdjacencyID(adjacencyID string) (hostID, srcNodeID string, ok bool) { + if !strings.HasPrefix(adjacencyID, ">") { + return "", "", false + } + // This relies on every node ID having hostID as its first scoped field. + adjacencyID = adjacencyID[1:] + fields := strings.SplitN(adjacencyID, ScopeDelim, 2) + if len(fields) != 2 { + return "", "", false + } + return fields[0], adjacencyID, true +} + +// MakeEdgeID produces an edge ID from composite parts. +func MakeEdgeID(srcNodeID, dstNodeID string) string { + return srcNodeID + EdgeDelim + dstNodeID +} + +// ParseEdgeID splits an edge ID to its composite parts. +func ParseEdgeID(edgeID string) (srcNodeID, dstNodeID string, ok bool) { + fields := strings.SplitN(edgeID, EdgeDelim, 2) + if len(fields) != 2 { + return "", "", false + } + return fields[0], fields[1], true +} + +// MakeEndpointNodeID produces an endpoint node ID from its composite parts. +func MakeEndpointNodeID(hostID, address, port string) string { + return MakeAddressNodeID(hostID, address) + ScopeDelim + port +} + +// MakeAddressNodeID produces an address node ID from its composite parts. +func MakeAddressNodeID(hostID, address string) string { + return hostID + ScopeDelim + address +} + +// MakeProcessNodeID produces a process node ID from its composite parts. +func MakeProcessNodeID(hostID, pid string) string { + return hostID + ScopeDelim + pid +} + +// MakeHostNodeID produces a host node ID from its composite parts. +func MakeHostNodeID(hostID string) string { + // hostIDs come from the probe and are presumed to be globally-unique. + // But, suffix something to elicit failures if we try to use probe host + // IDs directly as node IDs in the host topology. + return hostID + ScopeDelim + "" +} + +// MakePseudoNodeID produces a pseudo node ID from its composite parts. +func MakePseudoNodeID(parts ...string) string { + return strings.Join(append([]string{"pseudo"}, parts...), ScopeDelim) +} + +// IDAddresser tries to convert a node ID to a net.IP, if possible. +type IDAddresser func(string) net.IP + +// EndpointIDAddresser converts an endpoint node ID to an IP. +func EndpointIDAddresser(id string) net.IP { + fields := strings.SplitN(id, ScopeDelim, 3) + if len(fields) != 3 { + //log.Printf("EndpointIDAddresser: bad input %q", id) + return nil + } + return net.ParseIP(fields[1]) +} + +// AddressIDAddresser converts an address node ID to an IP. +func AddressIDAddresser(id string) net.IP { + fields := strings.SplitN(id, ScopeDelim, 2) + if len(fields) != 2 { + //log.Printf("AddressIDAddresser: bad input %q", id) + return nil + } + return net.ParseIP(fields[1]) +} + +// PanicIDAddresser will panic if it's ever called. It's used in topologies +// where there are never and edges, and so it's nonsensical to try and extract +// IPs from the node IDs. +func PanicIDAddresser(id string) net.IP { + panic(fmt.Sprintf("PanicIDAddresser called on %q", id)) +} diff --git a/report/ids_test.go b/report/ids_test.go new file mode 100644 index 0000000000..ab8a10e14b --- /dev/null +++ b/report/ids_test.go @@ -0,0 +1,147 @@ +package report_test + +import ( + "net" + "reflect" + "testing" + + "github.com/weaveworks/scope/report" +) + +func TestAdjacencyID(t *testing.T) { + for _, bad := range []string{ + client54001EndpointNodeID, + client54002EndpointNodeID, + unknown1EndpointNodeID, + unknown2EndpointNodeID, + unknown3EndpointNodeID, + clientAddressNodeID, + serverAddressNodeID, + unknownAddressNodeID, + clientHostNodeID, + serverHostNodeID, + ">1.2.3.4", + ">", + ";", + "", + } { + if hostID, srcNodeID, ok := report.ParseAdjacencyID(bad); ok { + t.Errorf("%q: expected failure, but got (%q, %q)", bad, hostID, srcNodeID) + } + } + + for input, want := range map[string]struct{ hostID, srcNodeID string }{ + report.MakeAdjacencyID("a", report.MakeEndpointNodeID("a", "b", "c")): {"a", report.MakeEndpointNodeID("a", "b", "c")}, + report.MakeAdjacencyID("a", report.MakeAddressNodeID("a", "b")): {"a", report.MakeAddressNodeID("a", "b")}, + report.MakeAdjacencyID("a", report.MakeProcessNodeID("a", "b")): {"a", report.MakeProcessNodeID("a", "b")}, + report.MakeAdjacencyID("a", report.MakeHostNodeID("a")): {"a", report.MakeHostNodeID("a")}, + ">host.com;1.2.3.4": {"host.com", "host.com;1.2.3.4"}, + ">a;b;c": {"a", "a;b;c"}, + ">a;b": {"a", "a;b"}, + ">a;": {"a", "a;"}, + ">;b": {"", ";b"}, + ">;": {"", ";"}, + } { + hostID, srcNodeID, ok := report.ParseAdjacencyID(input) + if !ok { + t.Errorf("%q: not OK", input) + continue + } + if want, have := want.hostID, hostID; want != have { + t.Errorf("%q: want %q, have %q", input, want, have) + } + if want, have := want.srcNodeID, srcNodeID; want != have { + t.Errorf("%q: want %q, have %q", input, want, have) + } + } +} + +func TestEdgeID(t *testing.T) { + for _, bad := range []string{ + client54001EndpointNodeID, + client54002EndpointNodeID, + unknown1EndpointNodeID, + unknown2EndpointNodeID, + unknown3EndpointNodeID, + clientAddressNodeID, + serverAddressNodeID, + unknownAddressNodeID, + clientHostNodeID, + serverHostNodeID, + ">1.2.3.4", + ">", + ";", + "", + } { + if srcNodeID, dstNodeID, ok := report.ParseEdgeID(bad); ok { + t.Errorf("%q: expected failure, but got (%q, %q)", bad, srcNodeID, dstNodeID) + } + } + + for input, want := range map[string]struct{ srcNodeID, dstNodeID string }{ + report.MakeEdgeID("a", report.MakeEndpointNodeID("a", "b", "c")): {"a", report.MakeEndpointNodeID("a", "b", "c")}, + report.MakeEdgeID("a", report.MakeAddressNodeID("a", "b")): {"a", report.MakeAddressNodeID("a", "b")}, + report.MakeEdgeID("a", report.MakeProcessNodeID("a", "b")): {"a", report.MakeProcessNodeID("a", "b")}, + report.MakeEdgeID("a", report.MakeHostNodeID("a")): {"a", report.MakeHostNodeID("a")}, + "host.com|1.2.3.4": {"host.com", "1.2.3.4"}, + "a|b;c": {"a", "b;c"}, + "a|b": {"a", "b"}, + "a|": {"a", ""}, + "|b": {"", "b"}, + "|": {"", ""}, + } { + srcNodeID, dstNodeID, ok := report.ParseEdgeID(input) + if !ok { + t.Errorf("%q: not OK", input) + continue + } + if want, have := want.srcNodeID, srcNodeID; want != have { + t.Errorf("%q: want %q, have %q", input, want, have) + } + if want, have := want.dstNodeID, dstNodeID; want != have { + t.Errorf("%q: want %q, have %q", input, want, have) + } + } +} + +func TestEndpointIDAddresser(t *testing.T) { + if nodeID := "1.2.4.5"; report.EndpointIDAddresser(nodeID) != nil { + t.Errorf("%q: bad node ID parsed as good", nodeID) + } + var ( + nodeID = report.MakeEndpointNodeID(clientHostID, clientAddress, "12345") + want = net.ParseIP(clientAddress) + have = report.EndpointIDAddresser(nodeID) + ) + if !reflect.DeepEqual(want, have) { + t.Errorf("want %s, have %s", want, have) + } +} + +func TestAddressIDAddresser(t *testing.T) { + if nodeID := "1.2.4.5"; report.AddressIDAddresser(nodeID) != nil { + t.Errorf("%q: bad node ID parsed as good", nodeID) + } + var ( + nodeID = report.MakeAddressNodeID(clientHostID, clientAddress) + want = net.ParseIP(clientAddress) + have = report.AddressIDAddresser(nodeID) + ) + if !reflect.DeepEqual(want, have) { + t.Errorf("want %s, have %s", want, have) + } +} + +func TestPanicIDAddresser(t *testing.T) { + if panicked := func() (recovered bool) { + defer func() { + if r := recover(); r != nil { + recovered = true + } + }() + report.PanicIDAddresser("irrelevant") + return false + }(); !panicked { + t.Errorf("expected panic, didn't get it") + } +} diff --git a/report/mapping.go b/report/mapping.go new file mode 100644 index 0000000000..17c0f2d319 --- /dev/null +++ b/report/mapping.go @@ -0,0 +1,258 @@ +package report + +import ( + "fmt" + "net" + "strings" +) + +// MapFunc deterministically maps nodes in a report to some other domain. +type MapFunc func(r Report, ts TopologySelector, nodeID string) (MappedNode, bool) + +// PseudoFunc creates MappedNodes representing pseudo nodes given the node ID. +// The node ID is prior to mapping. +type PseudoFunc func(nodeID string) MappedNode + +// TopologySelector chooses a topology from a report. +type TopologySelector func(Report) Topology + +// SelectEndpoint selects the endpoint topology. +func SelectEndpoint(r Report) Topology { return r.Endpoint } + +// SelectAddress selects the address topology. +func SelectAddress(r Report) Topology { return r.Address } + +// SelectProcess selects the process topology. +func SelectProcess(r Report) Topology { return r.Process } + +// SelectHost selects the host topology. +func SelectHost(r Report) Topology { return r.Host } + +// MappedNode is an intermediate form, produced by MapFunc and PseudoFunc. It +// represents a node from a report, after that node's been passed through a +// mapping transformation. Multiple report nodes may map to the same mapped +// node. +type MappedNode struct { + ID string + Major string + Minor string + Rank string +} + +// ProcessPID is a MapFunc that maps all nodes to their origin process PID. +// That is, all nodes with the same process PID (on the same host) will be +// mapped together. Nodes without processes are not mapped. +func ProcessPID(r Report, ts TopologySelector, nodeID string) (MappedNode, bool) { + md, ok := ts(r).NodeMetadatas[nodeID] + if !ok { + return MappedNode{}, false // programmer error + } + processNodeID, ok := md["process_node_id"] + if !ok { + return MappedNode{}, false // process not available + } + md, ok = r.Process.NodeMetadatas[processNodeID] + if !ok { + return MappedNode{}, false // programmer error + } + var ( + processName = md.GetDefault("process_name", "(unknown)") + hostName = md.GetDefault("host_name", "(unknown)") + processPID = md.GetDefault("pid", "?") + ) + return MappedNode{ + ID: processNodeID, + Major: processName, + Minor: fmt.Sprintf("%s (%s)", hostName, processPID), + Rank: hostName, + }, true +} + +// ProcessName is a MapFunc that maps all nodes to their origin process name. +// That is, all nodes with the same process name (independent of host) will be +// mapped together. Nodes without processes are not mapped. +func ProcessName(r Report, ts TopologySelector, nodeID string) (MappedNode, bool) { + md, ok := ts(r).NodeMetadatas[nodeID] + if !ok { + return MappedNode{}, false // programmer error + } + processNodeID, ok := md["process_node_id"] + if !ok { + return MappedNode{}, false // process not available + } + md, ok = r.Process.NodeMetadatas[processNodeID] + if !ok { + return MappedNode{}, false // programmer error + } + processName, ok := md["process_name"] + if !ok { + return MappedNode{}, false + } + return MappedNode{ + ID: processName, + Major: processName, + Minor: md.GetDefault("host_name", "(unknown)"), + Rank: processName, + }, true +} + +// ProcessContainer is a MapFunc that maps all nodes to their origin container +// ID. That is, all nodes running in the same Docker container ID (independent +// of host) will be mapped together. Nodes without containers are not mapped. +func ProcessContainer(r Report, ts TopologySelector, nodeID string) (MappedNode, bool) { + md, ok := ts(r).NodeMetadatas[nodeID] + if !ok { + return MappedNode{}, false // programmer error + } + processNodeID, ok := md["process_node_id"] + if !ok { + return MappedNode{}, false // process not available + } + md, ok = r.Process.NodeMetadatas[processNodeID] + if !ok { + return MappedNode{}, false // programmer error + } + dockerContainerID, ok := md["docker_container_id"] + if !ok { + return MappedNode{ + ID: "uncontained", + Major: "Uncontained", + Minor: "", + Rank: "uncontained", + }, true + } + var ( + dockerContainerName = md.GetDefault("docker_container_name", "(unknown)") + hostName = md.GetDefault("host_name", "(unknown)") + dockerImageID = md.GetDefault("docker_image_id", "unknown") + ) + return MappedNode{ + ID: dockerContainerID, + Major: dockerContainerName, + Minor: hostName, + Rank: dockerImageID, + }, true +} + +// ProcessContainerImage is a MapFunc that maps all nodes to their origin +// container image ID. That is, all nodes running from the same Docker image +// ID (independent of host) will be mapped together. Nodes without containers +// are not mapped. +func ProcessContainerImage(r Report, ts TopologySelector, nodeID string) (MappedNode, bool) { + md, ok := ts(r).NodeMetadatas[nodeID] + if !ok { + return MappedNode{}, false // programmer error + } + processNodeID, ok := md["process_node_id"] + if !ok { + return MappedNode{}, false // process not available + } + md, ok = r.Process.NodeMetadatas[processNodeID] + if !ok { + return MappedNode{}, false // programmer error + } + if _, ok := md["docker_container_id"]; !ok { + return MappedNode{ + ID: "uncontained", + Major: "Uncontained", + Minor: "", + Rank: "uncontained", + }, true + } + var ( + dockerImageID = md.GetDefault("docker_image_id", "unknown") + dockerImageName = md.GetDefault("docker_image_name", "unknown") + ) + return MappedNode{ + ID: dockerImageID, + Major: dockerImageName, + Minor: "", + Rank: dockerImageID, + }, true +} + +// AddressHostname is a MapFunc that maps all nodes to their origin host +// (hostname). That is, all nodes pulled from the same host will be mapped +// together. Nodes without information about origin host (via the address +// topology) are not mapped. +func AddressHostname(r Report, ts TopologySelector, nodeID string) (MappedNode, bool) { + md, ok := ts(r).NodeMetadatas[nodeID] + if !ok { + return MappedNode{}, false // programmer error + } + addressNodeID, ok := md["address_node_id"] + if !ok { + return MappedNode{}, false // process not available + } + md, ok = r.Address.NodeMetadatas[addressNodeID] + if !ok { + return MappedNode{}, false // programmer error + } + hostName := md.GetDefault("host_name", "(unknown)") + major, minor := hostName, "" + if fields := strings.SplitN(hostName, ".", 2); len(fields) == 2 { + major, minor = fields[0], fields[1] + } + return MappedNode{ + ID: addressNodeID, + Major: major, + Minor: minor, + Rank: addressNodeID, + }, true +} + +// BasicPseudoNode is a PseudoFunc that grants each node ID its own pseudo +// node. It's effectively an identity, or one-to-one, mapping. +func BasicPseudoNode(nodeID string) MappedNode { + if nodeID == TheInternet { + return MappedNode{ + ID: TheInternet, + Major: formatLabel(TheInternet), + Minor: "", + Rank: TheInternet, + } + } + return MappedNode{ + ID: MakePseudoNodeID(nodeID), + Major: formatLabel(nodeID), + Minor: "", + Rank: MakePseudoNodeID(nodeID), + } +} + +// GroupedPseudoNode is a PseudoFunc that maps every node ID to the same +// pseudo node. It's effectively a many-to-one mapping. +func GroupedPseudoNode(nodeID string) MappedNode { + if nodeID == TheInternet { + return MappedNode{ + ID: TheInternet, + Major: formatLabel(TheInternet), + Minor: "", + Rank: TheInternet, + } + } + return MappedNode{ + ID: MakePseudoNodeID("unknown"), + Major: "Unknown", + Minor: "", + Rank: MakePseudoNodeID("unknown"), + } +} + +// NoPseudoNode is (effectively) a PseudoFunc that suppresses the generation +// of all pseudo nodes. +var NoPseudoNode PseudoFunc // = nil + +func formatLabel(nodeID string) string { + if nodeID == TheInternet { + return "the Internet" + } + switch fields := strings.SplitN(nodeID, ScopeDelim, 3); { + case len(fields) < 2: + return nodeID + case len(fields) == 2: + return fields[1] + default: + return net.JoinHostPort(fields[1], fields[2]) + } +} diff --git a/report/mapping_functions.go b/report/mapping_functions.go deleted file mode 100644 index f739edcba3..0000000000 --- a/report/mapping_functions.go +++ /dev/null @@ -1,189 +0,0 @@ -package report - -import ( - "fmt" - "strings" -) - -const humanTheInternet = "the Internet" - -// MappedNode is returned by the MapFuncs. -type MappedNode struct { - ID string - Major string - Minor string - Rank string -} - -// MapFunc is anything which can take an arbitrary NodeMetadata, which is -// always one-to-one with nodes in a topology, and return a specific -// representation of the referenced node, in the form of a node ID and a -// human-readable major and minor labels. -// -// A single NodeMetadata can yield arbitrary many representations, including -// representations that reduce the cardinality of the set of nodes. -// -// If the final output parameter is false, the node shall be omitted from the -// rendered topology. -type MapFunc func(string, NodeMetadata) (MappedNode, bool) - -// PseudoFunc creates MappedNode representing pseudo nodes given the dstNodeID. -// The srcNode renderable node is essentially from MapFunc, representing one of -// the rendered nodes this pseudo node refers to. srcNodeID and dstNodeID are -// node IDs prior to mapping. -type PseudoFunc func(srcNodeID string, srcNode RenderableNode, dstNodeID string) (MappedNode, bool) - -// ProcessPID takes a node NodeMetadata from a Process topology, and returns a -// representation with the ID based on the process PID and the labels based -// on the process name. -func ProcessPID(_ string, m NodeMetadata) (MappedNode, bool) { - var ( - identifier = fmt.Sprintf("%s:%s:%s", "pid", m["domain"], m["pid"]) - minor = fmt.Sprintf("%s (%s)", m["domain"], m["pid"]) - show = m["pid"] != "" && m["name"] != "" - ) - - return MappedNode{ - ID: identifier, - Major: m["name"], - Minor: minor, - Rank: m["pid"], - }, show -} - -// ProcessName takes a node NodeMetadata from a Process topology, and returns a -// representation with the ID based on the process name (grouping all processes with -// the same name together). -func ProcessName(_ string, m NodeMetadata) (MappedNode, bool) { - show := m["pid"] != "" && m["name"] != "" - return MappedNode{ - ID: m["name"], - Major: m["name"], - Minor: "", - Rank: m["name"], - }, show -} - -// ProcessContainer maps Process topology nodes to the containers they run in. -// We consider container and image IDs to be globally unique, and so don't -// scope them further by e.g. host. If no container metadata is found, nodes -// are grouped into the Uncontained node. -func ProcessContainer(_ string, m NodeMetadata) (MappedNode, bool) { - var id, major, minor, rank string - if m["docker_id"] == "" { - id, major, minor, rank = "uncontained", "Uncontained", "", "uncontained" - } else { - id, major, minor, rank = m["docker_id"], m["docker_name"], m["domain"], m["docker_image_id"] - } - - return MappedNode{ - ID: id, - Major: major, - Minor: minor, - Rank: rank, - }, true -} - -// ProcessContainerImage maps Process topology nodes to the container images they run on. -// If no container metadata is found, nodes are grouped into the Uncontained node. -func ProcessContainerImage(_ string, m NodeMetadata) (MappedNode, bool) { - var id, major, minor, rank string - if m["docker_image_id"] == "" { - id, major, minor, rank = "uncontained", "Uncontained", "", "uncontained" - } else { - id, major, minor, rank = m["docker_image_id"], m["docker_image_name"], "", m["docker_image_id"] - } - - return MappedNode{ - ID: id, - Major: major, - Minor: minor, - Rank: rank, - }, true -} - -// NetworkHostname takes a node NodeMetadata from a Network topology, and -// returns a representation based on the hostname. Major label is the -// hostname, the minor label is the domain, if any. -func NetworkHostname(_ string, m NodeMetadata) (MappedNode, bool) { - var ( - name = m["name"] - domain = "" - parts = strings.SplitN(name, ".", 2) - ) - - if len(parts) == 2 { - domain = parts[1] - } - - return MappedNode{ - ID: fmt.Sprintf("host:%s", name), - Major: parts[0], - Minor: domain, - Rank: parts[0], - }, name != "" -} - -// GenericPseudoNode contains heuristics for building sensible pseudo nodes. -// It should go away. -func GenericPseudoNode(src string, srcMapped RenderableNode, dst string) (MappedNode, bool) { - var maj, min, outputID string - - if dst == TheInternet { - outputID = dst - maj, min = humanTheInternet, "" - } else { - // Rule for non-internet psuedo nodes; emit 1 new node for each - // dstNodeAddr, srcNodeAddr, srcNodePort. - srcNodeAddr, srcNodePort := trySplitAddr(src) - dstNodeAddr, _ := trySplitAddr(dst) - - outputID = strings.Join([]string{"pseudo:", dstNodeAddr, srcNodeAddr, srcNodePort}, ScopeDelim) - maj, min = dstNodeAddr, "" - } - - return MappedNode{ - ID: outputID, - Major: maj, - Minor: min, - }, true -} - -// GenericGroupedPseudoNode contains heuristics for building sensible pseudo nodes. -// It should go away. -func GenericGroupedPseudoNode(src string, srcMapped RenderableNode, dst string) (MappedNode, bool) { - var maj, min, outputID string - - if dst == TheInternet { - outputID = dst - maj, min = humanTheInternet, "" - } else { - // When grouping, emit one pseudo node per (srcNodeAddress, dstNodeAddr) - dstNodeAddr, _ := trySplitAddr(dst) - - outputID = strings.Join([]string{"pseudo:", dstNodeAddr, srcMapped.ID}, ScopeDelim) - maj, min = dstNodeAddr, "" - } - - return MappedNode{ - ID: outputID, - Major: maj, - Minor: min, - }, true -} - -// InternetOnlyPseudoNode never creates a pseudo node, unless it's the Internet. -func InternetOnlyPseudoNode(_ string, _ RenderableNode, dst string) (MappedNode, bool) { - if dst == TheInternet { - return MappedNode{ID: TheInternet, Major: humanTheInternet}, true - } - return MappedNode{}, false -} - -func trySplitAddr(addr string) (string, string) { - fields := strings.SplitN(addr, ScopeDelim, 3) - if len(fields) == 3 { - return fields[1], fields[2] - } - return fields[1], "" -} diff --git a/report/mapping_test.go b/report/mapping_test.go index 79a1eabbee..4815f6d027 100644 --- a/report/mapping_test.go +++ b/report/mapping_test.go @@ -1,110 +1,205 @@ -package report +package report_test import ( - "fmt" + "reflect" "testing" + + "github.com/weaveworks/scope/report" ) -func TestUngroupedMapping(t *testing.T) { - for i, c := range []struct { - f MapFunc - id string - meta NodeMetadata - wantOK bool - wantID, wantMajor, wantMinor, wantRank string +func TestSelectors(t *testing.T) { + for _, tuple := range []struct { + want report.Topology + selector report.TopologySelector }{ - { - f: NetworkHostname, - id: ScopeDelim + "1.2.3.4", - meta: NodeMetadata{ - "name": "my.host", - }, - wantOK: true, - wantID: "host:my.host", - wantMajor: "my", - wantMinor: "host", - wantRank: "my", - }, - { - f: NetworkHostname, - id: ScopeDelim + "1.2.3.4", - meta: NodeMetadata{ - "name": "localhost", - }, - wantOK: true, - wantID: "host:localhost", - wantMajor: "localhost", - wantMinor: "", - wantRank: "localhost", - }, - { - f: ProcessPID, - id: "not-used-beta", - meta: NodeMetadata{ - "pid": "42", - "name": "curl", - "domain": "hosta", - }, - wantOK: true, - wantID: "pid:hosta:42", - wantMajor: "curl", - wantMinor: "hosta (42)", - wantRank: "42", - }, - { - f: ProcessContainer, - id: "foo-id", - meta: NodeMetadata{ - "pid": "42", - "name": "curl", - "domain": "hosta", - }, - wantOK: true, - wantID: "uncontained", - wantMajor: "Uncontained", - wantMinor: "", - wantRank: "uncontained", - }, - { - f: ProcessContainer, - id: "bar-id", - meta: NodeMetadata{ - "pid": "42", - "name": "curl", - "domain": "hosta", - "docker_id": "d321fe0", - "docker_name": "walking_sparrow", - "docker_image_id": "1101fff", - "docker_image_name": "org/app:latest", - }, - wantOK: true, - wantID: "d321fe0", - wantMajor: "walking_sparrow", - wantMinor: "hosta", - wantRank: "1101fff", - }, + {reportFixture.Endpoint, report.SelectEndpoint}, + {reportFixture.Address, report.SelectAddress}, + {reportFixture.Process, report.SelectProcess}, + {reportFixture.Host, report.SelectHost}, } { - identity := fmt.Sprintf("(%d %s %v)", i, c.id, c.meta) - - m, haveOK := c.f(c.id, c.meta) - if want, have := c.wantOK, haveOK; want != have { - t.Errorf("%s: map OK error: want %v, have %v", identity, want, have) - } - if want, have := c.wantID, m.ID; want != have { - t.Errorf("%s: map ID error: want %#v, have %#v", identity, want, have) - } - if want, have := c.wantMajor, m.Major; want != have { - t.Errorf("%s: map major label: want %#v, have %#v", identity, want, have) + if want, have := tuple.want, tuple.selector(reportFixture); !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) } - if want, have := c.wantMinor, m.Minor; want != have { - t.Errorf("%s: map minor label: want %#v, have %#v", identity, want, have) + } +} + +func TestProcessPID(t *testing.T) { + if _, ok := report.ProcessPID(reportFixture, report.SelectEndpoint, "invalid-node-ID"); ok { + t.Errorf("want %v, have %v", false, true) + } + if _, ok := report.ProcessPID(reportFixture, report.SelectEndpoint, "process-not-available"); ok { + t.Errorf("want %v, have %v", false, true) + } + if _, ok := report.ProcessPID(reportFixture, report.SelectEndpoint, "process-badly-linked"); ok { + t.Errorf("want %v, have %v", false, true) + } + + want := report.MappedNode{ + ID: report.MakeProcessNodeID(clientHostID, "4242"), + Major: "curl", + Minor: "client.host.com (4242)", + Rank: "client.host.com", + } + have, ok := report.ProcessPID(reportFixture, report.SelectEndpoint, client54001EndpointNodeID) + if !ok { + t.Fatalf("want %v, have %v", true, false) + } + if !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } +} + +func TestProcessName(t *testing.T) { + if _, ok := report.ProcessName(reportFixture, report.SelectEndpoint, "invalid-node-ID"); ok { + t.Errorf("want %v, have %v", false, true) + } + if _, ok := report.ProcessName(reportFixture, report.SelectEndpoint, "process-not-available"); ok { + t.Errorf("want %v, have %v", false, true) + } + if _, ok := report.ProcessName(reportFixture, report.SelectEndpoint, "process-badly-linked"); ok { + t.Errorf("want %v, have %v", false, true) + } + + want := report.MappedNode{ + ID: "curl", + Major: "curl", + Minor: "client.host.com", + Rank: "curl", + } + have, ok := report.ProcessName(reportFixture, report.SelectEndpoint, client54001EndpointNodeID) + if !ok { + t.Fatalf("want %v, have %v", true, false) + } + if !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } +} + +func TestProcessContainer(t *testing.T) { + if _, ok := report.ProcessContainer(reportFixture, report.SelectEndpoint, "invalid-node-ID"); ok { + t.Errorf("want %v, have %v", false, true) + } + if _, ok := report.ProcessContainer(reportFixture, report.SelectEndpoint, "process-not-available"); ok { + t.Errorf("want %v, have %v", false, true) + } + if _, ok := report.ProcessContainer(reportFixture, report.SelectEndpoint, "process-badly-linked"); ok { + t.Errorf("want %v, have %v", false, true) + } + + have, ok := report.ProcessContainer(reportFixture, report.SelectEndpoint, "process-no-container") + if !ok { + t.Fatalf("want %v, have %v", true, false) + } + if want := uncontained; !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } + + have, ok = report.ProcessContainer(reportFixture, report.SelectEndpoint, client54001EndpointNodeID) + if !ok { + t.Fatalf("want %v, have %v", true, false) + } + if want := (report.MappedNode{ + ID: "a1b2c3d4e5", + Major: "fixture-container", + Minor: "client.host.com", + Rank: "0000000000", + }); !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } +} + +func TestProcessContainerImage(t *testing.T) { + if _, ok := report.ProcessContainerImage(reportFixture, report.SelectEndpoint, "invalid-node-ID"); ok { + t.Errorf("want %v, have %v", false, true) + } + if _, ok := report.ProcessContainerImage(reportFixture, report.SelectEndpoint, "process-not-available"); ok { + t.Errorf("want %v, have %v", false, true) + } + if _, ok := report.ProcessContainerImage(reportFixture, report.SelectEndpoint, "process-badly-linked"); ok { + t.Errorf("want %v, have %v", false, true) + } + + have, ok := report.ProcessContainerImage(reportFixture, report.SelectEndpoint, "process-no-container") + if !ok { + t.Fatalf("want %v, have %v", true, false) + } + if want := uncontained; !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } + + have, ok = report.ProcessContainerImage(reportFixture, report.SelectEndpoint, client54001EndpointNodeID) + if !ok { + t.Fatalf("want %v, have %v", true, false) + } + if want := (report.MappedNode{ + ID: "0000000000", + Major: "fixture/container:latest", + Minor: "", + Rank: "0000000000", + }); !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } +} + +func TestAddressHostname(t *testing.T) { + if _, ok := report.AddressHostname(reportFixture, report.SelectEndpoint, "invalid-node-ID"); ok { + t.Errorf("want %v, have %v", false, true) + } + if _, ok := report.AddressHostname(reportFixture, report.SelectEndpoint, "address-not-available"); ok { + t.Errorf("want %v, have %v", false, true) + } + if _, ok := report.AddressHostname(reportFixture, report.SelectEndpoint, "address-badly-linked"); ok { + t.Errorf("want %v, have %v", false, true) + } + + want := report.MappedNode{ + ID: report.MakeAddressNodeID(clientHostID, clientAddress), + Major: "client", + Minor: "host.com", + Rank: report.MakeAddressNodeID(clientHostID, clientAddress), + } + have, ok := report.AddressHostname(reportFixture, report.SelectEndpoint, client54001EndpointNodeID) + if !ok { + t.Fatalf("want %v, have %v", true, false) + } + if !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } +} + +func TestBasicPseudoNode(t *testing.T) { + for nodeID, want := range map[string]report.MappedNode{ + report.TheInternet: {report.TheInternet, "the Internet", "", "theinternet"}, + "x;y;z": {report.MakePseudoNodeID("x", "y", "z"), "y:z", "", report.MakePseudoNodeID("x", "y", "z")}, + "x;y": {report.MakePseudoNodeID("x", "y"), "y", "", report.MakePseudoNodeID("x", "y")}, + "x": {report.MakePseudoNodeID("x"), "x", "", report.MakePseudoNodeID("x")}, + } { + have := report.BasicPseudoNode(nodeID) + if !reflect.DeepEqual(want, have) { + t.Errorf("%q: %s", nodeID, diff(want, have)) + continue } - if want, have := c.wantRank, m.Rank; want != have { - t.Errorf("%s: map rank: want %#v, have %#v", identity, want, have) + } +} + +func TestGroupedPseudoNode(t *testing.T) { + for nodeID, want := range map[string]report.MappedNode{ + report.TheInternet: {report.TheInternet, "the Internet", "", "theinternet"}, + "x;y;z": {report.MakePseudoNodeID("unknown"), "Unknown", "", report.MakePseudoNodeID("unknown")}, + "x;y": {report.MakePseudoNodeID("unknown"), "Unknown", "", report.MakePseudoNodeID("unknown")}, + "x": {report.MakePseudoNodeID("unknown"), "Unknown", "", report.MakePseudoNodeID("unknown")}, + } { + have := report.GroupedPseudoNode(nodeID) + if !reflect.DeepEqual(want, have) { + t.Errorf("%q: %s", nodeID, diff(want, have)) + continue } } } -func TestGroupedMapping(t *testing.T) { - t.Skipf("not yet implemented") // TODO +var uncontained = report.MappedNode{ + ID: "uncontained", + Major: "Uncontained", + Minor: "", + Rank: "uncontained", } diff --git a/report/merge.go b/report/merge.go deleted file mode 100644 index 16d7bd0f1d..0000000000 --- a/report/merge.go +++ /dev/null @@ -1,92 +0,0 @@ -package report - -// Merge() functions for all topology datatypes. -// The general semantics are that the receiver is modified, and what's merged -// in isn't. - -// Merge merges another Report into the receiver. -func (r *Report) Merge(other Report) { - r.Process.Merge(other.Process) - r.Network.Merge(other.Network) - r.HostMetadatas.Merge(other.HostMetadatas) -} - -// Merge merges another Topology into the receiver. -func (t *Topology) Merge(other Topology) { - t.Adjacency.Merge(other.Adjacency) - t.EdgeMetadatas.Merge(other.EdgeMetadatas) - t.NodeMetadatas.Merge(other.NodeMetadatas) -} - -// Merge merges another Adjacency list into the receiver. -func (a *Adjacency) Merge(other Adjacency) { - for addr, adj := range other { - (*a)[addr] = (*a)[addr].Add(adj...) - } -} - -// Merge merges another NodeMetadatas into the receiver. -func (m *NodeMetadatas) Merge(other NodeMetadatas) { - for id, meta := range other { - if _, ok := (*m)[id]; !ok { - (*m)[id] = meta // not a copy - } - } -} - -// Merge merges another EdgeMetadatas into the receiver. -// If other is from another probe this is the union of both metadatas. Keys -// present in both are summed. -func (e *EdgeMetadatas) Merge(other EdgeMetadatas) { - for id, edgemeta := range other { - local := (*e)[id] - local.Merge(edgemeta) - (*e)[id] = local - } -} - -// Merge merges another HostMetadata into the receiver. -// It'll takes the lastest version if there are conflicts. -func (e *HostMetadatas) Merge(other HostMetadatas) { - for hostID, meta := range other { - if existing, ok := (*e)[hostID]; ok { - // Conflict. Take the newest. - if existing.Timestamp.After(meta.Timestamp) { - continue - } - } - (*e)[hostID] = meta - } -} - -// Merge merges another EdgeMetadata into the receiver. The two edge metadatas -// should represent the same edge on different times. -func (m *EdgeMetadata) Merge(other EdgeMetadata) { - if other.WithBytes { - m.WithBytes = true - m.BytesIngress += other.BytesIngress - m.BytesEgress += other.BytesEgress - } - if other.WithConnCountTCP { - m.WithConnCountTCP = true - if other.MaxConnCountTCP > m.MaxConnCountTCP { - m.MaxConnCountTCP = other.MaxConnCountTCP - } - } -} - -// Flatten sums two EdgeMetadatas, their 'Window's should be the same size. The -// two EdgeMetadatas should represent different edges at the same time. -func (m *EdgeMetadata) Flatten(other EdgeMetadata) { - if other.WithBytes { - m.WithBytes = true - m.BytesIngress += other.BytesIngress - m.BytesEgress += other.BytesEgress - } - if other.WithConnCountTCP { - m.WithConnCountTCP = true - // Note: summing of two maximums doesn't always give the true maximum. - // But it's our Best Effort effort. - m.MaxConnCountTCP += other.MaxConnCountTCP - } -} diff --git a/report/merge_test.go b/report/merge_test.go deleted file mode 100644 index 9ccd35b4a6..0000000000 --- a/report/merge_test.go +++ /dev/null @@ -1,416 +0,0 @@ -package report - -import ( - "reflect" - "testing" - "time" -) - -func TestMergeAdjacency(t *testing.T) { - for name, c := range map[string]struct { - a, b, want Adjacency - }{ - "Empty b": { - a: Adjacency{ - "hostA|:192.168.1.1:12345": NewIDList(":192.168.1.2:80"), - "hostA|:192.168.1.1:8888": NewIDList(":1.2.3.4:22"), - "hostB|:192.168.1.2:80": NewIDList(":192.168.1.1:12345"), - }, - b: Adjacency{}, - want: Adjacency{ - "hostA|:192.168.1.1:12345": NewIDList(":192.168.1.2:80"), - "hostA|:192.168.1.1:8888": NewIDList(":1.2.3.4:22"), - "hostB|:192.168.1.2:80": NewIDList(":192.168.1.1:12345"), - }, - }, - "Empty a": { - a: Adjacency{}, - b: Adjacency{ - "hostA|:192.168.1.1:12345": NewIDList(":192.168.1.2:80"), - "hostA|:192.168.1.1:8888": NewIDList(":1.2.3.4:22"), - "hostB|:192.168.1.2:80": NewIDList(":192.168.1.1:12345"), - }, - want: Adjacency{ - "hostA|:192.168.1.1:12345": NewIDList(":192.168.1.2:80"), - "hostA|:192.168.1.1:8888": NewIDList(":1.2.3.4:22"), - "hostB|:192.168.1.2:80": NewIDList(":192.168.1.1:12345"), - }, - }, - "Same address": { - a: Adjacency{ - "hostA|:192.168.1.1:12345": NewIDList(":192.168.1.2:80"), - }, - b: Adjacency{ - "hostA|:192.168.1.1:12345": NewIDList(":192.168.1.2:8080"), - }, - want: Adjacency{ - "hostA|:192.168.1.1:12345": NewIDList( - ":192.168.1.2:80", ":192.168.1.2:8080", - ), - }, - }, - "No duplicates": { - a: Adjacency{ - "hostA|:192.168.1.1:12345": NewIDList( - ":192.168.1.2:80", - ":192.168.1.2:8080", - ":192.168.1.2:555", - ), - }, - b: Adjacency{ - "hostA|:192.168.1.1:12345": NewIDList( - ":192.168.1.2:8080", - ":192.168.1.2:80", - ":192.168.1.2:444", - ), - }, - want: Adjacency{ - "hostA|:192.168.1.1:12345": []string{ - ":192.168.1.2:444", - ":192.168.1.2:555", - ":192.168.1.2:80", - ":192.168.1.2:8080", - }, - }, - }, - "Double keys": { - a: Adjacency{ - "key1": NewIDList("a", "c", "d", "b"), - "key2": NewIDList("c", "a"), - }, - b: Adjacency{ - "key1": NewIDList("a", "b", "e"), - "key3": NewIDList("e", "a", "a", "a", "e"), - }, - want: Adjacency{ - "key1": NewIDList("a", "b", "c", "d", "e"), - "key2": NewIDList("a", "c"), - "key3": NewIDList("a", "e"), - }, - }, - } { - have := c.a - have.Merge(c.b) - - if !reflect.DeepEqual(c.want, have) { - t.Errorf("%s: want\n\t%#v\nhave\n\t%#v", name, c.want, have) - } - } -} - -func TestMergeEdgeMetadatas(t *testing.T) { - for name, c := range map[string]struct { - a, b, want EdgeMetadatas - }{ - "Empty a": { - a: EdgeMetadatas{}, - b: EdgeMetadatas{ - "hostA|:192.168.1.1:12345|:192.168.1.2:80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - WithConnCountTCP: true, - MaxConnCountTCP: 2, - }, - }, - want: EdgeMetadatas{ - "hostA|:192.168.1.1:12345|:192.168.1.2:80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - WithConnCountTCP: true, - MaxConnCountTCP: 2, - }, - }, - }, - "Empty b": { - a: EdgeMetadatas{ - "hostA|:192.168.1.1:12345|:192.168.1.2:80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - }, - }, - b: EdgeMetadatas{}, - want: EdgeMetadatas{ - "hostA|:192.168.1.1:12345|:192.168.1.2:80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - }, - }, - }, - "Host merge": { - a: EdgeMetadatas{ - "hostA|:192.168.1.1:12345|:192.168.1.2:80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - WithConnCountTCP: true, - MaxConnCountTCP: 4, - }, - }, - b: EdgeMetadatas{ - "hostQ|:192.168.1.1:12345|:192.168.1.2:80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 1, - BytesIngress: 2, - WithConnCountTCP: true, - MaxConnCountTCP: 6, - }, - }, - want: EdgeMetadatas{ - "hostA|:192.168.1.1:12345|:192.168.1.2:80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - WithConnCountTCP: true, - MaxConnCountTCP: 4, - }, - "hostQ|:192.168.1.1:12345|:192.168.1.2:80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 1, - BytesIngress: 2, - WithConnCountTCP: true, - MaxConnCountTCP: 6, - }, - }, - }, - "Edge merge": { - a: EdgeMetadatas{ - "hostA|:192.168.1.1:12345|:192.168.1.2:80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - WithConnCountTCP: true, - MaxConnCountTCP: 7, - }, - }, - b: EdgeMetadatas{ - "hostA|:192.168.1.1:12345|:192.168.1.2:80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 1, - BytesIngress: 2, - WithConnCountTCP: true, - MaxConnCountTCP: 9, - }, - }, - want: EdgeMetadatas{ - "hostA|:192.168.1.1:12345|:192.168.1.2:80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 13, - BytesIngress: 2, - WithConnCountTCP: true, - MaxConnCountTCP: 9, - }, - }, - }, - } { - have := c.a - have.Merge(c.b) - - if !reflect.DeepEqual(c.want, have) { - t.Errorf("%s: want\n\t%#v, have\n\t%#v", name, c.want, have) - } - } -} - -func TestMergeHostMetadatas(t *testing.T) { - now := time.Now() - - for name, c := range map[string]struct { - a, b, want HostMetadatas - }{ - "Empty a": { - a: HostMetadatas{}, - b: HostMetadatas{ - "hostA": HostMetadata{ - Timestamp: now, - Hostname: "host-a", - OS: "linux", - }, - }, - want: HostMetadatas{ - "hostA": HostMetadata{ - Timestamp: now, - Hostname: "host-a", - OS: "linux", - }, - }, - }, - "Empty b": { - a: HostMetadatas{ - "hostA": HostMetadata{ - Timestamp: now, - Hostname: "host-a", - OS: "linux", - }, - }, - b: HostMetadatas{}, - want: HostMetadatas{ - "hostA": HostMetadata{ - Timestamp: now, - Hostname: "host-a", - OS: "linux", - }, - }, - }, - "Host merge": { - a: HostMetadatas{ - "hostA": HostMetadata{ - Timestamp: now, - Hostname: "host-a", - OS: "linux", - }, - }, - b: HostMetadatas{ - "hostB": HostMetadata{ - Timestamp: now, - Hostname: "host-b", - OS: "freedos", - }, - }, - want: HostMetadatas{ - "hostB": HostMetadata{ - Timestamp: now, - Hostname: "host-b", - OS: "freedos", - }, - "hostA": HostMetadata{ - Timestamp: now, - Hostname: "host-a", - OS: "linux", - }, - }, - }, - "Host conflict": { - a: HostMetadatas{ - "hostA": HostMetadata{ - Timestamp: now, - Hostname: "host-a", - OS: "linux1", - }, - }, - b: HostMetadatas{ - "hostA": HostMetadata{ - Timestamp: now.Add(-10 * time.Second), - Hostname: "host-a", - OS: "linux0", - }, - }, - want: HostMetadatas{ - "hostA": HostMetadata{ - Timestamp: now, - Hostname: "host-a", - OS: "linux1", - }, - }, - }, - } { - have := c.a - have.Merge(c.b) - - if !reflect.DeepEqual(c.want, have) { - t.Errorf("%s: want\n\t%#v, have\n\t%#v", name, c.want, have) - } - } -} - -func TestMergeNodeMetadatas(t *testing.T) { - for name, c := range map[string]struct { - a, b, want NodeMetadatas - }{ - "Empty a": { - a: NodeMetadatas{}, - b: NodeMetadatas{ - ":192.168.1.1:12345": NodeMetadata{ - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - }, - want: NodeMetadatas{ - ":192.168.1.1:12345": NodeMetadata{ - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - }, - }, - "Empty b": { - a: NodeMetadatas{ - ":192.168.1.1:12345": NodeMetadata{ - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - }, - b: NodeMetadatas{}, - want: NodeMetadatas{ - ":192.168.1.1:12345": NodeMetadata{ - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - }, - }, - "Simple merge": { - a: NodeMetadatas{ - ":192.168.1.1:12345": NodeMetadata{ - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - }, - b: NodeMetadatas{ - ":192.168.1.2:12345": NodeMetadata{ - "pid": "42", - "name": "curl", - "domain": "node-a.local", - }, - }, - want: NodeMetadatas{ - ":192.168.1.1:12345": NodeMetadata{ - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - ":192.168.1.2:12345": NodeMetadata{ - "pid": "42", - "name": "curl", - "domain": "node-a.local", - }, - }, - }, - "Merge conflict": { - a: NodeMetadatas{ - ":192.168.1.1:12345": NodeMetadata{ - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - }, - b: NodeMetadatas{ - ":192.168.1.1:12345": NodeMetadata{ // <-- same ID - "pid": "0", - "name": "curl", - "domain": "node-a.local", - }, - }, - want: NodeMetadatas{ - ":192.168.1.1:12345": NodeMetadata{ - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - }, - }, - } { - have := c.a - have.Merge(c.b) - - if !reflect.DeepEqual(c.want, have) { - t.Errorf("%s: want\n\t%#v, have\n\t%#v", name, c.want, have) - } - } -} diff --git a/report/metadata.go b/report/metadata.go deleted file mode 100644 index 7e5d3aa762..0000000000 --- a/report/metadata.go +++ /dev/null @@ -1,42 +0,0 @@ -package report - -// AggregateMetadata is a composable version of an EdgeMetadata. It's used -// when we want to merge nodes/edges for any reason. -// -// Even though we base it on EdgeMetadata, we can apply it to nodes, by -// summing up (merging) all of the {ingress, egress} metadatas of the -// {incoming, outgoing} edges to the node. -type AggregateMetadata map[string]int - -const ( - // KeyBytesIngress is the aggregate metadata key for the total count of - // ingress bytes. - KeyBytesIngress = "ingress_bytes" - // KeyBytesEgress is the aggregate metadata key for the total count of - // egress bytes. - KeyBytesEgress = "egress_bytes" - // KeyMaxConnCountTCP is the aggregate metadata key for the maximum number - // of simultaneous observed TCP connections in the window. - KeyMaxConnCountTCP = "max_conn_count_tcp" -) - -// Transform calculates a AggregateMetadata from an EdgeMetadata. -func (md EdgeMetadata) Transform() AggregateMetadata { - m := AggregateMetadata{} - if md.WithBytes { - m[KeyBytesIngress] = int(md.BytesIngress) - m[KeyBytesEgress] = int(md.BytesEgress) - } - if md.WithConnCountTCP { - // The maximum is the maximum. No need to calculate anything. - m[KeyMaxConnCountTCP] = int(md.MaxConnCountTCP) - } - return m -} - -// Merge adds the fields from AggregateMetadata to r. r must be initialized. -func (r *AggregateMetadata) Merge(other AggregateMetadata) { - for k, v := range other { - (*r)[k] += v - } -} diff --git a/report/metadata_test.go b/report/metadata_test.go deleted file mode 100644 index 8795f3c473..0000000000 --- a/report/metadata_test.go +++ /dev/null @@ -1,84 +0,0 @@ -package report - -import ( - "reflect" - "testing" -) - -func TestAggregateMetadata(t *testing.T) { - for from, want := range map[EdgeMetadata]AggregateMetadata{ - - // Simple connection count - EdgeMetadata{ - WithConnCountTCP: true, - MaxConnCountTCP: 400, - }: { - KeyMaxConnCountTCP: 400, - }, - - // Connection count rounding - EdgeMetadata{ - WithConnCountTCP: true, - MaxConnCountTCP: 4, - }: { - KeyMaxConnCountTCP: 4, - }, - - // 0 connections. - EdgeMetadata{ - WithConnCountTCP: true, - MaxConnCountTCP: 0, - }: { - KeyMaxConnCountTCP: 0, - }, - - // Egress - EdgeMetadata{ - WithBytes: true, - BytesEgress: 24, - BytesIngress: 0, - }: { - KeyBytesEgress: 24, - KeyBytesIngress: 0, - }, - - // Ingress - EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 1200, - }: { - KeyBytesEgress: 0, - KeyBytesIngress: 1200, - }, - - // Nothing there. - EdgeMetadata{}: {}, - } { - if have := from.Transform(); !reflect.DeepEqual(have, want) { - t.Errorf("have: %#v, want %#v", have, want) - } - - } -} - -func TestAggregateMetadataSum(t *testing.T) { - var ( - this = AggregateMetadata{ - "ingress_bytes": 3, - } - other = AggregateMetadata{ - "ingress_bytes": 333, - "egress_bytes": 3, - } - want = AggregateMetadata{ - "ingress_bytes": 336, - "egress_bytes": 3, - } - ) - - this.Merge(other) - if have := this; !reflect.DeepEqual(have, want) { - t.Errorf("have: %#v, want %#v", have, want) - } -} diff --git a/report/render.go b/report/render.go new file mode 100644 index 0000000000..c02faf7a51 --- /dev/null +++ b/report/render.go @@ -0,0 +1,114 @@ +package report + +import "fmt" + +// Render transforms a report into a set of RenderableNodes, which the UI will +// render collectively as a graph. RenderBy takes a MapFunc, which defines how +// to group and label nodes. Nodes with the same mapped IDs will be merged. +func Render(r Report, ts TopologySelector, mapper MapFunc, pseudo PseudoFunc) map[string]RenderableNode { + var ( + nodes = map[string]RenderableNode{} + t = ts(r) + ) + + // Build a set of RenderableNodes for all node IDs, and an ID lookup map. + // Multiple IDs can map to the same RenderableNodes. + source2mapped := map[string]string{} + for nodeID := range t.NodeMetadatas { + mapped, show := mapper(r, ts, nodeID) + if !show { + continue + } + + // mapped.ID needs not be unique over all IDs. We just overwrite the + // existing data, on the assumption that the MapFunc returns the same + // data. + nodes[mapped.ID] = RenderableNode{ + ID: mapped.ID, + LabelMajor: mapped.Major, + LabelMinor: mapped.Minor, + Rank: mapped.Rank, + Pseudo: false, + Adjacency: IDList{}, // later + Origins: IDList{}, // later + Metadata: AggregateMetadata{}, // later + } + source2mapped[nodeID] = mapped.ID + } + + // Walk the graph and make connections. + for adjacencyID, dstNodeIDs := range t.Adjacency { + srcHostID, srcNodeID, ok := ParseAdjacencyID(adjacencyID) + if !ok { + panic(fmt.Sprintf("badly formed Topology: invalid adjacencyID %q", adjacencyID)) + } + + srcMappedNodeID, ok := source2mapped[srcNodeID] + if !ok { + if pseudo == nil { + continue // oh well + } + pseudoNode := pseudo(srcNodeID) + srcMappedNodeID = pseudoNode.ID + source2mapped[srcNodeID] = srcMappedNodeID + nodes[srcMappedNodeID] = pseudo2renderable(pseudoNode) + } + + srcRenderableNode, ok := nodes[srcMappedNodeID] + if !ok { + panic(fmt.Sprintf("badly formed mapping: %q (via %q) has no renderable node", srcMappedNodeID, srcNodeID)) + } + + for _, dstNodeID := range dstNodeIDs { + dstMappedNodeID, ok := source2mapped[dstNodeID] + if !ok { + if pseudo == nil { + continue // oh well + } + pseudoNode := pseudo(dstNodeID) + dstMappedNodeID = pseudoNode.ID + source2mapped[dstNodeID] = dstMappedNodeID + nodes[dstMappedNodeID] = pseudo2renderable(pseudoNode) + } + + srcRenderableNode.Adjacency = srcRenderableNode.Adjacency.Add(dstMappedNodeID) + srcRenderableNode.Origins = srcRenderableNode.Origins.Add(MakeHostNodeID(srcHostID)) + srcRenderableNode.Origins = srcRenderableNode.Origins.Add(srcNodeID) + edgeID := MakeEdgeID(srcNodeID, dstNodeID) + if md, ok := t.EdgeMetadatas[edgeID]; ok { + srcRenderableNode.Metadata = srcRenderableNode.Metadata.Merge(md.Export()) + } + } + + nodes[srcMappedNodeID] = srcRenderableNode + } + + return nodes +} + +// RenderableNode is the data type that's yielded to the JavaScript layer as +// an element of a topology. It should contain information that's relevant +// to rendering a node when there are many nodes visible at once. +type RenderableNode struct { + ID string `json:"id"` // + LabelMajor string `json:"label_major"` // e.g. "process", human-readable + LabelMinor string `json:"label_minor,omitempty"` // e.g. "hostname", human-readable, optional + Rank string `json:"rank"` // to help the layout engine + Pseudo bool `json:"pseudo,omitempty"` // sort-of a placeholder node, for rendering purposes + Adjacency IDList `json:"adjacency,omitempty"` // Same-topology node IDs + Origins IDList `json:"origins,omitempty"` // Foreign-key-scoped, core node IDs that contributed to this RenderableNode + Metadata AggregateMetadata `json:"metadata"` // Numeric sums +} + +func pseudo2renderable(mappedNode MappedNode) RenderableNode { + return RenderableNode{ + ID: mappedNode.ID, + LabelMajor: mappedNode.Major, + LabelMinor: mappedNode.Minor, + Rank: mappedNode.Rank, + Pseudo: true, + Adjacency: IDList{}, // fill in later + Origins: IDList{}, // fill in later + Metadata: AggregateMetadata{}, // fill in later + } +} diff --git a/report/render_test.go b/report/render_test.go new file mode 100644 index 0000000000..7a3840e610 --- /dev/null +++ b/report/render_test.go @@ -0,0 +1,169 @@ +package report_test + +import ( + "reflect" + "testing" + + "github.com/weaveworks/scope/report" +) + +func TestRenderEndpointAsProcessNameWithoutPseudoNodes(t *testing.T) { + have := report.Render(reportFixture, report.SelectEndpoint, report.ProcessName, nil) + want := map[string]report.RenderableNode{ + "apache": { + ID: "apache", + LabelMajor: "apache", + LabelMinor: "(unknown)", // because we didn't set a host_name + Rank: "apache", + Pseudo: false, + Adjacency: report.MakeIDList( + "curl", + ), + Origins: report.MakeIDList( + server80EndpointNodeID, + serverHostNodeID, + ), + Metadata: report.AggregateMetadata{ + report.KeyBytesEgress: 100, // Note that when we don't render pseudonodes + report.KeyBytesIngress: 10, // we lose their metadata. Could be dangerous! + }, + }, + "curl": { + ID: "curl", + LabelMajor: "curl", + LabelMinor: "client.host.com", + Rank: "curl", + Pseudo: false, + Adjacency: report.MakeIDList( + "apache", + ), + Origins: report.MakeIDList( + client54001EndpointNodeID, + //client54002EndpointNodeID, + clientHostNodeID, + ), + Metadata: report.AggregateMetadata{ + report.KeyBytesEgress: 10, + report.KeyBytesIngress: 100, + }, + }, + } + if !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } +} + +func TestRenderEndpointAsProcessNameWithBasicPseudoNodes(t *testing.T) { + have := report.Render(reportFixture, report.SelectEndpoint, report.ProcessName, report.BasicPseudoNode) + want := map[string]report.RenderableNode{ + "apache": { + ID: "apache", + LabelMajor: "apache", + LabelMinor: "(unknown)", // because we didn't set a host_name + Rank: "apache", + Pseudo: false, + Adjacency: report.MakeIDList( + "curl", + report.MakePseudoNodeID(unknownHostID, unknownAddress, "10001"), + report.MakePseudoNodeID(unknownHostID, unknownAddress, "10002"), + report.MakePseudoNodeID(unknownHostID, unknownAddress, "10003"), + report.MakePseudoNodeID(clientHostID, clientAddress, "54002"), + ), + Origins: report.MakeIDList( + server80EndpointNodeID, + serverHostNodeID, + ), + Metadata: report.AggregateMetadata{ + report.KeyBytesEgress: 3100, // Here, the metadata is preserved, + report.KeyBytesIngress: 310, // thanks to the pseudonode. + }, + }, + "curl": { + ID: "curl", + LabelMajor: "curl", + LabelMinor: clientHostID, + Rank: "curl", + Pseudo: false, + Adjacency: report.MakeIDList( + "apache", + ), + Origins: report.MakeIDList( + client54001EndpointNodeID, + //client54002EndpointNodeID, + clientHostNodeID, + ), + Metadata: report.AggregateMetadata{ + report.KeyBytesEgress: 10, // Here, we lose some outgoing metadata, + report.KeyBytesIngress: 100, // but that's to be expected. + }, + }, + report.MakePseudoNodeID(unknownHostID, unknownAddress, "10001"): { + ID: report.MakePseudoNodeID(unknownHostID, unknownAddress, "10001"), + LabelMajor: unknownAddress + ":10001", + LabelMinor: "", + Rank: report.MakePseudoNodeID(unknownHostID, unknownAddress, "10001"), + Pseudo: true, + Adjacency: report.IDList{}, + Origins: report.IDList{}, + Metadata: report.AggregateMetadata{}, + }, + report.MakePseudoNodeID(unknownHostID, unknownAddress, "10002"): { + ID: report.MakePseudoNodeID(unknownHostID, unknownAddress, "10002"), + LabelMajor: unknownAddress + ":10002", + LabelMinor: "", + Rank: report.MakePseudoNodeID(unknownHostID, unknownAddress, "10002"), + Pseudo: true, + Adjacency: report.IDList{}, + Origins: report.IDList{}, + Metadata: report.AggregateMetadata{}, + }, + report.MakePseudoNodeID(unknownHostID, unknownAddress, "10003"): { + ID: report.MakePseudoNodeID(unknownHostID, unknownAddress, "10003"), + LabelMajor: unknownAddress + ":10003", + LabelMinor: "", + Rank: report.MakePseudoNodeID(unknownHostID, unknownAddress, "10003"), + Pseudo: true, + Adjacency: report.IDList{}, + Origins: report.IDList{}, + Metadata: report.AggregateMetadata{}, + }, + report.MakePseudoNodeID(clientHostID, clientAddress, "54002"): { + ID: report.MakePseudoNodeID(clientHostID, clientAddress, "54002"), + LabelMajor: clientAddress + ":54002", + LabelMinor: "", + Rank: report.MakePseudoNodeID(clientHostID, clientAddress, "54002"), + Pseudo: true, + Adjacency: report.MakeIDList( + "apache", + ), + Origins: report.MakeIDList( + client54002EndpointNodeID, + clientHostNodeID, + ), + Metadata: report.AggregateMetadata{ + report.KeyBytesEgress: 20, + report.KeyBytesIngress: 200, + }, + }, + } + if !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } +} + +func TestRenderPanicOnBadAdjacencyID(t *testing.T) { + if panicked := func() (recovered bool) { + defer func() { + if r := recover(); r != nil { + recovered = true + } + }() + r := report.MakeReport() + r.Endpoint.Adjacency["bad-adjacency-id"] = report.MakeIDList(server80EndpointNodeID) + report.Render(r, report.SelectEndpoint, report.ProcessPID, nil) + return false + }(); !panicked { + t.Errorf("expected panic, didn't get it") + } + +} diff --git a/report/report.go b/report/report.go index 0863539203..31049e7f9d 100644 --- a/report/report.go +++ b/report/report.go @@ -1,149 +1,211 @@ package report import ( - "encoding/json" + "fmt" "net" - "time" + "strings" ) -// Report is the internal structure produced and emitted by the probe, and -// operated-on (e.g. merged) by intermediaries and the app. The probe may fill -// in as many topologies as it's capable of producing, including none. -// -// Process, [Transport,] and Network topologies are distinct because the data -// sources are distinct. That is, the Process topology can only be populated -// by extant connections between processes, but the Network topology may be -// populated from e.g. system-level data sources. -// -// Since the data sources are fundamentally different for each topology, it -// might make sense to make them more distinct in the user interface. +// Report is the core data type. It's produced by probes, and consumed and +// stored by apps. It's composed of multiple topologies, each representing +// a different (related, but not equivalent) view of the network. type Report struct { - Process Topology - // Transport Topology - Network Topology - HostMetadatas + Endpoint Topology + Address Topology + Process Topology + Host Topology } -// HostMetadatas contains metadata about the host(s) represented in the Report. -type HostMetadatas map[string]HostMetadata +// MakeReport produces a new report, ready for use. It's the only correct way +// to produce reports for general use, so please use it. +func MakeReport() Report { + return Report{ + Endpoint: MakeTopology(), + Address: MakeTopology(), + Process: MakeTopology(), + Host: MakeTopology(), + } +} -// HostMetadata describes metadata that probes can collect about the host that -// they run on. It has a timestamp when the measurement was made. -type HostMetadata struct { - Timestamp time.Time - Hostname string - LocalNets []*net.IPNet - OS string - LoadOne, LoadFive, LoadFifteen float64 +// Copy returns a value copy, useful for tests. +func (r Report) Copy() Report { + return Report{ + Endpoint: r.Endpoint.Copy(), + Address: r.Address.Copy(), + Process: r.Process.Copy(), + Host: r.Host.Copy(), + } } -// RenderableNode is the data type that's yielded to the JavaScript layer as -// an element of a topology. It should contain information that's relevant -// to rendering a node when there are many nodes visible at once. -type RenderableNode struct { - ID string `json:"id"` // - LabelMajor string `json:"label_major"` // e.g. "process", human-readable - LabelMinor string `json:"label_minor,omitempty"` // e.g. "hostname", human-readable, optional - Rank string `json:"rank"` // to help the layout engine - Pseudo bool `json:"pseudo,omitempty"` // sort-of a placeholder node, for rendering purposes - Adjacency IDList `json:"adjacency,omitempty"` // Node IDs (in the same topology domain) - OriginHosts IDList `json:"origin_hosts,omitempty"` // Which hosts contributed information to this node - OriginNodes IDList `json:"origin_nodes,omitempty"` // Which origin nodes (depends on topology) contributed - Metadata AggregateMetadata `json:"metadata"` // Numeric sums +// Merge merges two reports together, returning the result. Always reassign +// the result of merge to the destination report. Merge is defined on report +// as a value-type, but report contains reference fields, so if you want to +// maintain immutable reports, use copy. +func (r Report) Merge(other Report) Report { + r.Endpoint = r.Endpoint.Merge(other.Endpoint) + r.Address = r.Address.Merge(other.Address) + r.Process = r.Process.Merge(other.Process) + r.Host = r.Host.Merge(other.Host) + return r } -// DetailedNode is the data type that's yielded to the JavaScript layer when -// we want deep information about an individual node. -type DetailedNode struct { - ID string `json:"id"` - LabelMajor string `json:"label_major"` - LabelMinor string `json:"label_minor,omitempty"` - Pseudo bool `json:"pseudo,omitempty"` - Tables []Table `json:"tables"` +// Squash squashes all non-local nodes in the report to a super-node called +// the Internet. +func (r Report) Squash() Report { + localNetworks := r.LocalNetworks() + r.Endpoint = r.Endpoint.Squash(EndpointIDAddresser, localNetworks) + r.Address = r.Address.Squash(AddressIDAddresser, localNetworks) + r.Process = r.Process.Squash(PanicIDAddresser, localNetworks) + r.Host = r.Host.Squash(PanicIDAddresser, localNetworks) + return r } -// Table is a dataset associated with a node. It will be displayed in the -// detail panel when a user clicks on a node. -type Table struct { - Title string `json:"title"` // e.g. Bandwidth - Numeric bool `json:"numeric"` // should the major column be right-aligned? - Rows []Row `json:"rows"` +// LocalNetworks returns a superset of the networks (think: CIDR) that are +// "local" from the perspective of each host represented in the report. It's +// used to determine which nodes in the report are "remote", i.e. outside of +// our domain of awareness. +func (r Report) LocalNetworks() []*net.IPNet { + var ipNets []*net.IPNet + for _, md := range r.Host.NodeMetadatas { + val, ok := md["local_networks"] + if !ok { + continue + } + outer: + for _, s := range strings.Fields(val) { + _, ipNet, err := net.ParseCIDR(s) + if err != nil { + continue + } + for _, existing := range ipNets { + if ipNet.String() == existing.String() { + continue outer + } + } + ipNets = append(ipNets, ipNet) + } + } + return ipNets } -// Row is a single entry in a Table dataset. -type Row struct { - Key string `json:"key"` // e.g. Ingress - ValueMajor string `json:"value_major"` // e.g. 25 - ValueMinor string `json:"value_minor,omitempty"` // e.g. KB/s +// EdgeMetadata gives the metadata of an edge from the perspective of the +// srcMappedID. Since an edgeID can have multiple edges on the address level, +// it uses the supplied mapping function to translate core node IDs to +// renderable mapped IDs. +func (r Report) EdgeMetadata(ts TopologySelector, mapper MapFunc, srcMappedID, dstMappedID string) EdgeMetadata { + t := ts(r) + result := EdgeMetadata{} + for edgeID, edgeMetadata := range t.EdgeMetadatas { + srcNodeID, dstNodeID, ok := ParseEdgeID(edgeID) + if !ok { + panic(fmt.Sprintf("invalid edge ID %q", edgeID)) + } + src, showSrc := mapper(r, ts, srcNodeID) // TODO srcNodeID == TheInternet checking? + dst, showDst := mapper(r, ts, dstNodeID) // TODO dstNodeID == TheInternet checking? + if showSrc && showDst && src.ID == srcMappedID && dst.ID == dstMappedID { + result = result.Flatten(edgeMetadata) + } + } + return result } -// NewReport makes a clean report, ready to Merge() other reports into. -func NewReport() Report { - return Report{ - Process: NewTopology(), - // Transport Topology - Network: NewTopology(), - HostMetadatas: map[string]HostMetadata{}, +// OriginTable produces a table (to be consumed directly by the UI) based on +// an origin ID, which is (optimistically) a node ID in one of our topologies. +func (r Report) OriginTable(originID string) (Table, bool) { + for nodeID, nodeMetadata := range r.Endpoint.NodeMetadatas { + if originID == nodeID { + return endpointOriginTable(nodeMetadata) + } + } + for nodeID, nodeMetadata := range r.Address.NodeMetadatas { + if originID == nodeID { + return addressOriginTable(nodeMetadata) + } } + for nodeID, nodeMetadata := range r.Process.NodeMetadatas { + if originID == nodeID { + return processOriginTable(nodeMetadata) + } + } + for nodeID, nodeMetadata := range r.Host.NodeMetadatas { + if originID == nodeID { + return hostOriginTable(nodeMetadata) + } + } + return Table{}, false } -// SquashRemote folds all remote nodes into a special supernode. It uses the -// LocalNets of the hosts in HostMetadata to determine which addresses are -// local. -func (r Report) SquashRemote() Report { - localNets := r.HostMetadatas.LocalNets() - return Report{ - Process: Squash(r.Process, AddressIPPort, localNets), - Network: Squash(r.Network, AddressIP, localNets), - HostMetadatas: r.HostMetadatas, +func endpointOriginTable(nmd NodeMetadata) (Table, bool) { + rows := []Row{} + if val, ok := nmd["endpoint"]; ok { + rows = append(rows, Row{"Endpoint", val, ""}) + } + if val, ok := nmd["host_name"]; ok { + rows = append(rows, Row{"Host name", val, ""}) } + return Table{ + Title: "Origin Endpoint", + Numeric: false, + Rows: rows, + }, len(rows) > 0 } -// LocalNets gives the union of all local network IPNets for all hosts -// represented in the HostMetadatas. -func (m HostMetadatas) LocalNets() []*net.IPNet { - var nets []*net.IPNet - for _, node := range m { - OUTER: - for _, local := range node.LocalNets { - for _, existing := range nets { - if existing == local { - continue OUTER - } - } - nets = append(nets, local) - } +func addressOriginTable(nmd NodeMetadata) (Table, bool) { + rows := []Row{} + if val, ok := nmd["address"]; ok { + rows = append(rows, Row{"Address", val, ""}) + } + if val, ok := nmd["host_name"]; ok { + rows = append(rows, Row{"Host name", val, ""}) } - return nets + return Table{ + Title: "Origin Address", + Numeric: false, + Rows: rows, + }, len(rows) > 0 } -// UnmarshalJSON is a custom JSON deserializer for HostMetadata to deal with -// the Localnets. -func (m *HostMetadata) UnmarshalJSON(data []byte) error { - type netmask struct { - IP net.IP - Mask []byte - } - tmpHMD := struct { - Timestamp time.Time - Hostname string - LocalNets []*netmask - OS string - LoadOne, LoadFive, LoadFifteen float64 - }{} - err := json.Unmarshal(data, &tmpHMD) - if err != nil { - return err +func processOriginTable(nmd NodeMetadata) (Table, bool) { + rows := []Row{} + if val, ok := nmd["process_name"]; ok { + rows = append(rows, Row{"Process name", val, ""}) } + if val, ok := nmd["pid"]; ok { + rows = append(rows, Row{"PID", val, ""}) + } + if val, ok := nmd["docker_container_id"]; ok { + rows = append(rows, Row{"Docker container ID", val, ""}) + } + if val, ok := nmd["docker_container_name"]; ok { + rows = append(rows, Row{"Docker container name", val, ""}) + } + if val, ok := nmd["docker_image_id"]; ok { + rows = append(rows, Row{"Docker image ID", val, ""}) + } + if val, ok := nmd["docker_image_name"]; ok { + rows = append(rows, Row{"Docker image name", val, ""}) + } + return Table{ + Title: "Origin Process", + Numeric: false, + Rows: rows, + }, len(rows) > 0 +} - m.Timestamp = tmpHMD.Timestamp - m.Hostname = tmpHMD.Hostname - m.OS = tmpHMD.OS - m.LoadOne = tmpHMD.LoadOne - m.LoadFive = tmpHMD.LoadFive - m.LoadFifteen = tmpHMD.LoadFifteen - for _, ln := range tmpHMD.LocalNets { - m.LocalNets = append(m.LocalNets, &net.IPNet{IP: ln.IP, Mask: ln.Mask}) - } - return nil +func hostOriginTable(nmd NodeMetadata) (Table, bool) { + rows := []Row{} + if val, ok := nmd["host_name"]; ok { + rows = append(rows, Row{"Host name", val, ""}) + } + if val, ok := nmd["load"]; ok { + rows = append(rows, Row{"Load", val, ""}) + } + if val, ok := nmd["os"]; ok { + rows = append(rows, Row{"Operating system", val, ""}) + } + return Table{ + Title: "Origin Host", + Numeric: false, + Rows: rows, + }, len(rows) > 0 } diff --git a/report/report_fixture_test.go b/report/report_fixture_test.go new file mode 100644 index 0000000000..256d8f010c --- /dev/null +++ b/report/report_fixture_test.go @@ -0,0 +1,166 @@ +package report_test + +import ( + "github.com/weaveworks/scope/report" +) + +var reportFixture = report.Report{ + Endpoint: report.Topology{ + Adjacency: report.Adjacency{ + report.MakeAdjacencyID(clientHostID, client54001EndpointNodeID): report.MakeIDList(server80EndpointNodeID), + report.MakeAdjacencyID(clientHostID, client54002EndpointNodeID): report.MakeIDList(server80EndpointNodeID), + report.MakeAdjacencyID(serverHostID, server80EndpointNodeID): report.MakeIDList(client54001EndpointNodeID, client54002EndpointNodeID, unknown1EndpointNodeID, unknown2EndpointNodeID, unknown3EndpointNodeID), + }, + NodeMetadatas: report.NodeMetadatas{ + client54001EndpointNodeID: report.NodeMetadata{ + "process_node_id": report.MakeProcessNodeID(clientHostID, "4242"), + "address_node_id": report.MakeAddressNodeID(clientHostID, clientAddress), + }, + client54002EndpointNodeID: report.NodeMetadata{ + //"process_node_id": report.MakeProcessNodeID(clientHostID, "4242"), // leave it out, to test a branch in Render + "address_node_id": report.MakeAddressNodeID(clientHostID, clientAddress), + }, + server80EndpointNodeID: report.NodeMetadata{ + "process_node_id": report.MakeProcessNodeID(serverHostID, "215"), + "address_node_id": report.MakeAddressNodeID(serverHostID, serverAddress), + }, + + "process-not-available": report.NodeMetadata{}, // for TestProcess{PID,Name,Container[Name]} + "process-badly-linked": report.NodeMetadata{"process_node_id": "none"}, // for TestProcess{PID,Name,Container[Name]} + "process-no-container": report.NodeMetadata{"process_node_id": "no-container"}, // for TestProcessContainer[Name] + "address-not-available": report.NodeMetadata{}, // for TestAddressHostname + "address-badly-linked": report.NodeMetadata{"address_node_id": "none"}, // for TestAddressHostname + }, + EdgeMetadatas: report.EdgeMetadatas{ + report.MakeEdgeID(client54001EndpointNodeID, server80EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 10, // src -> dst + BytesIngress: 100, // src <- dst + }, + report.MakeEdgeID(client54002EndpointNodeID, server80EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 20, + BytesIngress: 200, + }, + report.MakeEdgeID(server80EndpointNodeID, client54001EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 100, + BytesIngress: 10, + }, + report.MakeEdgeID(server80EndpointNodeID, client54002EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 200, + BytesIngress: 20, + }, + report.MakeEdgeID(server80EndpointNodeID, unknown1EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 400, + BytesIngress: 40, + }, + report.MakeEdgeID(server80EndpointNodeID, unknown2EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 800, + BytesIngress: 80, + }, + report.MakeEdgeID(server80EndpointNodeID, unknown3EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 1600, + BytesIngress: 160, + }, + }, + }, + Address: report.Topology{ + Adjacency: report.Adjacency{ + report.MakeAdjacencyID(clientHostID, clientAddressNodeID): report.MakeIDList(serverAddressNodeID), + report.MakeAdjacencyID(serverHostID, serverAddressNodeID): report.MakeIDList(clientAddressNodeID, unknownAddressNodeID), + }, + NodeMetadatas: report.NodeMetadatas{ + clientAddressNodeID: report.NodeMetadata{ + "host_name": "client.host.com", + }, + serverAddressNodeID: report.NodeMetadata{}, + + "no-host-name": report.NodeMetadata{}, + }, + EdgeMetadatas: report.EdgeMetadatas{ + report.MakeEdgeID(clientAddressNodeID, serverAddressNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 10 + 20 + 1, + BytesIngress: 100 + 200 + 2, + }, + report.MakeEdgeID(serverAddressNodeID, clientAddressNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 100 + 200 + 3, + BytesIngress: 10 + 20 + 4, + }, + report.MakeEdgeID(serverAddressNodeID, unknownAddressNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 400 + 800 + 1600 + 5, + BytesIngress: 40 + 80 + 160 + 6, + }, + }, + }, + Process: report.Topology{ + Adjacency: report.Adjacency{}, + NodeMetadatas: report.NodeMetadatas{ + report.MakeProcessNodeID(clientHostID, "4242"): report.NodeMetadata{ + "host_name": "client.host.com", + "pid": "4242", + "process_name": "curl", + "docker_container_id": "a1b2c3d4e5", + "docker_container_name": "fixture-container", + "docker_image_id": "0000000000", + "docker_image_name": "fixture/container:latest", + }, + report.MakeProcessNodeID(serverHostID, "215"): report.NodeMetadata{ + "pid": "215", + "process_name": "apache", + }, + + "no-container": report.NodeMetadata{}, + }, + EdgeMetadatas: report.EdgeMetadatas{}, + }, + Host: report.Topology{ + Adjacency: report.Adjacency{}, + NodeMetadatas: report.NodeMetadatas{ + report.MakeHostNodeID(clientHostID): report.NodeMetadata{ + "host_name": clientHostName, + "local_networks": "10.10.10.0/24", + "os": "OS/2", + "load": "0.11 0.22 0.33", + }, + report.MakeHostNodeID(serverHostID): report.NodeMetadata{ + "host_name": serverHostName, + "local_networks": "10.10.10.0/24", + "os": "Linux", + "load": "0.01 0.01 0.01", + }, + }, + EdgeMetadatas: report.EdgeMetadatas{}, + }, +} + +var ( + clientHostID = "client.host.com" + clientHostName = clientHostID + clientHostNodeID = report.MakeHostNodeID(clientHostID) + clientAddress = "10.10.10.20" + serverHostID = "server.host.com" + serverHostName = serverHostID + serverHostNodeID = report.MakeHostNodeID(serverHostID) + serverAddress = "10.10.10.1" + unknownHostID = "" // by definition, we don't know it + unknownAddress = "172.16.93.112" // will be a pseudonode, no corresponding host + + client54001EndpointNodeID = report.MakeEndpointNodeID(clientHostID, clientAddress, "54001") // i.e. curl + client54002EndpointNodeID = report.MakeEndpointNodeID(clientHostID, clientAddress, "54002") // also curl + server80EndpointNodeID = report.MakeEndpointNodeID(serverHostID, serverAddress, "80") // i.e. apache + unknown1EndpointNodeID = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10001") + unknown2EndpointNodeID = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10002") + unknown3EndpointNodeID = report.MakeEndpointNodeID(unknownHostID, unknownAddress, "10003") + + clientAddressNodeID = report.MakeAddressNodeID(clientHostID, clientAddress) + serverAddressNodeID = report.MakeAddressNodeID(serverHostID, serverAddress) + unknownAddressNodeID = report.MakeAddressNodeID(unknownHostID, unknownAddress) +) diff --git a/report/report_test.go b/report/report_test.go index 9852115dea..7216466110 100644 --- a/report/report_test.go +++ b/report/report_test.go @@ -1,37 +1,128 @@ -package report +package report_test import ( - "encoding/json" - "fmt" "net" + "reflect" "testing" - "time" + + "github.com/weaveworks/scope/report" ) -func TestHostJSON(t *testing.T) { - _, localNet, _ := net.ParseCIDR("192.168.1.2/16") - host := HostMetadata{ - Timestamp: time.Now(), - Hostname: "euclid", - LocalNets: []*net.IPNet{localNet}, - OS: "linux", +func TestReportCopy(t *testing.T) { + one := report.MakeReport() + two := one.Copy() + two.Merge(reportFixture) + if want, have := report.MakeReport(), one; !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) } - e, err := json.Marshal(host) - if err != nil { - t.Fatalf("Marshal error: %v", err) +} + +func TestReportLocalNetworks(t *testing.T) { + r := report.MakeReport() + r = r.Merge(report.Report{Host: report.Topology{NodeMetadatas: report.NodeMetadatas{ + "nonets": {}, + "foo": {"local_networks": "10.0.0.1/8 192.168.1.1/24 10.0.0.1/8 badnet/33"}, + }}}) + if want, have := []*net.IPNet{ + mustParseCIDR("10.0.0.1/8"), + mustParseCIDR("192.168.1.1/24"), + }, r.LocalNetworks(); !reflect.DeepEqual(want, have) { + t.Errorf("want %+v, have %+v", want, have) } +} - var hostAgain HostMetadata - err = json.Unmarshal(e, &hostAgain) - if err != nil { - t.Fatalf("Unarshal error: %v", err) +func TestReportSquash(t *testing.T) { + { + want := report.Adjacency{ + report.MakeAdjacencyID(clientHostID, client54001EndpointNodeID): report.MakeIDList(server80EndpointNodeID), + report.MakeAdjacencyID(clientHostID, client54002EndpointNodeID): report.MakeIDList(server80EndpointNodeID), + report.MakeAdjacencyID(serverHostID, server80EndpointNodeID): report.MakeIDList(client54001EndpointNodeID, client54002EndpointNodeID, report.TheInternet), + } + have := reportFixture.Squash().Endpoint.Adjacency + if !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } + } + { + want := report.Adjacency{ + report.MakeAdjacencyID(clientHostID, clientAddressNodeID): report.MakeIDList(serverAddressNodeID), + report.MakeAdjacencyID(serverHostID, serverAddressNodeID): report.MakeIDList(clientAddressNodeID, report.TheInternet), + } + have := reportFixture.Squash().Address.Adjacency + if !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } + } +} + +func TestReportEdgeMetadata(t *testing.T) { + have := reportFixture.EdgeMetadata(report.SelectEndpoint, report.ProcessName, "apache", "curl") + want := report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 100, + BytesIngress: 10, } + if !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } +} - // need to compare pointers. No fun. - want := fmt.Sprintf("%+v", host) - got := fmt.Sprintf("%+v", hostAgain) - if want != got { - t.Errorf("Host not the same. Want \n%+v, got \n%+v", want, got) +func TestReportOriginTable(t *testing.T) { + if _, ok := reportFixture.OriginTable("not-found"); ok { + t.Errorf("unknown origin ID gave unexpected success") } + for originID, want := range map[string]report.Table{ + report.MakeProcessNodeID(clientHostID, "4242"): { + Title: "Origin Process", + Numeric: false, + Rows: []report.Row{ + {"Host name", "client.host.com", ""}, + }, + }, + clientAddressNodeID: { + Title: "Origin Address", + Numeric: false, + Rows: []report.Row{ + {"Host name", "client.host.com", ""}, + }, + }, + report.MakeProcessNodeID(clientHostID, "4242"): { + Title: "Origin Process", + Numeric: false, + Rows: []report.Row{ + {"Process name", "curl", ""}, + {"PID", "4242", ""}, + {"Docker container ID", "a1b2c3d4e5", ""}, + {"Docker container name", "fixture-container", ""}, + {"Docker image ID", "0000000000", ""}, + {"Docker image name", "fixture/container:latest", ""}, + }, + }, + serverHostNodeID: { + Title: "Origin Host", + Numeric: false, + Rows: []report.Row{ + {"Host name", "server.host.com", ""}, + {"Load", "0.01 0.01 0.01", ""}, + {"Operating system", "Linux", ""}, + }, + }, + } { + have, ok := reportFixture.OriginTable(originID) + if !ok { + t.Errorf("%q: not OK", originID) + continue + } + if !reflect.DeepEqual(want, have) { + t.Errorf("%q: %s", originID, diff(want, have)) + } + } +} +func mustParseCIDR(s string) *net.IPNet { + _, ipNet, err := net.ParseCIDR(s) + if err != nil { + panic(err) + } + return ipNet } diff --git a/report/squash.go b/report/squash.go deleted file mode 100644 index 3c6e46bd2d..0000000000 --- a/report/squash.go +++ /dev/null @@ -1,59 +0,0 @@ -package report - -import ( - "net" - "strings" -) - -const ( - // TheInternet is the ID that we assign to the super-node composed of all - // remote nodes that have been squashed together. - TheInternet = "theinternet" -) - -// Squash takes a Topology, and folds all remote nodes into a supernode. -func Squash(t Topology, f IDAddresser, localNets []*net.IPNet) Topology { - newTopo := NewTopology() - isRemote := func(ip net.IP) bool { return !netsContain(localNets, ip) } - - // If any node ID on the right-hand (destination) side of an adjacency - // list is remote, rename it to TheInternet. (We'll never have remote - // nodes on the left-hand (source) side of an adjacency list, by - // definition.) - for nodeID, adjacent := range t.Adjacency { - var newAdjacency IDList - for _, adjacentID := range adjacent { - if isRemote(f(adjacentID)) { - adjacentID = TheInternet - } - newAdjacency = newAdjacency.Add(adjacentID) - } - newTopo.Adjacency[nodeID] = newAdjacency - } - - // Edge metadata keys are "|". If the dst node - // ID is remote, rename it to TheInternet. - for key, metadata := range t.EdgeMetadatas { - parts := strings.SplitN(key, IDDelim, 2) - if ip := f(parts[1]); ip != nil && isRemote(ip) { - key = parts[0] + IDDelim + TheInternet - } - - // Could be we're merging two keys into one now. - summedMetadata := newTopo.EdgeMetadatas[key] - summedMetadata.Flatten(metadata) - newTopo.EdgeMetadatas[key] = summedMetadata - } - - newTopo.NodeMetadatas = t.NodeMetadatas - return newTopo -} - -func netsContain(nets []*net.IPNet, ip net.IP) bool { - for _, net := range nets { - if net.Contains(ip) { - return true - } - } - return false -} diff --git a/report/squash_test.go b/report/squash_test.go deleted file mode 100644 index 2e5cf4abab..0000000000 --- a/report/squash_test.go +++ /dev/null @@ -1,256 +0,0 @@ -package report - -import ( - "net" - "reflect" - "testing" -) - -var ( - _, netdot1, _ = net.ParseCIDR("192.168.1.0/24") - _, netdot2, _ = net.ParseCIDR("192.168.2.0/24") -) - -func reportToSquash() Report { - return Report{ - Process: Topology{ - Adjacency: Adjacency{ - "hostA|;192.168.1.1;12345": []string{";192.168.1.2;80"}, - "hostA|;192.168.1.1;8888": []string{";1.2.3.4;22", ";1.2.3.4;23"}, - "hostB|;192.168.1.2;80": []string{";192.168.1.1;12345"}, - "hostZ|;192.168.2.2;80": []string{";192.168.1.1;12345"}, - }, - EdgeMetadatas: EdgeMetadatas{ - ";192.168.1.1;12345|;192.168.1.2;80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - }, - ";192.168.1.1;8888|;1.2.3.4;22": EdgeMetadata{ - WithBytes: true, - BytesEgress: 200, - BytesIngress: 0, - }, - ";192.168.1.1;8888|;1.2.3.4;23": EdgeMetadata{ - WithBytes: true, - BytesEgress: 200, - BytesIngress: 0, - }, - ";192.168.1.2;80|;192.168.1.1;12345": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - ";192.168.2.2;80|;192.168.1.1;12345": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - }, - NodeMetadatas: NodeMetadatas{ - ";192.168.1.1;12345": NodeMetadata{ - "pid": "23128", - "name": "curl", - "domain": "node-a.local", - }, - ";192.168.1.1;8888": NodeMetadata{ - "pid": "55100", - "name": "ssh", - "domain": "node-a.local", - }, - ";192.168.1.2;80": NodeMetadata{ - "pid": "215", - "name": "apache", - "domain": "node-b.local", - }, - ";192.168.2.2;80": NodeMetadata{ - "pid": "213", - "name": "apache", - "domain": "node-z.local", - }, - }, - }, - - Network: Topology{ - Adjacency: Adjacency{ - "hostA|;192.168.1.1": []string{";192.168.1.2", ";1.2.3.4"}, - "hostB|;192.168.1.2": []string{";192.168.1.1"}, - "hostZ|;192.168.2.2": []string{";192.168.1.1"}, - }, - EdgeMetadatas: EdgeMetadatas{ - ";192.168.1.1|;192.168.1.2": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - }, - ";192.168.1.1|;1.2.3.4": EdgeMetadata{ - WithBytes: true, - BytesEgress: 200, - BytesIngress: 0, - }, - ";192.168.1.2|;192.168.1.1": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - ";192.168.2.2|;192.168.1.1": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - }, - NodeMetadatas: NodeMetadatas{ - ";192.168.1.1": NodeMetadata{ - "name": "host-a", - }, - ";192.168.1.2": NodeMetadata{ - "name": "host-b", - }, - ";192.168.2.2": NodeMetadata{ - "name": "host-z", - }, - }, - }, - - HostMetadatas: HostMetadatas{ - "hostA": HostMetadata{ - Hostname: "node-a.local", - OS: "Linux", - LocalNets: []*net.IPNet{netdot1}, - }, - "hostB": HostMetadata{ - Hostname: "node-b.local", - OS: "Linux", - LocalNets: []*net.IPNet{netdot1}, - }, - "hostZ": HostMetadata{ - Hostname: "node-z.local", - OS: "Linux", - LocalNets: []*net.IPNet{netdot2}, - }, - }, - } -} - -func TestSquashTopology(t *testing.T) { - // Tests just a topology - want := Topology{ - Adjacency: Adjacency{ - "hostA|;192.168.1.1;12345": []string{";192.168.1.2;80"}, - "hostA|;192.168.1.1;8888": []string{"theinternet"}, - "hostB|;192.168.1.2;80": []string{";192.168.1.1;12345"}, - "hostZ|;192.168.2.2;80": []string{";192.168.1.1;12345"}, - }, - EdgeMetadatas: EdgeMetadatas{ - ";192.168.1.1;12345|;192.168.1.2;80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - }, - ";192.168.1.1;8888|theinternet": EdgeMetadata{ - WithBytes: true, - BytesEgress: 2 * 200, - BytesIngress: 2 * 0, - }, - ";192.168.1.2;80|;192.168.1.1;12345": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - ";192.168.2.2;80|;192.168.1.1;12345": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - }, - NodeMetadatas: reportToSquash().Process.NodeMetadatas, - } - - have := Squash(reportToSquash().Process, AddressIPPort, reportToSquash().HostMetadatas.LocalNets()) - if !reflect.DeepEqual(want, have) { - t.Errorf("want\n\t%#v, have\n\t%#v", want, have) - } -} - -func TestSquashReport(t *testing.T) { - // Tests a full report squash. - want := Report{ - Process: Topology{ - Adjacency: Adjacency{ - "hostA|;192.168.1.1;12345": []string{";192.168.1.2;80"}, - "hostA|;192.168.1.1;8888": []string{"theinternet"}, - "hostB|;192.168.1.2;80": []string{";192.168.1.1;12345"}, - "hostZ|;192.168.2.2;80": []string{";192.168.1.1;12345"}, - }, - EdgeMetadatas: EdgeMetadatas{ - ";192.168.1.1;12345|;192.168.1.2;80": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - }, - ";192.168.1.1;8888|theinternet": EdgeMetadata{ - WithBytes: true, - BytesEgress: 2 * 200, - BytesIngress: 2 * 0, - }, - ";192.168.1.2;80|;192.168.1.1;12345": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - ";192.168.2.2;80|;192.168.1.1;12345": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - }, - NodeMetadatas: reportToSquash().Process.NodeMetadatas, - }, - Network: Topology{ - Adjacency: Adjacency{ - "hostA|;192.168.1.1": []string{";192.168.1.2", "theinternet"}, - "hostB|;192.168.1.2": []string{";192.168.1.1"}, - "hostZ|;192.168.2.2": []string{";192.168.1.1"}, - }, - EdgeMetadatas: EdgeMetadatas{ - ";192.168.1.1|;192.168.1.2": EdgeMetadata{ - WithBytes: true, - BytesEgress: 12, - BytesIngress: 0, - }, - ";192.168.1.1|theinternet": EdgeMetadata{ - WithBytes: true, - BytesEgress: 200, - BytesIngress: 0, - }, - ";192.168.1.2|;192.168.1.1": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - ";192.168.2.2|;192.168.1.1": EdgeMetadata{ - WithBytes: true, - BytesEgress: 0, - BytesIngress: 12, - }, - }, - NodeMetadatas: NodeMetadatas{ - ";192.168.1.1": NodeMetadata{ - "name": "host-a", - }, - ";192.168.1.2": NodeMetadata{ - "name": "host-b", - }, - ";192.168.2.2": NodeMetadata{ - "name": "host-z", - }, - }, - }, - HostMetadatas: reportToSquash().HostMetadatas, - } - - have := reportToSquash().SquashRemote() - if !reflect.DeepEqual(want, have) { - t.Errorf("want\n\t%#v, have\n\t%#v", want, have) - } -} diff --git a/report/topology.go b/report/topology.go index d9b4541551..ea2382315a 100644 --- a/report/topology.go +++ b/report/topology.go @@ -1,209 +1,289 @@ package report -import ( - "reflect" - "strings" -) - -const ( - // ScopeDelim separates the scope portion of an address from the address - // string itself. - ScopeDelim = ";" - - // IDDelim separates fields in a node ID. - IDDelim = "|" - - localUnknown = "localUnknown" -) +import "net" -// Topology describes a specific view of a network. It consists of nodes and -// edges, represented by Adjacency, and metadata about those nodes and edges, -// represented by EdgeMetadatas and NodeMetadatas respectively. +// Topology represents a directed graph of nodes and edges. This is the core +// data type, made of directly observed (via the probe) nodes, and their +// connections. Not to be confused with an API topology, which is made of +// renderable nodes, and produced by passing a report (i.e. multiple +// topologies) through a rendering transformation. type Topology struct { Adjacency - EdgeMetadatas NodeMetadatas + EdgeMetadatas } -// Adjacency is an adjacency-list encoding of the topology. Keys are node IDs, -// as produced by the relevant MappingFunc for the topology. -type Adjacency map[string]IDList +// MakeTopology produces a new topology, ready for use. It's the only correct +// way to produce topologies for general use, so please use it. +func MakeTopology() Topology { + return Topology{ + Adjacency: Adjacency{}, + NodeMetadatas: NodeMetadatas{}, + EdgeMetadatas: EdgeMetadatas{}, + } +} -// EdgeMetadatas collect metadata about each edge in a topology. Keys are a -// concatenation of node IDs. -type EdgeMetadatas map[string]EdgeMetadata +// Copy returns a value copy, useful for tests. +func (t Topology) Copy() Topology { + return Topology{ + Adjacency: t.Adjacency.Copy(), + NodeMetadatas: t.NodeMetadatas.Copy(), + EdgeMetadatas: t.EdgeMetadatas.Copy(), + } +} -// NodeMetadatas collect metadata about each node in a topology. Keys are node -// IDs. -type NodeMetadatas map[string]NodeMetadata +// Merge merges two topologies together, returning the result. Always reassign +// the result of merge to the destination. Merge is defined on topology as a +// value-type, but topology contains reference fields, so if you want to +// maintain immutability, use copy. +func (t Topology) Merge(other Topology) Topology { + t.Adjacency = t.Adjacency.Merge(other.Adjacency) + t.NodeMetadatas = t.NodeMetadatas.Merge(other.NodeMetadatas) + t.EdgeMetadatas = t.EdgeMetadatas.Merge(other.EdgeMetadatas) + return t +} -// EdgeMetadata describes a superset of the metadata that probes can -// conceivably (and usefully) collect about an edge between two nodes in any -// topology. -type EdgeMetadata struct { - WithBytes bool `json:"with_bytes,omitempty"` - BytesIngress uint `json:"bytes_ingress,omitempty"` // dst -> src - BytesEgress uint `json:"bytes_egress,omitempty"` // src -> dst +// Squash squashes all non-local nodes in the topology to a super-node called +// the Internet. +func (t Topology) Squash(f IDAddresser, localNets []*net.IPNet) Topology { + isRemote := func(ip net.IP) bool { return !netsContain(localNets, ip) } + for srcID, dstIDs := range t.Adjacency { + newDstIDs := make(IDList, 0, len(dstIDs)) + for _, dstID := range dstIDs { + if ip := f(dstID); ip != nil && isRemote(ip) { + dstID = TheInternet + } + newDstIDs = newDstIDs.Add(dstID) + } + t.Adjacency[srcID] = newDstIDs + } + return t +} - WithConnCountTCP bool `json:"with_conn_count_tcp,omitempty"` - MaxConnCountTCP uint `json:"max_conn_count_tcp,omitempty"` +func netsContain(nets []*net.IPNet, ip net.IP) bool { + for _, net := range nets { + if net.Contains(ip) { + return true + } + } + return false } -// NodeMetadata describes a superset of the metadata that probes can collect -// about a given node in a given topology. Right now it's a weakly-typed map, -// which should probably change (see comment on type MapFunc). -type NodeMetadata map[string]string +// Adjacency represents an adjacency list. +type Adjacency map[string]IDList -// NewTopology gives you a Topology. -func NewTopology() Topology { - return Topology{ - Adjacency: map[string]IDList{}, - EdgeMetadatas: map[string]EdgeMetadata{}, - NodeMetadatas: map[string]NodeMetadata{}, +// Copy returns a value copy, useful for tests. +func (a Adjacency) Copy() Adjacency { + cp := make(Adjacency, len(a)) + for id, idList := range a { + cp[id] = idList.Copy() } + return cp } -// RenderBy transforms a given Topology into a set of RenderableNodes, which -// the UI will render collectively as a graph. Note that a RenderableNode will -// always be rendered with other nodes, and therefore contains limited detail. -// -// RenderBy takes a a MapFunc, which defines how to group and label nodes. Npdes -// with the same mapped IDs will be merged. -func (t Topology) RenderBy(mapFunc MapFunc, pseudoFunc PseudoFunc) map[string]RenderableNode { - nodes := map[string]RenderableNode{} - - // Build a set of RenderableNodes for all non-pseudo probes, and an - // addressID to nodeID lookup map. Multiple addressIDs can map to the same - // RenderableNodes. - address2mapped := map[string]string{} - for addressID, metadata := range t.NodeMetadatas { - mapped, ok := mapFunc(addressID, metadata) - if !ok { - continue - } +// Merge merges two adjacencies together, returning the union set of IDs. +// Always reassign the result of merge to the destination. Merge is defined +// on adjacency as a value-type, but adjacency is itself a reference type, so +// if you want to maintain immutability, use copy. +func (a Adjacency) Merge(other Adjacency) Adjacency { + for id, idList := range other { + a[id] = a[id].Add(idList...) + } + return a +} - // mapped.ID needs not be unique over all addressIDs. If not, we just overwrite - // the existing data, on the assumption that the MapFunc returns the same - // data. - nodes[mapped.ID] = RenderableNode{ - ID: mapped.ID, - LabelMajor: mapped.Major, - LabelMinor: mapped.Minor, - Rank: mapped.Rank, - Pseudo: false, - Adjacency: IDList{}, // later - OriginHosts: IDList{}, // later - OriginNodes: IDList{}, // later - Metadata: AggregateMetadata{}, // later - } - address2mapped[addressID] = mapped.ID - } - - // Walk the graph and make connections. - for src, dsts := range t.Adjacency { - var ( - fields = strings.SplitN(src, IDDelim, 2) // "|
" - srcOriginHostID = fields[0] - srcNodeAddress = fields[1] - srcRenderableID = address2mapped[srcNodeAddress] // must exist - srcRenderableNode = nodes[srcRenderableID] // must exist - ) - - for _, dstNodeAddress := range dsts { - dstRenderableID, ok := address2mapped[dstNodeAddress] - if !ok { - pseudoNode, ok := pseudoFunc(srcNodeAddress, srcRenderableNode, dstNodeAddress) - if !ok { - continue - } - dstRenderableID = pseudoNode.ID - nodes[dstRenderableID] = RenderableNode{ - ID: pseudoNode.ID, - LabelMajor: pseudoNode.Major, - LabelMinor: pseudoNode.Minor, - Pseudo: true, - Metadata: AggregateMetadata{}, // populated below - or not? - } - address2mapped[dstNodeAddress] = dstRenderableID - } +// NodeMetadatas represents multiple node metadatas, keyed by node ID. +type NodeMetadatas map[string]NodeMetadata - srcRenderableNode.Adjacency = srcRenderableNode.Adjacency.Add(dstRenderableID) - srcRenderableNode.OriginHosts = srcRenderableNode.OriginHosts.Add(srcOriginHostID) - srcRenderableNode.OriginNodes = srcRenderableNode.OriginNodes.Add(srcNodeAddress) - edgeID := srcNodeAddress + IDDelim + dstNodeAddress - if md, ok := t.EdgeMetadatas[edgeID]; ok { - srcRenderableNode.Metadata.Merge(md.Transform()) - } - } +// Copy returns a value copy, useful for tests. +func (nms NodeMetadatas) Copy() NodeMetadatas { + cp := make(NodeMetadatas, len(nms)) + for id, nm := range nms { + cp[id] = nm.Copy() + } + return cp +} - nodes[srcRenderableID] = srcRenderableNode +// Merge merges two node metadata collections together, returning a semantic +// union set of metadatas. In the cases where keys conflict within an +// individual node metadata map, the other (right-hand) side wins. Always +// reassign the result of merge to the destination. Merge is defined on the +// value-type, but node metadata collection is itself a reference type, so if +// you want to maintain immutability, use copy. +func (nms NodeMetadatas) Merge(other NodeMetadatas) NodeMetadatas { + for id, md := range other { + if _, ok := nms[id]; !ok { + nms[id] = NodeMetadata{} + } + nms[id] = nms[id].Merge(md) } + return nms +} + +// EdgeMetadatas represents multiple edge metadatas, keyed by edge ID. +type EdgeMetadatas map[string]EdgeMetadata - return nodes +// Copy returns a value copy, useful for tests. +func (ems EdgeMetadatas) Copy() EdgeMetadatas { + cp := make(EdgeMetadatas, len(ems)) + for id, em := range ems { + cp[id] = em.Copy() + } + return cp } -// EdgeMetadata gives the metadata of an edge from the perspective of the -// srcRenderableID. Since an edgeID can have multiple edges on the address -// level, it uses the supplied mapping function to translate address IDs to -// renderable node (mapped) IDs. -func (t Topology) EdgeMetadata(mapFunc MapFunc, srcRenderableID, dstRenderableID string) EdgeMetadata { - metadata := EdgeMetadata{} - for edgeID, edgeMeta := range t.EdgeMetadatas { - edgeParts := strings.SplitN(edgeID, IDDelim, 2) - src := edgeParts[0] - if src != TheInternet { - mapped, _ := mapFunc(src, t.NodeMetadatas[src]) - src = mapped.ID - } - dst := edgeParts[1] - if dst != TheInternet { - mapped, _ := mapFunc(dst, t.NodeMetadatas[dst]) - dst = mapped.ID - } - if src == srcRenderableID && dst == dstRenderableID { - metadata.Flatten(edgeMeta) +// Merge merges two edge metadata collections together, returning a logical +// union set of metadatas. Always reassign the result of merge to the +// destination. Merge is defined on the value-type, but edge metadata +// collection is itself a reference type, so if you want to maintain +// immutability, use copy. +func (ems EdgeMetadatas) Merge(other EdgeMetadatas) EdgeMetadatas { + for id, md := range other { + if _, ok := ems[id]; !ok { + ems[id] = EdgeMetadata{} } + ems[id] = ems[id].Merge(md) } - return metadata + return ems } -// Diff is returned by TopoDiff. It represents the changes between two -// RenderableNode maps. -type Diff struct { - Add []RenderableNode `json:"add"` - Update []RenderableNode `json:"update"` - Remove []string `json:"remove"` +// NodeMetadata is a simple string-to-string map. +type NodeMetadata map[string]string + +// Copy returns a value copy, useful for tests. +func (nm NodeMetadata) Copy() NodeMetadata { + cp := make(NodeMetadata, len(nm)) + for k, v := range nm { + cp[k] = v + } + return cp } -// TopoDiff gives you the diff to get from A to B. -func TopoDiff(a, b map[string]RenderableNode) Diff { - diff := Diff{} +// Merge merges two node metadata maps together. In case of conflict, the +// other (right-hand) side wins. Always reassign the result of merge to the +// destination. Merge is defined on the value-type, but node metadata map is +// itself a reference type, so if you want to maintain immutability, use copy. +func (nm NodeMetadata) Merge(other NodeMetadata) NodeMetadata { + for k, v := range other { + nm[k] = v // other takes precedence + } + return nm +} - notSeen := map[string]struct{}{} - for k := range a { - notSeen[k] = struct{}{} +// GetDefault returns the value for key, or def if key is not defined. +func (nm NodeMetadata) GetDefault(key, def string) string { + val, ok := nm[key] + if !ok { + return def } + return val +} + +// EdgeMetadata represents aggregatable information about a specific edge +// between two nodes. EdgeMetadata is frequently merged; be careful to think +// about merge semantics when modifying this structure. +type EdgeMetadata struct { + WithBytes bool `json:"with_bytes,omitempty"` + BytesEgress uint `json:"bytes_egress,omitempty"` // src -> dst + BytesIngress uint `json:"bytes_ingress,omitempty"` // src <- dst + + WithConnCountTCP bool `json:"with_conn_count_tcp,omitempty"` + MaxConnCountTCP uint `json:"max_conn_count_tcp,omitempty"` +} + +// Copy returns a value copy, useful for tests. It actually doesn't do +// anything here, since EdgeMetadata has no reference fields. But we keep it, +// for API consistency. +func (em EdgeMetadata) Copy() EdgeMetadata { + return em +} - for k, node := range b { - if _, ok := a[k]; !ok { - diff.Add = append(diff.Add, node) - } else if !reflect.DeepEqual(node, a[k]) { - diff.Update = append(diff.Update, node) +// Merge merges two edge metadata structs together. This is an important +// operation, so please think carefully when adding things here. Always +// reassign the result of merge to the destination, as merge is defined on a +// value-type, and there are no reference types here. +func (em EdgeMetadata) Merge(other EdgeMetadata) EdgeMetadata { + if other.WithBytes { + em.WithBytes = true + em.BytesIngress += other.BytesIngress + em.BytesEgress += other.BytesEgress + } + if other.WithConnCountTCP { + em.WithConnCountTCP = true + if other.MaxConnCountTCP > em.MaxConnCountTCP { + em.MaxConnCountTCP = other.MaxConnCountTCP } - delete(notSeen, k) } + return em +} - // leftover keys - for k := range notSeen { - diff.Remove = append(diff.Remove, k) +// Flatten sums two EdgeMetadatas. They must represent the same window of +// time, i.e. both EdgeMetadatas should be from the same report. +func (em EdgeMetadata) Flatten(other EdgeMetadata) EdgeMetadata { + if other.WithBytes { + em.WithBytes = true + em.BytesIngress += other.BytesIngress + em.BytesEgress += other.BytesEgress + } + if other.WithConnCountTCP { + em.WithConnCountTCP = true + // Note that summing of two maximums doesn't always give the true + // maximum. But it's our Best-Effort effort. + em.MaxConnCountTCP += other.MaxConnCountTCP } + return em +} - return diff +// Export transforms an EdgeMetadata to an AggregateMetadata. +func (em EdgeMetadata) Export() AggregateMetadata { + amd := AggregateMetadata{} + if em.WithBytes { + amd[KeyBytesIngress] = int(em.BytesIngress) + amd[KeyBytesEgress] = int(em.BytesEgress) + } + if em.WithConnCountTCP { + // The maximum is the maximum. No need to calculate anything. + amd[KeyMaxConnCountTCP] = int(em.MaxConnCountTCP) + } + return amd } -// ByID is a sort interface for a RenderableNode slice. -type ByID []RenderableNode +// AggregateMetadata is a composable version of an EdgeMetadata. It's used +// when we want to merge nodes/edges for any reason. +// +// It takes its data from EdgeMetadatas, but we can apply it to nodes, by +// summing up (flattening) all of the {ingress, egress} metadatas of the +// {incoming, outgoing} edges to the node. +type AggregateMetadata map[string]int + +const ( + // KeyBytesIngress is the aggregate metadata key for the total count of + // ingress bytes. + KeyBytesIngress = "ingress_bytes" + // KeyBytesEgress is the aggregate metadata key for the total count of + // egress bytes. + KeyBytesEgress = "egress_bytes" + // KeyMaxConnCountTCP is the aggregate metadata key for the maximum number + // of simultaneous observed TCP connections in the window. + KeyMaxConnCountTCP = "max_conn_count_tcp" +) + +// Copy returns a value copy, useful for tests. +func (amd AggregateMetadata) Copy() AggregateMetadata { + cp := make(AggregateMetadata, len(amd)) + for k, v := range amd { + cp[k] = v + } + return cp +} -func (r ByID) Len() int { return len(r) } -func (r ByID) Swap(i, j int) { r[i], r[j] = r[j], r[i] } -func (r ByID) Less(i, j int) bool { return r[i].ID < r[j].ID } +// Merge merges two aggregate metadatas together. Always reassign the result +// of merge to the destination. Merge is defined on the value-type, but +// aggregate metadata is itself a reference type, so if you want to maintain +// immutability, use copy. +func (amd AggregateMetadata) Merge(other AggregateMetadata) AggregateMetadata { + for k, v := range other { + amd[k] += v + } + return amd +} diff --git a/report/topology_test.go b/report/topology_test.go index 9a6d8ff267..82f4a9e4e1 100644 --- a/report/topology_test.go +++ b/report/topology_test.go @@ -1,405 +1,165 @@ -package report +package report_test import ( "reflect" - "sort" "testing" - "github.com/davecgh/go-spew/spew" - "github.com/pmezard/go-difflib/difflib" + "github.com/weaveworks/scope/report" ) -func init() { - spew.Config.SortKeys = true // :\ -} - -const ( - client54001 = ScopeDelim + "10.10.10.20" + ScopeDelim + "54001" // curl (1) - client54002 = ScopeDelim + "10.10.10.20" + ScopeDelim + "54002" // curl (2) - unknownClient1 = ScopeDelim + "10.10.10.10" + ScopeDelim + "54010" // we want to ensure two unknown clients, connnected - unknownClient2 = ScopeDelim + "10.10.10.10" + ScopeDelim + "54020" // to the same server, are deduped. - unknownClient3 = ScopeDelim + "10.10.10.11" + ScopeDelim + "54020" // Check this one isn't deduped - server80 = ScopeDelim + "192.168.1.1" + ScopeDelim + "80" // apache - - clientIP = ScopeDelim + "10.10.10.20" - serverIP = ScopeDelim + "192.168.1.1" - randomIP = ScopeDelim + "172.16.11.9" // only in Network topology - unknownIP = ScopeDelim + "10.10.10.10" -) - -var ( - report = Report{ - Process: Topology{ - Adjacency: Adjacency{ - "client.hostname.com" + IDDelim + client54001: NewIDList(server80), - "client.hostname.com" + IDDelim + client54002: NewIDList(server80), - "server.hostname.com" + IDDelim + server80: NewIDList(client54001, client54002, unknownClient1, unknownClient2, unknownClient3), - }, - NodeMetadatas: NodeMetadatas{ - // NodeMetadata is arbitrary. We're free to put only precisely what we - // care to test into the fixture. Just be sure to include the bits - // that the mapping funcs extract :) - client54001: NodeMetadata{ - "name": "curl", - "domain": "client-54001-domain", - "pid": "10001", - }, - client54002: NodeMetadata{ - "name": "curl", // should be same as above! - "domain": "client-54002-domain", // may be different than above - "pid": "10001", // should be same as above! - }, - server80: NodeMetadata{ - "name": "apache", - "domain": "server-80-domain", - "pid": "215", - }, - }, - EdgeMetadatas: EdgeMetadatas{ - client54001 + IDDelim + server80: EdgeMetadata{ - WithBytes: true, - BytesIngress: 100, - BytesEgress: 10, - }, - client54002 + IDDelim + server80: EdgeMetadata{ - WithBytes: true, - BytesIngress: 200, - BytesEgress: 20, - }, - - server80 + IDDelim + client54001: EdgeMetadata{ - WithBytes: true, - BytesIngress: 10, - BytesEgress: 100, - }, - server80 + IDDelim + client54002: EdgeMetadata{ - WithBytes: true, - BytesIngress: 20, - BytesEgress: 200, - }, - server80 + IDDelim + unknownClient1: EdgeMetadata{ - WithBytes: true, - BytesIngress: 30, - BytesEgress: 300, - }, - server80 + IDDelim + unknownClient2: EdgeMetadata{ - WithBytes: true, - BytesIngress: 40, - BytesEgress: 400, - }, - server80 + IDDelim + unknownClient3: EdgeMetadata{ - WithBytes: true, - BytesIngress: 50, - BytesEgress: 500, - }, - }, - }, - Network: Topology{ - Adjacency: Adjacency{ - "client.hostname.com" + IDDelim + clientIP: NewIDList(serverIP), - "random.hostname.com" + IDDelim + randomIP: NewIDList(serverIP), - "server.hostname.com" + IDDelim + serverIP: NewIDList(clientIP, unknownIP), // no backlink to random - }, - NodeMetadatas: NodeMetadatas{ - clientIP: NodeMetadata{ - "name": "client.hostname.com", // hostname - }, - randomIP: NodeMetadata{ - "name": "random.hostname.com", // hostname - }, - serverIP: NodeMetadata{ - "name": "server.hostname.com", // hostname - }, - }, - EdgeMetadatas: EdgeMetadatas{ - clientIP + IDDelim + serverIP: EdgeMetadata{ - WithConnCountTCP: true, - MaxConnCountTCP: 3, - }, - randomIP + IDDelim + serverIP: EdgeMetadata{ - WithConnCountTCP: true, - MaxConnCountTCP: 20, // dangling connections, weird but possible - }, - serverIP + IDDelim + clientIP: EdgeMetadata{ - WithConnCountTCP: true, - MaxConnCountTCP: 3, - }, - serverIP + IDDelim + unknownIP: EdgeMetadata{ - WithConnCountTCP: true, - MaxConnCountTCP: 7, - }, - }, - }, +func TestGetDefault(t *testing.T) { + md := report.NodeMetadata{"a": "1"} + if want, have := "1", md.GetDefault("a", "2"); want != have { + t.Errorf("want %q, have %q", want, have) } -) + if want, have := "2", md.GetDefault("b", "2"); want != have { + t.Errorf("want %q, have %q", want, have) + } +} -func TestRenderByProcessPID(t *testing.T) { - want := map[string]RenderableNode{ - "pid:client-54001-domain:10001": { - ID: "pid:client-54001-domain:10001", - LabelMajor: "curl", - LabelMinor: "client-54001-domain (10001)", - Rank: "10001", - Pseudo: false, - Adjacency: NewIDList("pid:server-80-domain:215"), - OriginHosts: NewIDList("client.hostname.com"), - OriginNodes: NewIDList(";10.10.10.20;54001"), - Metadata: AggregateMetadata{ - KeyBytesIngress: 100, - KeyBytesEgress: 10, - }, - }, - "pid:client-54002-domain:10001": { - ID: "pid:client-54002-domain:10001", - LabelMajor: "curl", - LabelMinor: "client-54002-domain (10001)", - Rank: "10001", // same process - Pseudo: false, - Adjacency: NewIDList("pid:server-80-domain:215"), - OriginHosts: NewIDList("client.hostname.com"), - OriginNodes: NewIDList(";10.10.10.20;54002"), - Metadata: AggregateMetadata{ - KeyBytesIngress: 200, - KeyBytesEgress: 20, +func TestTopologyMerge(t *testing.T) { + want := report.Topology{ + Adjacency: report.Adjacency{ + report.MakeAdjacencyID(clientHostID, client54001EndpointNodeID): report.MakeIDList(server80EndpointNodeID), + report.MakeAdjacencyID(fooHostID, foo42001EndpointNodeID): report.MakeIDList(server80EndpointNodeID), + }, + NodeMetadatas: report.NodeMetadatas{ + client54001EndpointNodeID: report.NodeMetadata{}, + foo42001EndpointNodeID: report.NodeMetadata{}, + }, + EdgeMetadatas: report.EdgeMetadatas{ + report.MakeEdgeID(client54001EndpointNodeID, server80EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 30, + BytesIngress: 300, + WithConnCountTCP: true, + MaxConnCountTCP: 9, + }, + report.MakeEdgeID(foo42001EndpointNodeID, server80EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 40, + BytesIngress: 400, + WithConnCountTCP: false, }, }, - "pid:server-80-domain:215": { - ID: "pid:server-80-domain:215", - LabelMajor: "apache", - LabelMinor: "server-80-domain (215)", - Rank: "215", - Pseudo: false, - Adjacency: NewIDList( - "pid:client-54001-domain:10001", - "pid:client-54002-domain:10001", - "pseudo:;10.10.10.10;192.168.1.1;80", - "pseudo:;10.10.10.11;192.168.1.1;80", - ), - OriginHosts: NewIDList("server.hostname.com"), - OriginNodes: NewIDList(";192.168.1.1;80"), - Metadata: AggregateMetadata{ - KeyBytesIngress: 150, - KeyBytesEgress: 1500, - }, - }, - "pseudo:;10.10.10.10;192.168.1.1;80": { - ID: "pseudo:;10.10.10.10;192.168.1.1;80", - LabelMajor: "10.10.10.10", - Pseudo: true, - Metadata: AggregateMetadata{}, - }, - "pseudo:;10.10.10.11;192.168.1.1;80": { - ID: "pseudo:;10.10.10.11;192.168.1.1;80", - LabelMajor: "10.10.10.11", - Pseudo: true, - Metadata: AggregateMetadata{}, - }, } - have := report.Process.RenderBy(ProcessPID, GenericPseudoNode) + have := topologyFixtureA.Copy().Merge(topologyFixtureB) if !reflect.DeepEqual(want, have) { - t.Error("\n" + diff(want, have)) + t.Errorf(diff(want, have)) } } -func TestRenderByProcessPIDGrouped(t *testing.T) { - // For grouped, I've somewhat arbitrarily chosen to squash together all - // processes with the same name by removing the PID and domain (host) - // dimensions from the ID. That could be changed. - want := map[string]RenderableNode{ - "curl": { - ID: "curl", - LabelMajor: "curl", - LabelMinor: "", - Rank: "curl", - Pseudo: false, - Adjacency: NewIDList("apache"), - OriginHosts: NewIDList("client.hostname.com"), - OriginNodes: NewIDList(";10.10.10.20;54001", ";10.10.10.20;54002"), - Metadata: AggregateMetadata{ - KeyBytesIngress: 300, - KeyBytesEgress: 30, - }, - }, - "apache": { - ID: "apache", - LabelMajor: "apache", - LabelMinor: "", - Rank: "apache", - Pseudo: false, - Adjacency: NewIDList( - "curl", - "pseudo:;10.10.10.10;apache", - "pseudo:;10.10.10.11;apache", - ), - OriginHosts: NewIDList("server.hostname.com"), - OriginNodes: NewIDList(";192.168.1.1;80"), - Metadata: AggregateMetadata{ - KeyBytesIngress: 150, - KeyBytesEgress: 1500, - }, - }, - "pseudo:;10.10.10.10;apache": { - ID: "pseudo:;10.10.10.10;apache", - LabelMajor: "10.10.10.10", - Pseudo: true, - Metadata: AggregateMetadata{}, - }, - "pseudo:;10.10.10.11;apache": { - ID: "pseudo:;10.10.10.11;apache", - LabelMajor: "10.10.10.11", - Pseudo: true, - Metadata: AggregateMetadata{}, - }, +func TestAdjacencyCopy(t *testing.T) { + one := report.Adjacency{"a": report.MakeIDList("b", "c")} + two := one.Copy() + one["a"].Add("d") + if want, have := (report.Adjacency{"a": report.MakeIDList("b", "c")}), two; !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) } - have := report.Process.RenderBy(ProcessName, GenericGroupedPseudoNode) - if !reflect.DeepEqual(want, have) { - t.Error("\n" + diff(want, have)) +} + +func TestNodeMetadatasCopy(t *testing.T) { + one := report.NodeMetadatas{"a": report.NodeMetadata{"b": "c"}} + two := one.Copy() + one["a"].Merge(report.NodeMetadata{"d": "e"}) + if want, have := (report.NodeMetadatas{"a": report.NodeMetadata{"b": "c"}}), two; !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) } } -func TestRenderByNetworkHostname(t *testing.T) { - want := map[string]RenderableNode{ - "host:client.hostname.com": { - ID: "host:client.hostname.com", - LabelMajor: "client", // before first . - LabelMinor: "hostname.com", // after first . - Rank: "client", - Pseudo: false, - Adjacency: NewIDList("host:server.hostname.com"), - OriginHosts: NewIDList("client.hostname.com"), - OriginNodes: NewIDList(";10.10.10.20"), - Metadata: AggregateMetadata{ - KeyMaxConnCountTCP: 3, - }, - }, - "host:random.hostname.com": { - ID: "host:random.hostname.com", - LabelMajor: "random", // before first . - LabelMinor: "hostname.com", // after first . - Rank: "random", - Pseudo: false, - Adjacency: NewIDList("host:server.hostname.com"), - OriginHosts: NewIDList("random.hostname.com"), - OriginNodes: NewIDList(";172.16.11.9"), - Metadata: AggregateMetadata{ - KeyMaxConnCountTCP: 20, - }, - }, - "host:server.hostname.com": { - ID: "host:server.hostname.com", - LabelMajor: "server", // before first . - LabelMinor: "hostname.com", // after first . - Rank: "server", - Pseudo: false, - Adjacency: NewIDList("host:client.hostname.com", "pseudo:;10.10.10.10;192.168.1.1;"), - OriginHosts: NewIDList("server.hostname.com"), - OriginNodes: NewIDList(";192.168.1.1"), - Metadata: AggregateMetadata{ - KeyMaxConnCountTCP: 10, - }, - }, - "pseudo:;10.10.10.10;192.168.1.1;": { - ID: "pseudo:;10.10.10.10;192.168.1.1;", - LabelMajor: "10.10.10.10", - LabelMinor: "", // after first . - Rank: "", - Pseudo: true, - Adjacency: nil, - OriginHosts: nil, - OriginNodes: nil, - Metadata: AggregateMetadata{}, - }, +func TestEdgeMetadatasCopy(t *testing.T) { + one := report.EdgeMetadatas{"a": report.EdgeMetadata{WithBytes: true}} + two := one.Copy() + one["a"].Merge(report.EdgeMetadata{WithConnCountTCP: true}) + if want, have := (report.EdgeMetadatas{"a": report.EdgeMetadata{WithBytes: true}}), two; !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) + } +} + +func TestEdgeMetadataFlatten(t *testing.T) { + want := report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 30, + BytesIngress: 300, + WithConnCountTCP: true, + MaxConnCountTCP: 11, // not 9! } - have := report.Network.RenderBy(NetworkHostname, GenericPseudoNode) + var ( + edgeID = report.MakeEdgeID(client54001EndpointNodeID, server80EndpointNodeID) + have = topologyFixtureA.EdgeMetadatas[edgeID].Flatten(topologyFixtureB.EdgeMetadatas[edgeID]) + ) if !reflect.DeepEqual(want, have) { - t.Error("\n" + diff(want, have)) + t.Error(diff(want, have)) } } -func TestTopoDiff(t *testing.T) { - nodea := RenderableNode{ - ID: "nodea", - LabelMajor: "Node A", - LabelMinor: "'ts an a", - Pseudo: false, - Adjacency: []string{ - "nodeb", - }, - } - nodeap := nodea - nodeap.Adjacency = []string{ - "nodeb", - "nodeq", // not the same anymore +func TestEdgeMetadataExport(t *testing.T) { + want := report.AggregateMetadata{ + report.KeyBytesEgress: 10, + report.KeyBytesIngress: 100, + report.KeyMaxConnCountTCP: 2, } - nodeb := RenderableNode{ - ID: "nodeb", - LabelMajor: "Node B", + edgeID := report.MakeEdgeID(client54001EndpointNodeID, server80EndpointNodeID) + have := topologyFixtureA.EdgeMetadatas[edgeID].Export() + if !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) } +} - // Helper to make RenderableNode maps. - nodes := func(ns ...RenderableNode) map[string]RenderableNode { - r := map[string]RenderableNode{} - for _, n := range ns { - r[n.ID] = n - } - return r +func TestAggregateMetadataMerge(t *testing.T) { + want := report.AggregateMetadata{report.KeyBytesIngress: 10 + 20} + have := (report.AggregateMetadata{report.KeyBytesIngress: 10}).Merge(report.AggregateMetadata{report.KeyBytesIngress: 20}) + if !reflect.DeepEqual(want, have) { + t.Error(diff(want, have)) } +} - for _, c := range []struct { - label string - have, want Diff - }{ - { - label: "basecase: empty -> something", - have: TopoDiff(nodes(), nodes(nodea, nodeb)), - want: Diff{ - Add: []RenderableNode{nodea, nodeb}, - }, +var ( + fooHostID = "foo.host.com" + fooAddress = "10.10.10.99" + foo42001EndpointNodeID = report.MakeEndpointNodeID(fooHostID, fooAddress, "42001") + fooAddressNodeID = report.MakeAddressNodeID(fooHostID, fooAddress) + + topologyFixtureA = report.Topology{ + Adjacency: report.Adjacency{ + report.MakeAdjacencyID(clientHostID, client54001EndpointNodeID): report.MakeIDList(server80EndpointNodeID), }, - { - label: "basecase: something -> empty", - have: TopoDiff(nodes(nodea, nodeb), nodes()), - want: Diff{ - Remove: []string{"nodea", "nodeb"}, - }, + NodeMetadatas: report.NodeMetadatas{ + client54001EndpointNodeID: report.NodeMetadata{}, }, - { - label: "add and remove", - have: TopoDiff(nodes(nodea), nodes(nodeb)), - want: Diff{ - Add: []RenderableNode{nodeb}, - Remove: []string{"nodea"}, + EdgeMetadatas: report.EdgeMetadatas{ + report.MakeEdgeID(client54001EndpointNodeID, server80EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 10, + BytesIngress: 100, + WithConnCountTCP: true, + MaxConnCountTCP: 2, }, }, - { - label: "no change", - have: TopoDiff(nodes(nodea), nodes(nodea)), - want: Diff{}, + } + + topologyFixtureB = report.Topology{ + Adjacency: report.Adjacency{ + report.MakeAdjacencyID(clientHostID, client54001EndpointNodeID): report.MakeIDList(server80EndpointNodeID), + report.MakeAdjacencyID(fooHostID, foo42001EndpointNodeID): report.MakeIDList(server80EndpointNodeID), + }, + NodeMetadatas: report.NodeMetadatas{ + client54001EndpointNodeID: report.NodeMetadata{}, + foo42001EndpointNodeID: report.NodeMetadata{}, }, - { - label: "change a single node", - have: TopoDiff(nodes(nodea), nodes(nodeap)), - want: Diff{ - Update: []RenderableNode{nodeap}, + EdgeMetadatas: report.EdgeMetadatas{ + report.MakeEdgeID(client54001EndpointNodeID, server80EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 20, + BytesIngress: 200, + WithConnCountTCP: true, + MaxConnCountTCP: 9, + }, + report.MakeEdgeID(foo42001EndpointNodeID, server80EndpointNodeID): report.EdgeMetadata{ + WithBytes: true, + BytesEgress: 40, + BytesIngress: 400, }, }, - } { - sort.Strings(c.have.Remove) - sort.Sort(ByID(c.have.Add)) - sort.Sort(ByID(c.have.Update)) - if !reflect.DeepEqual(c.want, c.have) { - t.Errorf("%s\n%s", c.label, diff(c.want, c.have)) - } } -} - -func diff(want, have interface{}) string { - text, _ := difflib.GetUnifiedDiffString(difflib.UnifiedDiff{ - A: difflib.SplitLines(spew.Sdump(want)), - B: difflib.SplitLines(spew.Sdump(have)), - FromFile: "want", - ToFile: "have", - Context: 3, - }) - return text -} +) diff --git a/xfer/collector.go b/xfer/collector.go index 8ac6631a9f..1a31c605f5 100644 --- a/xfer/collector.go +++ b/xfer/collector.go @@ -17,51 +17,61 @@ const ( var ( // MaxBackoff is the maximum time between connect retries. - MaxBackoff = 2 * time.Minute // externally configurable. + // It's exported so it's externally configurable. + MaxBackoff = 2 * time.Minute + + tick = time.Tick ) -// Collector connects to probes over TCP and merges reports published by those +// Collector describes anything that can have addresses added and removed, and +// which produces reports that represent aggregate reports from all collected +// addresses. +type Collector interface { + Add(string) + Remove(string) + Reports() <-chan report.Report + Stop() +} + +// realCollector connects to probes over TCP and merges reports published by those // probes into a single one. -type Collector struct { +type realCollector struct { in chan report.Report out chan report.Report + peekc chan chan report.Report add chan string remove chan string - quit chan chan struct{} + quit chan struct{} } -// NewCollector starts the report collector. -func NewCollector(batchTime time.Duration) *Collector { - c := &Collector{ +// NewCollector produces and returns a report collector. +func NewCollector(batchTime time.Duration) Collector { + c := &realCollector{ in: make(chan report.Report), out: make(chan report.Report), + peekc: make(chan chan report.Report), add: make(chan string), remove: make(chan string), - quit: make(chan chan struct{}), + quit: make(chan struct{}), } - go c.loop(batchTime) - return c } -func (c *Collector) loop(batchTime time.Duration) { +func (c *realCollector) loop(batchTime time.Duration) { var ( - tick = time.Tick(batchTime) - current = report.NewReport() + tick = tick(batchTime) + current = report.MakeReport() addrs = map[string]chan struct{}{} - wg = &sync.WaitGroup{} // individual collector goroutines + wg = &sync.WaitGroup{} // per-address goroutines ) add := func(ip string) { if _, ok := addrs[ip]; ok { return } - addrs[ip] = make(chan struct{}) - wg.Add(1) - go func(quit chan struct{}) { defer wg.Done() reportCollector(ip, c.in, quit) @@ -73,7 +83,6 @@ func (c *Collector) loop(batchTime time.Duration) { if !ok { return // hmm } - close(q) delete(addrs, ip) } @@ -82,7 +91,10 @@ func (c *Collector) loop(batchTime time.Duration) { select { case <-tick: c.out <- current - current = report.NewReport() + current = report.MakeReport() + + case pc := <-c.peekc: + pc <- current case r := <-c.in: current.Merge(r) @@ -93,47 +105,41 @@ func (c *Collector) loop(batchTime time.Duration) { case ip := <-c.remove: remove(ip) - case q := <-c.quit: + case <-c.quit: for _, q := range addrs { close(q) } wg.Wait() - close(q) return } } } -// Stop shuts down a collector and all connections to probes. -func (c *Collector) Stop() { - q := make(chan struct{}) - c.quit <- q - <-q +// Add adds an address to be collected from. +func (c *realCollector) Add(addr string) { + c.add <- addr } -// AddAddress adds the passed IP to the collector, and starts (trying to) -// collect reports from the remote Publisher. -func (c *Collector) AddAddress(ip string) { - c.add <- ip +// Remove removes a previously-added address. +func (c *realCollector) Remove(addr string) { + c.remove <- addr } -// AddAddresses adds the passed IPs to the collector, and starts (trying to) -// collect reports from the remote Publisher. -func (c *Collector) AddAddresses(ips []string) { - for _, addr := range ips { - c.AddAddress(addr) - } +// Reports returns the report chan. It must be consumed by the client, or the +// collector will break. +func (c *realCollector) Reports() <-chan report.Report { + return c.out } -// RemoveAddress removes the passed IP from the collector, and stops -// collecting reports from the remote Publisher. -func (c *Collector) RemoveAddress(ip string) { - c.remove <- ip +func (c *realCollector) peek() report.Report { + pc := make(chan report.Report) + c.peekc <- pc + return <-pc } -// Reports returns the channel where aggregate reports are sent. -func (c *Collector) Reports() <-chan report.Report { - return c.out +// Stop terminates the collector. +func (c *realCollector) Stop() { + close(c.quit) } // reportCollector is the loop to connect to a single Probe. It'll keep @@ -188,7 +194,7 @@ func reportCollector(ip string, col chan<- report.Report, quit <-chan struct{}) log.Printf("decode error: %v", err) break } - //log.Printf("collector: got a report from %v", ip) + log.Printf("collector: got a report from %v", ip) select { case col <- report: diff --git a/xfer/collector_test.go b/xfer/collector_test.go index f839471353..0708bdcfc9 100644 --- a/xfer/collector_test.go +++ b/xfer/collector_test.go @@ -1,69 +1,102 @@ -package xfer_test +package xfer import ( - "bytes" "encoding/gob" "io/ioutil" "log" "net" + "runtime" "testing" "time" "github.com/weaveworks/scope/report" - "github.com/weaveworks/scope/xfer" ) func TestCollector(t *testing.T) { log.SetOutput(ioutil.Discard) - // Build the address - port := ":12345" - addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1"+port) + // Swap out ticker + publish := make(chan time.Time) + oldTick := tick + tick = func(time.Duration) <-chan time.Time { return publish } + defer func() { tick = oldTick }() + + // Build a collector + collector := NewCollector(time.Second) + defer collector.Stop() + + concreteCollector, ok := collector.(*realCollector) + if !ok { + t.Fatal("type assertion failure") + } + + // Build a test publisher + reports := make(chan interface{}) + ln := testPublisher(t, reports) + defer ln.Close() + + // Connect the collector to the test publisher + addr := ln.Addr().String() + collector.Add(addr) + collector.Add(addr) // test duplicate case + runtime.Gosched() // make sure it connects + + // Push a report through everything + reports <- report.Report{Host: report.Topology{NodeMetadatas: report.NodeMetadatas{"a": report.NodeMetadata{}}}} + poll(t, time.Millisecond, func() bool { return len(concreteCollector.peek().Host.NodeMetadatas) == 1 }, "missed the report") + go func() { publish <- time.Now() }() + if want, have := 1, len((<-collector.Reports()).Host.NodeMetadatas); want != have { + t.Errorf("want %d, have %d", want, have) + } + + collector.Remove(addr) + collector.Remove(addr) // test duplicate case +} + +func TestCollectorQuitWithActiveConnections(t *testing.T) { + c := NewCollector(time.Second) + c.Add("1.2.3.4:56789") + c.Stop() +} + +func testPublisher(t *testing.T, input <-chan interface{}) net.Listener { + addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:0") if err != nil { t.Fatal(err) } - - // Start a raw publisher ln, err := net.ListenTCP("tcp4", addr) if err != nil { t.Fatal(err) } - defer ln.Close() - - // Accept one connection, write one report - data := make(chan []byte) go func() { conn, err := ln.Accept() if err != nil { - t.Error(err) + t.Log(err) return } defer conn.Close() - - if _, err := conn.Write(<-data); err != nil { - t.Error(err) - return + for { + enc := gob.NewEncoder(conn) + for v := range input { + if err := enc.Encode(v); err != nil { + t.Error(err) + return + } + } } }() + return ln +} - // Start a collector - batchTime := 10 * time.Millisecond - c := xfer.NewCollector(batchTime) - c.AddAddress("127.0.0.1" + port) - gate := make(chan struct{}) - go func() { <-c.Reports(); c.Stop(); close(gate) }() - - // Publish a message - var buf bytes.Buffer - if err := gob.NewEncoder(&buf).Encode(report.Report{}); err != nil { - t.Fatal(err) - } - data <- buf.Bytes() - - // Check it was collected and forwarded - select { - case <-gate: - case <-time.After(2 * batchTime): - t.Errorf("timeout waiting for report") +func poll(t *testing.T, d time.Duration, condition func() bool, msg string) { + deadline := time.Now().Add(d) + for { + if time.Now().After(deadline) { + t.Fatal(msg) + } + if condition() { + return + } + time.Sleep(d / 10) } } diff --git a/xfer/merge_test.go b/xfer/merge_test.go deleted file mode 100644 index 4fd8c90f08..0000000000 --- a/xfer/merge_test.go +++ /dev/null @@ -1,70 +0,0 @@ -package xfer_test - -import ( - "io/ioutil" - "log" - "testing" - "time" - - "github.com/weaveworks/scope/report" - "github.com/weaveworks/scope/xfer" -) - -func TestMerge(t *testing.T) { - log.SetOutput(ioutil.Discard) - - var ( - p1Addr = "localhost:7888" - p2Addr = "localhost:7889" - ) - - p1, err := xfer.NewTCPPublisher(p1Addr) - if err != nil { - t.Fatal(err) - } - defer p1.Close() - - p2, err := xfer.NewTCPPublisher(p2Addr) - if err != nil { - t.Fatal(err) - } - defer p2.Close() - - batchTime := 100 * time.Millisecond - c := xfer.NewCollector(batchTime) - c.AddAddress(p1Addr) - c.AddAddress(p2Addr) - defer c.Stop() - time.Sleep(batchTime / 10) // connect - - { - r := report.NewReport() - r.HostMetadatas["p1"] = report.HostMetadata{Hostname: "test1"} - p1.Publish(r) - } - { - r := report.NewReport() - r.HostMetadatas["p2"] = report.HostMetadata{Hostname: "test2"} - p2.Publish(r) - } - - success := make(chan struct{}) - go func() { - defer close(success) - for r := range c.Reports() { - if r.HostMetadatas["p1"].Hostname != "test1" { - continue - } - if r.HostMetadatas["p2"].Hostname != "test2" { - continue - } - return - } - }() - - select { - case <-success: - case <-time.After(batchTime): - t.Errorf("collector didn't capture both reports") - } -} diff --git a/xfer/publisher_test.go b/xfer/publisher_test.go index 3e770dfe19..f86f41d23b 100644 --- a/xfer/publisher_test.go +++ b/xfer/publisher_test.go @@ -2,12 +2,11 @@ package xfer_test import ( "encoding/gob" - "fmt" "io/ioutil" "log" "net" + "runtime" "testing" - "time" "github.com/weaveworks/scope/report" "github.com/weaveworks/scope/xfer" @@ -16,63 +15,23 @@ import ( func TestTCPPublisher(t *testing.T) { log.SetOutput(ioutil.Discard) - // Choose a port - port, err := getFreePort() - if err != nil { - t.Fatal(err) - } - - // Start a publisher - p, err := xfer.NewTCPPublisher(port) - if err != nil { - t.Fatal(err) - } - defer p.Close() - - // Start a raw listener - conn, err := net.Dial("tcp4", "127.0.0.1"+port) - if err != nil { - t.Fatal(err) - } - defer conn.Close() - time.Sleep(time.Millisecond) - - // Publish a message - p.Publish(report.Report{}) - - // Check it was received - var r report.Report - if err := gob.NewDecoder(conn).Decode(&r); err != nil { - t.Fatal(err) - } -} - -func TestPublisherClosesDuplicateConnections(t *testing.T) { - log.SetOutput(ioutil.Discard) - - // Choose a port - port, err := getFreePort() - if err != nil { - t.Fatal(err) - } - - // Start a publisher - p, err := xfer.NewTCPPublisher(port) + addr := getFreeAddr(t) + p, err := xfer.NewTCPPublisher(addr) if err != nil { t.Fatal(err) } defer p.Close() // Connect a listener - conn, err := net.Dial("tcp4", "127.0.0.1"+port) + conn, err := net.Dial("tcp4", addr) if err != nil { t.Fatal(err) } defer conn.Close() - time.Sleep(time.Millisecond) + runtime.Gosched() - // Try to connect the same listener - dupconn, err := net.Dial("tcp4", "127.0.0.1"+port) + // Connect a duplicate listener + dupconn, err := net.Dial("tcp4", addr) if err != nil { t.Fatal(err) } @@ -84,7 +43,7 @@ func TestPublisherClosesDuplicateConnections(t *testing.T) { // The first listener should receive it var r report.Report if err := gob.NewDecoder(conn).Decode(&r); err != nil { - t.Fatal(err) + t.Error(err) } // The duplicate listener should have an error @@ -95,15 +54,15 @@ func TestPublisherClosesDuplicateConnections(t *testing.T) { } } -func getFreePort() (string, error) { +func getFreeAddr(t *testing.T) string { ln, err := net.Listen("tcp4", ":0") if err != nil { - return "", fmt.Errorf("Listen: %v", err) + t.Fatalf("Listen: %v", err) } defer ln.Close() _, port, err := net.SplitHostPort(ln.Addr().String()) if err != nil { - return "", fmt.Errorf("SplitHostPort(%s): %v", ln.Addr().String(), err) + t.Fatalf("SplitHostPort(%s): %v", ln.Addr().String(), err) } - return ":" + port, nil + return "127.0.0.1:" + port }