From 49dae07cca685ae72ffc1f2d06ff5f1f5b84bbaa Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Fri, 19 Jun 2015 10:43:21 +0000 Subject: [PATCH] Make render module produce The Internet pseudo node. --- app/api_topology_test.go | 23 ++-- app/mock_reporter_test.go | 20 ++-- app/router.go | 3 +- experimental/bridge/main.go | 31 +----- experimental/graphviz/handle.go | 2 +- probe/main.go | 4 +- probe/spy.go | 1 + render/detailed_node.go | 5 +- render/detailed_node_test.go | 2 +- render/mapping.go | 137 ++++++++++++++---------- render/mapping_test.go | 67 ------------ render/render.go | 9 +- render/theinternet.go | 51 +++++++++ render/theinternet_test.go | 50 +++++++++ render/topologies.go | 30 +++++- render/topologies_test.go | 182 +++++++++++++++----------------- report/id.go | 8 -- report/id_test.go | 14 --- report/report.go | 15 ++- 19 files changed, 346 insertions(+), 308 deletions(-) delete mode 100644 render/mapping_test.go create mode 100644 render/theinternet.go create mode 100644 render/theinternet_test.go diff --git a/app/api_topology_test.go b/app/api_topology_test.go index 043e5fd721..10a7cb8829 100644 --- a/app/api_topology_test.go +++ b/app/api_topology_test.go @@ -80,34 +80,35 @@ func TestAPITopologyHosts(t *testing.T) { t.Fatal(err) } equals(t, 3, len(topo.Nodes)) - node, ok := topo.Nodes["host:host-b"] + node, ok := topo.Nodes["host:hostB"] if !ok { - t.Errorf("missing host:host-b node") + t.Errorf("missing host:hostB node") + t.Errorf("%+v", topo) } - equals(t, report.MakeIDList("host:host-a"), node.Adjacency) + equals(t, report.MakeIDList("host:hostA"), node.Adjacency) equals(t, report.MakeIDList( report.MakeAddressNodeID("hostB", "192.168.1.2"), report.MakeHostNodeID("hostB"), ), node.Origins) - equals(t, "host-b", node.LabelMajor) - equals(t, "", node.LabelMinor) - equals(t, "host-b", node.Rank) + equals(t, "node-b", node.LabelMajor) + equals(t, "local", node.LabelMinor) + equals(t, "local", node.Rank) equals(t, false, node.Pseudo) } { - body := getRawJSON(t, ts, "/api/topology/hosts/host:host-b") + body := getRawJSON(t, ts, "/api/topology/hosts/host:hostB") var node APINode if err := json.Unmarshal(body, &node); err != nil { t.Fatal(err) } - equals(t, "host:host-b", node.Node.ID) - equals(t, "host-b", node.Node.LabelMajor) - equals(t, "", node.Node.LabelMinor) + equals(t, "host:hostB", node.Node.ID) + equals(t, "node-b", node.Node.LabelMajor) + equals(t, "local", node.Node.LabelMinor) equals(t, false, node.Node.Pseudo) // Let's not unit-test the specific content of the detail tables } { - body := getRawJSON(t, ts, "/api/topology/hosts/host:host-b/host:host-a") + body := getRawJSON(t, ts, "/api/topology/hosts/host:hostB/host:hostA") var edge APIEdge if err := json.Unmarshal(body, &edge); err != nil { t.Fatalf("JSON parse error: %s", err) diff --git a/app/mock_reporter_test.go b/app/mock_reporter_test.go index 6e6e6d2728..2e730026a6 100644 --- a/app/mock_reporter_test.go +++ b/app/mock_reporter_test.go @@ -142,11 +142,11 @@ func (s StaticReport) Report() report.Report { }, NodeMetadatas: report.NodeMetadatas{ report.MakeAddressNodeID("hostA", "192.168.1.1"): report.NodeMetadata{ - "name": "host-a", + "addr": "192.168.1.1", report.HostNodeID: report.MakeHostNodeID("hostA"), }, report.MakeAddressNodeID("hostB", "192.168.1.2"): report.NodeMetadata{ - "name": "host-b", + "addr": "192.168.1.2", report.HostNodeID: report.MakeHostNodeID("hostB"), }, }, @@ -157,15 +157,17 @@ func (s StaticReport) Report() report.Report { EdgeMetadatas: report.EdgeMetadatas{}, NodeMetadatas: report.NodeMetadatas{ report.MakeHostNodeID("hostA"): report.NodeMetadata{ - "host_name": "node-a.local", - "os": "Linux", - "local_networks": localNet.String(), - "load": "3.14 2.71 1.61", + "host_name": "node-a.local", + "os": "Linux", + "local_networks": localNet.String(), + "load": "3.14 2.71 1.61", + report.HostNodeID: report.MakeHostNodeID("hostA"), }, report.MakeHostNodeID("hostB"): report.NodeMetadata{ - "host_name": "node-b.local", - "os": "Linux", - "local_networks": localNet.String(), + "host_name": "node-b.local", + "os": "Linux", + "local_networks": localNet.String(), + report.HostNodeID: report.MakeHostNodeID("hostB"), }, }, }, diff --git a/app/router.go b/app/router.go index 0d0d5b7438..7ed7402465 100644 --- a/app/router.go +++ b/app/router.go @@ -6,7 +6,6 @@ import ( "github.com/gorilla/mux" "github.com/weaveworks/scope/render" - "github.com/weaveworks/scope/report" ) // Router gives of the HTTP dispatcher. It will always use the embedded HTML @@ -70,7 +69,7 @@ var topologyRegistry = map[string]topologyView{ "hosts": { human: "Hosts", parent: "", - renderer: render.LeafMap{Selector: report.SelectAddress, Mapper: render.NetworkHostname, Pseudo: render.GenericPseudoNode}, + renderer: render.HostRenderer, }, } diff --git a/experimental/bridge/main.go b/experimental/bridge/main.go index 803c7821e7..e2e98810e4 100644 --- a/experimental/bridge/main.go +++ b/experimental/bridge/main.go @@ -12,6 +12,7 @@ import ( "syscall" "time" + "github.com/weaveworks/scope/render" "github.com/weaveworks/scope/report" "github.com/weaveworks/scope/xfer" ) @@ -112,34 +113,6 @@ func makeAvoid(fixed []string) map[string]struct{} { return avoid } -// LocalNetworks returns a superset of the networks (think: CIDRs) that are -// "local" from the perspective of each host represented in the report. It's -// used to determine which nodes in the report are "remote", i.e. outside of -// our infrastructure. -func LocalNetworks(r report.Report) []*net.IPNet { - var ipNets []*net.IPNet - for _, md := range r.Host.NodeMetadatas { - val, ok := md["local_networks"] - if !ok { - continue - } - outer: - for _, s := range strings.Fields(val) { - _, ipNet, err := net.ParseCIDR(s) - if err != nil { - continue - } - for _, existing := range ipNets { - if ipNet.String() == existing.String() { - continue outer - } - } - ipNets = append(ipNets, ipNet) - } - } - return ipNets -} - // discover reads reports from a collector and republishes them on the // publisher, while scanning the reports for IPs to connect to. Only addresses // in the network topology of the report are considered. IPs listed in fixed @@ -155,7 +128,7 @@ func discover(c collector, p publisher, fixed []string) { var ( now = time.Now() - localNets = LocalNetworks(r) + localNets = render.LocalNetworks(r) ) for _, adjacent := range r.Address.Adjacency { diff --git a/experimental/graphviz/handle.go b/experimental/graphviz/handle.go index 03c4e65a8f..783153212b 100644 --- a/experimental/graphviz/handle.go +++ b/experimental/graphviz/handle.go @@ -105,7 +105,7 @@ func engine(r *http.Request) string { func mapFunc(r *http.Request) render.LeafMapFunc { switch strings.ToLower(r.FormValue("map_func")) { case "hosts", "networkhost", "networkhostname": - return render.NetworkHostname + return render.MapAddressIdentity } return render.MapProcessIdentity } diff --git a/probe/main.go b/probe/main.go index ca42f6fa2b..da19c77f41 100644 --- a/probe/main.go +++ b/probe/main.go @@ -115,13 +115,15 @@ func main() { select { case <-pubTick: publishTicks.WithLabelValues().Add(1) - r.Host = hostTopology(hostID, hostName) publisher.Publish(r) r = report.MakeReport() case <-spyTick: r.Merge(spy(hostID, hostName, *spyProcs)) + // Do this every tick so it gets tagged by the OriginHostTagger + r.Host = hostTopology(hostID, hostName) + // TODO abstract PIDTree to a process provider, and provide an // alternate implementation for Darwin. if runtime.GOOS == linux { diff --git a/probe/spy.go b/probe/spy.go index 72e6bcb248..aa9e1025c3 100644 --- a/probe/spy.go +++ b/probe/spy.go @@ -54,6 +54,7 @@ func addConnection( if _, ok := r.Address.NodeMetadatas[scopedLocal]; !ok { r.Address.NodeMetadatas[scopedLocal] = report.NodeMetadata{ "name": hostName, + "addr": c.LocalAddress.String(), } } diff --git a/render/detailed_node.go b/render/detailed_node.go index 748fb84618..46fb737bf8 100644 --- a/render/detailed_node.go +++ b/render/detailed_node.go @@ -121,12 +121,9 @@ func endpointOriginTable(nmd report.NodeMetadata) (Table, bool) { func addressOriginTable(nmd report.NodeMetadata) (Table, bool) { rows := []Row{} - if val, ok := nmd["address"]; ok { + if val, ok := nmd["addr"]; ok { rows = append(rows, Row{"Address", val, ""}) } - if val, ok := nmd["host_name"]; ok { - rows = append(rows, Row{"Host name", val, ""}) - } return Table{ Title: "Origin Address", Numeric: false, diff --git a/render/detailed_node_test.go b/render/detailed_node_test.go index f40c7e6991..19f03296c0 100644 --- a/render/detailed_node_test.go +++ b/render/detailed_node_test.go @@ -25,7 +25,7 @@ func TestOriginTable(t *testing.T) { Title: "Origin Address", Numeric: false, Rows: []render.Row{ - {"Host name", clientHostName, ""}, + {"Address", clientIP, ""}, }, }, serverProcessNodeID: { diff --git a/render/mapping.go b/render/mapping.go index 01110b4503..dd38d17040 100644 --- a/render/mapping.go +++ b/render/mapping.go @@ -2,6 +2,7 @@ package render import ( "fmt" + "net" "strings" "github.com/weaveworks/scope/report" @@ -12,7 +13,8 @@ const ( UncontainedID = "uncontained" UncontainedMajor = "Uncontained" - humanTheInternet = "the Internet" + TheInternetID = "theinternet" + TheInternetMajor = "The Internet" ) // LeafMapFunc is anything which can take an arbitrary NodeMetadata, which is @@ -31,7 +33,7 @@ type LeafMapFunc func(report.NodeMetadata) (RenderableNode, bool) // The srcNode renderable node is essentially from MapFunc, representing one of // the rendered nodes this pseudo node refers to. srcNodeID and dstNodeID are // node IDs prior to mapping. -type PseudoFunc func(srcNodeID string, srcNode RenderableNode, dstNodeID string) (RenderableNode, bool) +type PseudoFunc func(srcNodeID string, srcNode RenderableNode, dstNodeID string, local Networks) (RenderableNode, bool) // MapFunc is anything which can take an arbitrary RenderableNode and // return another RenderableNode. @@ -100,6 +102,40 @@ func MapContainerImageIdentity(m report.NodeMetadata) (RenderableNode, bool) { return NewRenderableNode(id, major, "", rank, m), true } +// MapAddressIdentity maps a address topology node to address RenderableNode +// node. As it is only ever run on address topology nodes, we can safely +// assume the presence of certain keys. +func MapAddressIdentity(m report.NodeMetadata) (RenderableNode, bool) { + var ( + id = fmt.Sprintf("address:%s:%s", report.ExtractHostID(m), m["addr"]) + major = m["addr"] + minor = report.ExtractHostID(m) + rank = major + ) + + return NewRenderableNode(id, major, minor, rank, m), true +} + +// MapHostIdentity maps a host topology node to host RenderableNode +// node. As it is only ever run on host topology nodes, we can safely +// assume the presence of certain keys. +func MapHostIdentity(m report.NodeMetadata) (RenderableNode, bool) { + var ( + id = fmt.Sprintf("host:%s", report.ExtractHostID(m)) + hostname = m["host_name"] + parts = strings.SplitN(hostname, ".", 2) + major, minor, rank = "", "", "" + ) + + if len(parts) == 2 { + major, minor, rank = parts[0], parts[1], parts[1] + } else { + major = hostname + } + + return NewRenderableNode(id, major, minor, rank, m), true +} + // MapEndpoint2Process maps endpoint RenderableNodes to process // RenderableNodes. // @@ -138,6 +174,13 @@ func MapEndpoint2Process(n RenderableNode) (RenderableNode, bool) { // It does not have enough info to do that, and the resulting graph // must be merged with a container graph to get that info. func MapProcess2Container(n RenderableNode) (RenderableNode, bool) { + // Propogate the internet pseudo node + if n.ID == TheInternetID { + return n, true + } + + // Otherwise, if the process is not in a container, group it + // into an "Uncontained" node id, ok := n.NodeMetadata["docker_container_id"] if !ok || n.Pseudo { return newDerivedPseudoNode(UncontainedID, UncontainedMajor, n), true @@ -181,6 +224,13 @@ func MapProcess2Name(n RenderableNode) (RenderableNode, bool) { // It does not have enough info to do that, and the resulting graph // must be merged with a container graph to get that info. func MapContainer2ContainerImage(n RenderableNode) (RenderableNode, bool) { + // Propogate the internet pseudo node + if n.ID == TheInternetID { + return n, true + } + + // Otherwise, if the process is not in a container, group it + // into an "Uncontained" node id, ok := n.NodeMetadata["docker_image_id"] if !ok || n.Pseudo { return newDerivedPseudoNode(UncontainedID, UncontainedMajor, n), true @@ -189,69 +239,46 @@ func MapContainer2ContainerImage(n RenderableNode) (RenderableNode, bool) { return newDerivedNode(id, n), true } -// NetworkHostname takes a node NodeMetadata and returns a representation -// based on the hostname. Major label is the hostname, the minor label is the -// domain, if any. -func NetworkHostname(m report.NodeMetadata) (RenderableNode, bool) { - var ( - name = m["name"] - domain = "" - parts = strings.SplitN(name, ".", 2) - ) - - if len(parts) == 2 { - domain = parts[1] +// MapAddress2Host maps address RenderableNodes to host RenderableNodes. +// +// Otherthan pseudo nodes, we can assume all nodes have a HostID +func MapAddress2Host(n RenderableNode) (RenderableNode, bool) { + if n.Pseudo { + return n, true } - return NewRenderableNode(fmt.Sprintf("host:%s", name), parts[0], domain, parts[0], m), name != "" + id := fmt.Sprintf("host:%s", report.ExtractHostID(n.NodeMetadata)) + return newDerivedNode(id, n), true } -// GenericPseudoNode contains heuristics for building sensible pseudo nodes. -// It should go away. -func GenericPseudoNode(src string, srcMapped RenderableNode, dst string) (RenderableNode, bool) { - var maj, min, outputID string - - if dst == report.TheInternet { - outputID = dst - maj, min = humanTheInternet, "" - } else { - // Rule for non-internet psuedo nodes; emit 1 new node for each +// GenericPseudoNode makes a PseudoFunc given an addresser. The returned +// PseudoFunc will produce Internet pseudo nodes for addresses not in +// the report's local networks. Otherwise, the returned function will +// produce a single pseudo node per (dst address, src address, src port). +func GenericPseudoNode(addresser func(id string) net.IP) PseudoFunc { + return func(src string, srcMapped RenderableNode, dst string, local Networks) (RenderableNode, bool) { + // Use the addresser to extract the destination IP + dstNodeAddr := addresser(dst) + + // If the dstNodeAddr is not in a network local to this report, we emit an + // internet node + if !local.Contains(dstNodeAddr) { + return newPseudoNode(TheInternetID, TheInternetMajor, ""), true + } + + // Otherwise, the rule for non-internet psuedo nodes; emit 1 new node for each // dstNodeAddr, srcNodeAddr, srcNodePort. srcNodeAddr, srcNodePort := trySplitAddr(src) - dstNodeAddr, _ := trySplitAddr(dst) - outputID = report.MakePseudoNodeID(dstNodeAddr, srcNodeAddr, srcNodePort) - maj, min = dstNodeAddr, "" + outputID := report.MakePseudoNodeID(dstNodeAddr.String(), srcNodeAddr, srcNodePort) + major := dstNodeAddr.String() + return newPseudoNode(outputID, major, ""), true } - - return newPseudoNode(outputID, maj, min), true } -// GenericGroupedPseudoNode contains heuristics for building sensible pseudo nodes. -// It should go away. -func GenericGroupedPseudoNode(src string, srcMapped RenderableNode, dst string) (RenderableNode, bool) { - var maj, min, outputID string - - if dst == report.TheInternet { - outputID = dst - maj, min = humanTheInternet, "" - } else { - // When grouping, emit one pseudo node per (srcNodeAddress, dstNodeAddr) - dstNodeAddr, _ := trySplitAddr(dst) - - outputID = report.MakePseudoNodeID(dstNodeAddr, srcMapped.ID) - maj, min = dstNodeAddr, "" - } - - return newPseudoNode(outputID, maj, min), true -} - -// InternetOnlyPseudoNode never creates a pseudo node, unless it's the Internet. -func InternetOnlyPseudoNode(_ string, _ RenderableNode, dst string) (RenderableNode, bool) { - if dst == report.TheInternet { - return newPseudoNode(report.TheInternet, humanTheInternet, ""), true - } - return RenderableNode{}, false +// PanicPseudoNode just panics; it is for Topologies without edges +func PanicPseudoNode(src string, srcMapped RenderableNode, dst string, local Networks) (RenderableNode, bool) { + panic(dst) } // trySplitAddr is basically ParseArbitraryNodeID, since its callsites diff --git a/render/mapping_test.go b/render/mapping_test.go deleted file mode 100644 index 2d2cbabdbf..0000000000 --- a/render/mapping_test.go +++ /dev/null @@ -1,67 +0,0 @@ -package render_test - -import ( - "fmt" - "testing" - - "github.com/weaveworks/scope/render" - "github.com/weaveworks/scope/report" -) - -func TestUngroupedMapping(t *testing.T) { - for i, c := range []struct { - f render.LeafMapFunc - id string - meta report.NodeMetadata - wantOK bool - wantID, wantMajor, wantMinor, wantRank string - }{ - { - f: render.NetworkHostname, - id: report.MakeAddressNodeID("", "1.2.3.4"), - meta: report.NodeMetadata{ - "name": "my.host", - }, - wantOK: true, - wantID: "host:my.host", - wantMajor: "my", - wantMinor: "host", - wantRank: "my", - }, - { - f: render.NetworkHostname, - id: report.MakeAddressNodeID("", "1.2.3.4"), - meta: report.NodeMetadata{ - "name": "localhost", - }, - wantOK: true, - wantID: "host:localhost", - wantMajor: "localhost", - wantMinor: "", - wantRank: "localhost", - }, - } { - identity := fmt.Sprintf("(%d %s %v)", i, c.id, c.meta) - - m, haveOK := c.f(c.meta) - if want, have := c.wantOK, haveOK; want != have { - t.Errorf("%s: map OK error: want %v, have %v", identity, want, have) - } - if want, have := c.wantID, m.ID; want != have { - t.Errorf("%s: map ID error: want %#v, have %#v", identity, want, have) - } - if want, have := c.wantMajor, m.LabelMajor; want != have { - t.Errorf("%s: map major label: want %#v, have %#v", identity, want, have) - } - if want, have := c.wantMinor, m.LabelMinor; want != have { - t.Errorf("%s: map minor label: want %#v, have %#v", identity, want, have) - } - if want, have := c.wantRank, m.Rank; want != have { - t.Errorf("%s: map rank: want %#v, have %#v", identity, want, have) - } - } -} - -func TestGroupedMapping(t *testing.T) { - t.Skipf("not yet implemented") // TODO -} diff --git a/render/render.go b/render/render.go index 1b1236e854..1fffa5e94f 100644 --- a/render/render.go +++ b/render/render.go @@ -144,8 +144,11 @@ func (m Map) AggregateMetadata(rpt report.Report, srcRenderableID, dstRenderable // // Nodes with the same mapped IDs will be merged. func (m LeafMap) Render(rpt report.Report) RenderableNodes { - t := m.Selector(rpt) - nodes := RenderableNodes{} + var ( + t = m.Selector(rpt) + nodes = RenderableNodes{} + localNetworks = LocalNetworks(rpt) + ) // Build a set of RenderableNodes for all non-pseudo probes, and an // addressID to nodeID lookup map. Multiple addressIDs can map to the same @@ -189,7 +192,7 @@ func (m LeafMap) Render(rpt report.Report) RenderableNodes { for _, dstNodeID := range dsts { dstRenderableID, ok := source2mapped[dstNodeID] if !ok { - pseudoNode, ok := m.Pseudo(srcNodeID, srcRenderableNode, dstNodeID) + pseudoNode, ok := m.Pseudo(srcNodeID, srcRenderableNode, dstNodeID, localNetworks) if !ok { continue } diff --git a/render/theinternet.go b/render/theinternet.go new file mode 100644 index 0000000000..8bdcda9375 --- /dev/null +++ b/render/theinternet.go @@ -0,0 +1,51 @@ +package render + +import ( + "net" + "strings" + + "github.com/weaveworks/scope/report" +) + +// Networks represent a set of subnets local to a report. +type Networks []*net.IPNet + +// LocalNetworks returns a superset of the networks (think: CIDRs) that are +// "local" from the perspective of each host represented in the report. It's +// used to determine which nodes in the report are "remote", i.e. outside of +// our infrastructure. +func LocalNetworks(r report.Report) Networks { + var ( + result = Networks{} + networks = map[string]struct{}{} + ) + + for _, md := range r.Host.NodeMetadatas { + val, ok := md["local_networks"] + if !ok { + continue + } + for _, s := range strings.Fields(val) { + _, ipNet, err := net.ParseCIDR(s) + if err != nil { + continue + } + _, ok := networks[ipNet.String()] + if !ok { + result = append(result, ipNet) + networks[ipNet.String()] = struct{}{} + } + } + } + return result +} + +// Contains returns true if IP is in Networks. +func (n Networks) Contains(ip net.IP) bool { + for _, net := range n { + if net.Contains(ip) { + return true + } + } + return false +} diff --git a/render/theinternet_test.go b/render/theinternet_test.go new file mode 100644 index 0000000000..cb04f6d2eb --- /dev/null +++ b/render/theinternet_test.go @@ -0,0 +1,50 @@ +package render_test + +import ( + "net" + "reflect" + "testing" + + "github.com/weaveworks/scope/render" + "github.com/weaveworks/scope/report" + "github.com/weaveworks/scope/test" +) + +func TestReportLocalNetworks(t *testing.T) { + r := report.MakeReport() + r.Merge(report.Report{Host: report.Topology{NodeMetadatas: report.NodeMetadatas{ + "nonets": {}, + "foo": {"local_networks": "10.0.0.1/8 192.168.1.1/24 10.0.0.1/8 badnet/33"}, + }}}) + want := render.Networks([]*net.IPNet{ + mustParseCIDR("10.0.0.1/8"), + mustParseCIDR("192.168.1.1/24"), + }) + have := render.LocalNetworks(r) + if !reflect.DeepEqual(want, have) { + t.Errorf("%s", test.Diff(want, have)) + } +} + +func TestContains(t *testing.T) { + networks := render.Networks([]*net.IPNet{ + mustParseCIDR("10.0.0.1/8"), + mustParseCIDR("192.168.1.1/24"), + }) + + if networks.Contains(net.ParseIP("52.52.52.52")) { + t.Errorf("52.52.52.52 not in %v", networks) + } + + if !networks.Contains(net.ParseIP("10.0.0.1")) { + t.Errorf("10.0.0.1 in %v", networks) + } +} + +func mustParseCIDR(s string) *net.IPNet { + _, ipNet, err := net.ParseCIDR(s) + if err != nil { + panic(err) + } + return ipNet +} diff --git a/render/topologies.go b/render/topologies.go index 78eefa6cb5..c25e3353cb 100644 --- a/render/topologies.go +++ b/render/topologies.go @@ -8,7 +8,7 @@ import ( var EndpointRenderer = LeafMap{ Selector: report.SelectEndpoint, Mapper: MapEndpointIdentity, - Pseudo: GenericPseudoNode, + Pseudo: GenericPseudoNode(report.EndpointIDAddresser), } // ProcessRenderer is a Renderer which produces a renderable process @@ -21,7 +21,7 @@ var ProcessRenderer = MakeReduce( LeafMap{ Selector: report.SelectProcess, Mapper: MapProcessIdentity, - Pseudo: GenericPseudoNode, + Pseudo: PanicPseudoNode, }, ) @@ -42,7 +42,7 @@ var ContainerRenderer = MakeReduce( LeafMap{ Selector: report.SelectContainer, Mapper: MapContainerIdentity, - Pseudo: GenericPseudoNode, + Pseudo: PanicPseudoNode, }, ) @@ -56,6 +56,28 @@ var ContainerImageRenderer = MakeReduce( LeafMap{ Selector: report.SelectContainerImage, Mapper: MapContainerImageIdentity, - Pseudo: GenericPseudoNode, + Pseudo: PanicPseudoNode, + }, +) + +// AddressRenderer is a Renderer which produces a renderable address +// graph from the address topology. +var AddressRenderer = LeafMap{ + Selector: report.SelectAddress, + Mapper: MapAddressIdentity, + Pseudo: GenericPseudoNode(report.AddressIDAddresser), +} + +// HostRenderer is a Renderer which produces a renderable host +// graph from the host topology and address graph. +var HostRenderer = MakeReduce( + Map{ + MapFunc: MapAddress2Host, + Renderer: AddressRenderer, + }, + LeafMap{ + Selector: report.SelectHost, + Mapper: MapHostIdentity, + Pseudo: PanicPseudoNode, }, ) diff --git a/render/topologies_test.go b/render/topologies_test.go index 9a6eab79df..920b409d12 100644 --- a/render/topologies_test.go +++ b/render/topologies_test.go @@ -10,10 +10,12 @@ import ( "github.com/weaveworks/scope/test" ) +// This is an example Report: +// 2 hosts with probes installed - client & server. + var ( clientHostID = "client.hostname.com" serverHostID = "server.hostname.com" - randomHostID = "random.hostname.com" unknownHostID = "" clientIP = "10.10.10.20" @@ -31,19 +33,14 @@ var ( clientHostNodeID = report.MakeHostNodeID(clientHostID) serverHostNodeID = report.MakeHostNodeID(serverHostID) - randomHostNodeID = report.MakeHostNodeID(randomHostID) client54001NodeID = report.MakeEndpointNodeID(clientHostID, clientIP, clientPort54001) // curl (1) client54002NodeID = report.MakeEndpointNodeID(clientHostID, clientIP, clientPort54002) // curl (2) + server80NodeID = report.MakeEndpointNodeID(serverHostID, serverIP, serverPort) // apache unknownClient1NodeID = report.MakeEndpointNodeID(serverHostID, "10.10.10.10", "54010") // we want to ensure two unknown clients, connnected unknownClient2NodeID = report.MakeEndpointNodeID(serverHostID, "10.10.10.10", "54020") // to the same server, are deduped. unknownClient3NodeID = report.MakeEndpointNodeID(serverHostID, "10.10.10.11", "54020") // Check this one isn't deduped - server80NodeID = report.MakeEndpointNodeID(serverHostID, serverIP, serverPort) // apache - - clientAddressNodeID = report.MakeAddressNodeID(clientHostID, "10.10.10.20") - serverAddressNodeID = report.MakeAddressNodeID(serverHostID, "192.168.1.1") - randomAddressNodeID = report.MakeAddressNodeID(randomHostID, "172.16.11.9") // only in Address topology - unknownAddressNodeID = report.MakeAddressNodeID(unknownHostID, "10.10.10.10") + randomClientNodeID = report.MakeEndpointNodeID(serverHostID, "51.52.53.54", "12345") // this should become an internet node clientProcessNodeID = report.MakeProcessNodeID(clientHostID, clientPID) serverProcessNodeID = report.MakeProcessNodeID(serverHostID, serverPID) @@ -58,6 +55,33 @@ var ( serverContainerImageID = "imageid456" clientContainerImageNodeID = report.MakeContainerNodeID(clientHostID, clientContainerImageID) serverContainerImageNodeID = report.MakeContainerNodeID(serverHostID, serverContainerImageID) + + clientAddressNodeID = report.MakeAddressNodeID(clientHostID, "10.10.10.20") + serverAddressNodeID = report.MakeAddressNodeID(serverHostID, "192.168.1.1") + unknownAddress1NodeID = report.MakeAddressNodeID(serverHostID, "10.10.10.10") + unknownAddress2NodeID = report.MakeAddressNodeID(serverHostID, "10.10.10.11") + randomAddressNodeID = report.MakeAddressNodeID(serverHostID, "51.52.53.54") // this should become an internet node + + unknownPseudoNode1ID = "pseudo;10.10.10.10;192.168.1.1;80" + unknownPseudoNode2ID = "pseudo;10.10.10.11;192.168.1.1;80" + unknownPseudoNode1 = render.RenderableNode{ + ID: unknownPseudoNode1ID, + LabelMajor: "10.10.10.10", + Pseudo: true, + AggregateMetadata: render.AggregateMetadata{}, + } + unknownPseudoNode2 = render.RenderableNode{ + ID: unknownPseudoNode2ID, + LabelMajor: "10.10.10.11", + Pseudo: true, + AggregateMetadata: render.AggregateMetadata{}, + } + theInternetNode = render.RenderableNode{ + ID: render.TheInternetID, + LabelMajor: render.TheInternetMajor, + Pseudo: true, + AggregateMetadata: render.AggregateMetadata{}, + } ) var ( @@ -66,7 +90,9 @@ var ( Adjacency: report.Adjacency{ report.MakeAdjacencyID(client54001NodeID): report.MakeIDList(server80NodeID), report.MakeAdjacencyID(client54002NodeID): report.MakeIDList(server80NodeID), - report.MakeAdjacencyID(server80NodeID): report.MakeIDList(client54001NodeID, client54002NodeID, unknownClient1NodeID, unknownClient2NodeID, unknownClient3NodeID), + report.MakeAdjacencyID(server80NodeID): report.MakeIDList( + client54001NodeID, client54002NodeID, unknownClient1NodeID, unknownClient2NodeID, + unknownClient3NodeID, randomClientNodeID), }, NodeMetadatas: report.NodeMetadatas{ // NodeMetadata is arbitrary. We're free to put only precisely what we @@ -186,21 +212,16 @@ var ( Address: report.Topology{ Adjacency: report.Adjacency{ report.MakeAdjacencyID(clientAddressNodeID): report.MakeIDList(serverAddressNodeID), - report.MakeAdjacencyID(randomAddressNodeID): report.MakeIDList(serverAddressNodeID), - report.MakeAdjacencyID(serverAddressNodeID): report.MakeIDList(clientAddressNodeID, unknownAddressNodeID), // no backlink to random + report.MakeAdjacencyID(serverAddressNodeID): report.MakeIDList( + clientAddressNodeID, unknownAddress1NodeID, unknownAddress2NodeID, randomAddressNodeID), // no backlinks to unknown/random }, NodeMetadatas: report.NodeMetadatas{ clientAddressNodeID: report.NodeMetadata{ - "name": "client.hostname.com", // hostname - "host_name": "client.hostname.com", + "addr": clientIP, report.HostNodeID: clientHostNodeID, }, - randomAddressNodeID: report.NodeMetadata{ - "name": "random.hostname.com", // hostname - report.HostNodeID: randomHostNodeID, - }, serverAddressNodeID: report.NodeMetadata{ - "name": "server.hostname.com", // hostname + "addr": serverIP, report.HostNodeID: serverHostNodeID, }, }, @@ -209,28 +230,28 @@ var ( WithConnCountTCP: true, MaxConnCountTCP: 3, }, - report.MakeEdgeID(randomAddressNodeID, serverAddressNodeID): report.EdgeMetadata{ - WithConnCountTCP: true, - MaxConnCountTCP: 20, // dangling connections, weird but possible - }, report.MakeEdgeID(serverAddressNodeID, clientAddressNodeID): report.EdgeMetadata{ WithConnCountTCP: true, MaxConnCountTCP: 3, }, - report.MakeEdgeID(serverAddressNodeID, unknownAddressNodeID): report.EdgeMetadata{ - WithConnCountTCP: true, - MaxConnCountTCP: 7, - }, }, }, Host: report.Topology{ Adjacency: report.Adjacency{}, NodeMetadatas: report.NodeMetadatas{ + clientHostNodeID: report.NodeMetadata{ + "host_name": clientHostName, + "local_networks": "10.10.10.0/24", + "os": "Linux", + "load": "0.01 0.01 0.01", + report.HostNodeID: clientHostNodeID, + }, serverHostNodeID: report.NodeMetadata{ - "host_name": serverHostName, - "local_networks": "10.10.10.0/24", - "os": "Linux", - "load": "0.01 0.01 0.01", + "host_name": serverHostName, + "local_networks": "10.10.10.0/24", + "os": "Linux", + "load": "0.01 0.01 0.01", + report.HostNodeID: serverHostNodeID, }, }, EdgeMetadatas: report.EdgeMetadatas{}, @@ -282,8 +303,9 @@ func TestProcessRenderer(t *testing.T) { Pseudo: false, Adjacency: report.MakeIDList( clientProcessID, - "pseudo;10.10.10.10;192.168.1.1;80", - "pseudo;10.10.10.11;192.168.1.1;80", + unknownPseudoNode1ID, + unknownPseudoNode2ID, + render.TheInternetID, ), Origins: report.MakeIDList(server80NodeID, serverProcessNodeID, serverHostNodeID), AggregateMetadata: render.AggregateMetadata{ @@ -301,18 +323,9 @@ func TestProcessRenderer(t *testing.T) { Origins: report.MakeIDList(nonContainerProcessNodeID, serverHostNodeID), AggregateMetadata: render.AggregateMetadata{}, }, - "pseudo;10.10.10.10;192.168.1.1;80": { - ID: "pseudo;10.10.10.10;192.168.1.1;80", - LabelMajor: "10.10.10.10", - Pseudo: true, - AggregateMetadata: render.AggregateMetadata{}, - }, - "pseudo;10.10.10.11;192.168.1.1;80": { - ID: "pseudo;10.10.10.11;192.168.1.1;80", - LabelMajor: "10.10.10.11", - Pseudo: true, - AggregateMetadata: render.AggregateMetadata{}, - }, + unknownPseudoNode1ID: unknownPseudoNode1, + unknownPseudoNode2ID: unknownPseudoNode2, + render.TheInternetID: theInternetNode, } have := render.ProcessRenderer.Render(rpt) have = trimNodeMetadata(have) @@ -347,8 +360,9 @@ func TestProcessNameRenderer(t *testing.T) { Pseudo: false, Adjacency: report.MakeIDList( "curl", - "pseudo;10.10.10.10;192.168.1.1;80", - "pseudo;10.10.10.11;192.168.1.1;80", + unknownPseudoNode1ID, + unknownPseudoNode2ID, + render.TheInternetID, ), Origins: report.MakeIDList(server80NodeID, serverProcessNodeID, serverHostNodeID), AggregateMetadata: render.AggregateMetadata{ @@ -365,18 +379,9 @@ func TestProcessNameRenderer(t *testing.T) { Origins: report.MakeIDList(nonContainerProcessNodeID, serverHostNodeID), AggregateMetadata: render.AggregateMetadata{}, }, - "pseudo;10.10.10.10;192.168.1.1;80": { - ID: "pseudo;10.10.10.10;192.168.1.1;80", - LabelMajor: "10.10.10.10", - Pseudo: true, - AggregateMetadata: render.AggregateMetadata{}, - }, - "pseudo;10.10.10.11;192.168.1.1;80": { - ID: "pseudo;10.10.10.11;192.168.1.1;80", - LabelMajor: "10.10.10.11", - Pseudo: true, - AggregateMetadata: render.AggregateMetadata{}, - }, + unknownPseudoNode1ID: unknownPseudoNode1, + unknownPseudoNode2ID: unknownPseudoNode2, + render.TheInternetID: theInternetNode, } have := render.ProcessNameRenderer.Render(rpt) have = trimNodeMetadata(have) @@ -406,7 +411,7 @@ func TestContainerRenderer(t *testing.T) { LabelMinor: serverHostName, Rank: serverContainerImageID, Pseudo: false, - Adjacency: report.MakeIDList(clientContainerID, render.UncontainedID), + Adjacency: report.MakeIDList(clientContainerID, render.UncontainedID, render.TheInternetID), Origins: report.MakeIDList(serverContainerNodeID, server80NodeID, serverProcessNodeID, serverHostNodeID), AggregateMetadata: render.AggregateMetadata{ render.KeyBytesIngress: 150, @@ -422,6 +427,7 @@ func TestContainerRenderer(t *testing.T) { Origins: report.MakeIDList(nonContainerProcessNodeID, serverHostNodeID), AggregateMetadata: render.AggregateMetadata{}, }, + render.TheInternetID: theInternetNode, } have := render.ContainerRenderer.Render(rpt) have = trimNodeMetadata(have) @@ -451,7 +457,7 @@ func TestContainerImageRenderer(t *testing.T) { LabelMinor: "", Rank: serverContainerImageID, Pseudo: false, - Adjacency: report.MakeIDList(clientContainerImageID, render.UncontainedID), + Adjacency: report.MakeIDList(clientContainerImageID, render.UncontainedID, render.TheInternetID), Origins: report.MakeIDList(serverContainerImageNodeID, serverContainerNodeID, server80NodeID, serverProcessNodeID, serverHostNodeID), AggregateMetadata: render.AggregateMetadata{ render.KeyBytesIngress: 150, @@ -467,6 +473,7 @@ func TestContainerImageRenderer(t *testing.T) { Origins: report.MakeIDList(nonContainerProcessNodeID, serverHostNodeID), AggregateMetadata: render.AggregateMetadata{}, }, + render.TheInternetID: theInternetNode, } have := render.ContainerImageRenderer.Render(rpt) have = trimNodeMetadata(have) @@ -475,60 +482,47 @@ func TestContainerImageRenderer(t *testing.T) { } } -func TestRenderByNetworkHostname(t *testing.T) { +func TestHostRenderer(t *testing.T) { want := render.RenderableNodes{ - "host:client.hostname.com": { - ID: "host:client.hostname.com", - LabelMajor: "client", // before first . + "host:server.hostname.com": { + ID: "host:server.hostname.com", + LabelMajor: "server", // before first . LabelMinor: "hostname.com", // after first . - Rank: "client", + Rank: "hostname.com", Pseudo: false, - Adjacency: report.MakeIDList("host:server.hostname.com"), - Origins: report.MakeIDList(report.MakeHostNodeID("client.hostname.com"), report.MakeAddressNodeID("client.hostname.com", "10.10.10.20")), + Adjacency: report.MakeIDList("host:client.hostname.com", render.TheInternetID, "pseudo;10.10.10.10;192.168.1.1;", "pseudo;10.10.10.11;192.168.1.1;"), + Origins: report.MakeIDList(serverHostNodeID, serverAddressNodeID), AggregateMetadata: render.AggregateMetadata{ render.KeyMaxConnCountTCP: 3, }, }, - "host:random.hostname.com": { - ID: "host:random.hostname.com", - LabelMajor: "random", // before first . + "host:client.hostname.com": { + ID: "host:client.hostname.com", + LabelMajor: "client", // before first . LabelMinor: "hostname.com", // after first . - Rank: "random", + Rank: "hostname.com", Pseudo: false, Adjacency: report.MakeIDList("host:server.hostname.com"), - Origins: report.MakeIDList(report.MakeHostNodeID("random.hostname.com"), report.MakeAddressNodeID("random.hostname.com", "172.16.11.9")), + Origins: report.MakeIDList(clientHostNodeID, clientAddressNodeID), AggregateMetadata: render.AggregateMetadata{ - render.KeyMaxConnCountTCP: 20, - }, - }, - "host:server.hostname.com": { - ID: "host:server.hostname.com", - LabelMajor: "server", // before first . - LabelMinor: "hostname.com", // after first . - Rank: "server", - Pseudo: false, - Adjacency: report.MakeIDList("host:client.hostname.com", "pseudo;10.10.10.10;192.168.1.1;"), - Origins: report.MakeIDList(report.MakeHostNodeID("server.hostname.com"), report.MakeAddressNodeID("server.hostname.com", "192.168.1.1")), - AggregateMetadata: render.AggregateMetadata{ - render.KeyMaxConnCountTCP: 10, + render.KeyMaxConnCountTCP: 3, }, }, "pseudo;10.10.10.10;192.168.1.1;": { ID: "pseudo;10.10.10.10;192.168.1.1;", LabelMajor: "10.10.10.10", - LabelMinor: "", // after first . - Rank: "", Pseudo: true, - Adjacency: nil, - Origins: nil, AggregateMetadata: render.AggregateMetadata{}, }, + "pseudo;10.10.10.11;192.168.1.1;": { + ID: "pseudo;10.10.10.11;192.168.1.1;", + LabelMajor: "10.10.10.11", + Pseudo: true, + AggregateMetadata: render.AggregateMetadata{}, + }, + render.TheInternetID: theInternetNode, } - have := render.LeafMap{ - Selector: report.SelectAddress, - Mapper: render.NetworkHostname, - Pseudo: render.GenericPseudoNode, - }.Render(rpt) + have := render.HostRenderer.Render(rpt) have = trimNodeMetadata(have) if !reflect.DeepEqual(want, have) { t.Error("\n" + test.Diff(want, have)) diff --git a/report/id.go b/report/id.go index f29a919056..f8756a8215 100644 --- a/report/id.go +++ b/report/id.go @@ -1,7 +1,6 @@ package report import ( - "fmt" "net" "strings" ) @@ -131,13 +130,6 @@ func AddressIDAddresser(id string) net.IP { return net.ParseIP(fields[1]) } -// PanicIDAddresser will panic if it's ever called. It's used in topologies -// where there are never any edges, and so it's nonsensical to try and extract -// IPs from the node IDs. -func PanicIDAddresser(id string) net.IP { - panic(fmt.Sprintf("PanicIDAddresser called on %q", id)) -} - func isLoopback(address string) bool { ip := net.ParseIP(address) return ip != nil && ip.IsLoopback() diff --git a/report/id_test.go b/report/id_test.go index 23c7d65e3e..0ac85fc7fb 100644 --- a/report/id_test.go +++ b/report/id_test.go @@ -126,17 +126,3 @@ func TestAddressIDAddresser(t *testing.T) { t.Errorf("want %s, have %s", want, have) } } - -func TestPanicIDAddresser(t *testing.T) { - if panicked := func() (recovered bool) { - defer func() { - if r := recover(); r != nil { - recovered = true - } - }() - report.PanicIDAddresser("irrelevant") - return false - }(); !panicked { - t.Errorf("expected panic, didn't get it") - } -} diff --git a/report/report.go b/report/report.go index aabd75d2b2..dc8f72d724 100644 --- a/report/report.go +++ b/report/report.go @@ -58,11 +58,6 @@ func SelectProcess(r Report) Topology { return r.Process } -// SelectAddress selects the address topology. -func SelectAddress(r Report) Topology { - return r.Address -} - // SelectContainer selects the container topology. func SelectContainer(r Report) Topology { return r.Container @@ -73,6 +68,16 @@ func SelectContainerImage(r Report) Topology { return r.ContainerImage } +// SelectAddress selects the address topology. +func SelectAddress(r Report) Topology { + return r.Address +} + +// SelectHost selects the address topology. +func SelectHost(r Report) Topology { + return r.Host +} + // MakeReport makes a clean report, ready to Merge() other reports into. func MakeReport() Report { return Report{