From 23fdf47291a917a555119a6f5b3283bb608ca9ef Mon Sep 17 00:00:00 2001 From: Paul Bellamy Date: Wed, 27 Apr 2016 11:26:16 +0100 Subject: [PATCH] Filter by Kubernetes Namespaces --- app/api_topologies.go | 137 ++++++++++++++++++++++++---------------- app/api_topology.go | 50 +++++++++------ app/router.go | 6 +- probe/kubernetes/pod.go | 35 +++++----- render/filters.go | 8 +++ render/pod.go | 56 ++++++++-------- 6 files changed, 170 insertions(+), 122 deletions(-) diff --git a/app/api_topologies.go b/app/api_topologies.go index 0d4a61fdd2..7f841973c0 100644 --- a/app/api_topologies.go +++ b/app/api_topologies.go @@ -1,6 +1,7 @@ package app import ( + "fmt" "net/http" "sort" "sync" @@ -8,6 +9,7 @@ import ( "github.com/gorilla/mux" "golang.org/x/net/context" + "github.com/weaveworks/scope/probe/kubernetes" "github.com/weaveworks/scope/render" "github.com/weaveworks/scope/report" ) @@ -21,30 +23,6 @@ var ( ) func init() { - serviceFilters := []APITopologyOptionGroup{ - { - ID: "system", - Default: "application", - Options: []APITopologyOption{ - {"system", "System services", render.IsSystem}, - {"application", "Application services", render.IsApplication}, - {"both", "Both", render.Noop}, - }, - }, - } - - podFilters := []APITopologyOptionGroup{ - { - ID: "system", - Default: "application", - Options: []APITopologyOption{ - {"system", "System pods", render.IsSystem}, - {"application", "Application pods", render.IsApplication}, - {"both", "Both", render.Noop}, - }, - }, - } - containerFilters := []APITopologyOptionGroup{ { ID: "system", @@ -116,19 +94,12 @@ func init() { Name: "by DNS name", Options: containerFilters, }, - APITopologyDesc{ - id: "hosts", - renderer: render.HostRenderer, - Name: "Hosts", - Rank: 4, - }, APITopologyDesc{ id: "pods", renderer: render.PodRenderer, Name: "Pods", Rank: 3, HideIfEmpty: true, - Options: podFilters, }, APITopologyDesc{ id: "pods-by-service", @@ -136,11 +107,60 @@ func init() { renderer: render.PodServiceRenderer, Name: "by service", HideIfEmpty: true, - Options: serviceFilters, + }, + APITopologyDesc{ + id: "hosts", + renderer: render.HostRenderer, + Name: "Hosts", + Rank: 4, }, ) } +// kubernetesFilters generates the current kubernetes filters based on the +// available k8s topologies. +func kubernetesFilters(namespaces ...string) APITopologyOptionGroup { + options := APITopologyOptionGroup{ID: "namespace", Default: "all"} + for _, namespace := range namespaces { + options.Options = append(options.Options, APITopologyOption{namespace, namespace, render.IsNamespace(namespace)}) + } + options.Options = append(options.Options, APITopologyOption{"all", "All Namespaces", render.Noop}) + return options +} + +// updateFilters updates the available filters based on the current report. +// Currently only kubernetes changes. +func updateFilters(rpt report.Report, topologies []APITopologyDesc) []APITopologyDesc { + namespaces := map[string]struct{}{} + for _, t := range []report.Topology{rpt.Pod, rpt.Service} { + for _, n := range t.Nodes { + if namespace, ok := n.Latest.Lookup(kubernetes.Namespace); ok { + namespaces[namespace] = struct{}{} + } + } + } + var ns []string + for namespace := range namespaces { + ns = append(ns, namespace) + } + sort.Strings(ns) + for i, t := range topologies { + if t.id == "pods" || t.id == "pods-by-service" { + topologies[i] = updateTopologyFilters(t, []APITopologyOptionGroup{kubernetesFilters(ns...)}) + } + } + return topologies +} + +// updateTopologyFilters recursively sets the options on a topology description +func updateTopologyFilters(t APITopologyDesc, options []APITopologyOptionGroup) APITopologyDesc { + t.Options = options + for i, sub := range t.SubTopologies { + t.SubTopologies[i] = updateTopologyFilters(sub, options) + } + return t +} + // registry is a threadsafe store of the available topologies type registry struct { sync.RWMutex @@ -239,23 +259,27 @@ func (r *registry) makeTopologyList(rep Reporter) CtxHandlerFunc { respondWith(w, http.StatusInternalServerError, err.Error()) return } - topologies := r.renderTopologies(report, req) - respondWith(w, http.StatusOK, topologies) + respondWith(w, http.StatusOK, r.renderTopologies(report, req)) } } func (r *registry) renderTopologies(rpt report.Report, req *http.Request) []APITopologyDesc { topologies := []APITopologyDesc{} + req.ParseForm() + values := map[string]string{} + for k, vs := range req.Form { + values[k] = vs[0] + } r.walk(func(desc APITopologyDesc) { - renderer, decorator := renderedForRequest(req, desc) + renderer, decorator, _ := r.rendererForTopology(desc.id, values, rpt) desc.Stats = decorateWithStats(rpt, renderer, decorator) for i := range desc.SubTopologies { - renderer, decorator := renderedForRequest(req, desc.SubTopologies[i]) + renderer, decorator, _ := r.rendererForTopology(desc.id, values, rpt) desc.SubTopologies[i].Stats = decorateWithStats(rpt, renderer, decorator) } topologies = append(topologies, desc) }) - return topologies + return updateFilters(rpt, topologies) } func decorateWithStats(rpt report.Report, renderer render.Renderer, decorator render.Decorator) topologyStats { @@ -280,10 +304,16 @@ func decorateWithStats(rpt report.Report, renderer render.Renderer, decorator re } } -func renderedForRequest(r *http.Request, topology APITopologyDesc) (render.Renderer, render.Decorator) { +func (r *registry) rendererForTopology(id string, values map[string]string, rpt report.Report) (render.Renderer, render.Decorator, error) { + topology, ok := r.get(id) + if !ok { + return nil, nil, fmt.Errorf("topology not found: %s", id) + } + topology = updateFilters(rpt, []APITopologyDesc{topology})[0] + var filters []render.FilterFunc for _, group := range topology.Options { - value := r.FormValue(group.ID) + value := values[group.ID] for _, opt := range group.Options { if opt.filter == nil { continue @@ -299,23 +329,22 @@ func renderedForRequest(r *http.Request, topology APITopologyDesc) (render.Rende return render.MakeFilter(render.ComposeFilterFuncs(filters...), renderer) } } - return topology.renderer, decorator + return topology.renderer, decorator, nil } -type reportRenderHandler func( - context.Context, - Reporter, render.Renderer, render.Decorator, - http.ResponseWriter, *http.Request, -) +type reportRenderHandler func(context.Context, Reporter, http.ResponseWriter, *http.Request) -func (r *registry) captureRenderer(rep Reporter, f reportRenderHandler) CtxHandlerFunc { - return func(ctx context.Context, w http.ResponseWriter, req *http.Request) { - topology, ok := r.get(mux.Vars(req)["topology"]) - if !ok { - http.NotFound(w, req) - return - } - renderer, decorator := renderedForRequest(req, topology) - f(ctx, rep, renderer, decorator, w, req) +func (r *registry) rendererForRequest(req *http.Request, rpt report.Report) (render.Renderer, render.Decorator, error) { + req.ParseForm() + values := map[string]string{} + for k, vs := range req.Form { + values[k] = vs[0] + } + return r.rendererForTopology(mux.Vars(req)["topology"], values, rpt) +} + +func captureReporter(rep Reporter, f reportRenderHandler) CtxHandlerFunc { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request) { + f(ctx, rep, w, r) } } diff --git a/app/api_topology.go b/app/api_topology.go index d9ecd73788..da55bdd58e 100644 --- a/app/api_topology.go +++ b/app/api_topology.go @@ -9,7 +9,6 @@ import ( "golang.org/x/net/context" "github.com/weaveworks/scope/common/xfer" - "github.com/weaveworks/scope/render" "github.com/weaveworks/scope/render/detailed" ) @@ -28,27 +27,24 @@ type APINode struct { } // Full topology. -func handleTopology( - ctx context.Context, - rep Reporter, renderer render.Renderer, decorator render.Decorator, - w http.ResponseWriter, r *http.Request, -) { +func handleTopology(ctx context.Context, rep Reporter, w http.ResponseWriter, r *http.Request) { report, err := rep.Report(ctx) if err != nil { respondWith(w, http.StatusInternalServerError, err.Error()) return } + renderer, decorator, err := topologyRegistry.rendererForRequest(r, report) + if err != nil { + http.NotFound(w, r) + return + } respondWith(w, http.StatusOK, APITopology{ Nodes: detailed.Summaries(report, renderer.Render(report, decorator)), }) } // Websocket for the full topology. This route overlaps with the next. -func handleWs( - ctx context.Context, - rep Reporter, renderer render.Renderer, decorator render.Decorator, - w http.ResponseWriter, r *http.Request, -) { +func handleWs(ctx context.Context, rep Reporter, w http.ResponseWriter, r *http.Request) { if err := r.ParseForm(); err != nil { respondWith(w, http.StatusInternalServerError, err.Error()) return @@ -61,22 +57,31 @@ func handleWs( return } } - handleWebsocket(ctx, w, r, rep, renderer, decorator, loop) + handleWebsocket(ctx, w, r, rep, loop) } // Individual nodes. -func handleNode( - ctx context.Context, - rep Reporter, renderer render.Renderer, _ render.Decorator, - w http.ResponseWriter, r *http.Request, -) { +func handleNode(ctx context.Context, rep Reporter, w http.ResponseWriter, r *http.Request) { var ( vars = mux.Vars(r) topologyID = vars["topology"] nodeID = vars["id"] report, err = rep.Report(ctx) - rendered = renderer.Render(report, render.FilterNoop) - node, ok = rendered[nodeID] + ) + if err != nil { + respondWith(w, http.StatusInternalServerError, err.Error()) + return + } + + renderer, _, err := topologyRegistry.rendererForRequest(r, report) + if err != nil { + http.NotFound(w, r) + return + } + + var ( + rendered = renderer.Render(report, nil) + node, ok = rendered[nodeID] ) if err != nil { respondWith(w, http.StatusInternalServerError, err.Error()) @@ -94,8 +99,6 @@ func handleWebsocket( w http.ResponseWriter, r *http.Request, rep Reporter, - renderer render.Renderer, - decorator render.Decorator, loop time.Duration, ) { conn, err := xfer.Upgrade(w, r, nil) @@ -132,6 +135,11 @@ func handleWebsocket( log.Errorf("Error generating report: %v", err) return } + renderer, decorator, err := topologyRegistry.rendererForRequest(r, report) + if err != nil { + log.Errorf("Error generating report: %v", err) + return + } newTopo := detailed.Summaries(report, renderer.Render(report, decorator)) diff := detailed.TopoDiff(previousTopo, newTopo) previousTopo = newTopo diff --git a/app/router.go b/app/router.go index d2ca136dda..d8467f4e79 100644 --- a/app/router.go +++ b/app/router.go @@ -89,11 +89,11 @@ func RegisterTopologyRoutes(router *mux.Router, r Reporter) { get.HandleFunc("/api/topology", gzipHandler(requestContextDecorator(topologyRegistry.makeTopologyList(r)))) get.HandleFunc("/api/topology/{topology}", - gzipHandler(requestContextDecorator(topologyRegistry.captureRenderer(r, handleTopology)))) + gzipHandler(requestContextDecorator(captureReporter(r, handleTopology)))) get.HandleFunc("/api/topology/{topology}/ws", - requestContextDecorator(topologyRegistry.captureRenderer(r, handleWs))) // NB not gzip! + requestContextDecorator(captureReporter(r, handleWs))) // NB not gzip! get.MatcherFunc(URLMatcher("/api/topology/{topology}/{id}")).HandlerFunc( - gzipHandler(requestContextDecorator(topologyRegistry.captureRenderer(r, handleNode)))) + gzipHandler(requestContextDecorator(captureReporter(r, handleNode)))) get.HandleFunc("/api/report", gzipHandler(requestContextDecorator(makeRawReportHandler(r)))) get.HandleFunc("/api/probes", diff --git a/probe/kubernetes/pod.go b/probe/kubernetes/pod.go index b739386a1e..b9cba337fa 100644 --- a/probe/kubernetes/pod.go +++ b/probe/kubernetes/pod.go @@ -28,7 +28,7 @@ type Pod interface { ID() string Name() string Namespace() string - ContainerIDs() []string + ContainerIDs() report.StringSet Created() string AddServiceID(id string) Labels() labels.Labels @@ -38,13 +38,13 @@ type Pod interface { type pod struct { *api.Pod - serviceIDs []string + serviceIDs report.StringSet Node *api.Node } // NewPod creates a new Pod func NewPod(p *api.Pod) Pod { - return &pod{Pod: p} + return &pod{Pod: p, serviceIDs: report.MakeStringSet()} } func (p *pod) ID() string { @@ -63,10 +63,10 @@ func (p *pod) Created() string { return p.ObjectMeta.CreationTimestamp.Format(time.RFC822) } -func (p *pod) ContainerIDs() []string { - ids := []string{} +func (p *pod) ContainerIDs() report.StringSet { + ids := report.MakeStringSet() for _, container := range p.Status.ContainerStatuses { - ids = append(ids, strings.TrimPrefix(container.ContainerID, "docker://")) + ids = ids.Add(strings.TrimPrefix(container.ContainerID, "docker://")) } return ids } @@ -76,7 +76,7 @@ func (p *pod) Labels() labels.Labels { } func (p *pod) AddServiceID(id string) { - p.serviceIDs = append(p.serviceIDs, id) + p.serviceIDs = p.serviceIDs.Add(id) } func (p *pod) State() string { @@ -89,18 +89,17 @@ func (p *pod) NodeName() string { func (p *pod) GetNode(probeID string) report.Node { n := report.MakeNodeWith(report.MakePodNodeID(p.Namespace(), p.Name()), map[string]string{ - PodID: p.ID(), - PodName: p.Name(), - Namespace: p.Namespace(), - PodCreated: p.Created(), - PodContainerIDs: strings.Join(p.ContainerIDs(), " "), - PodState: p.State(), - PodIP: p.Status.PodIP, + PodID: p.ID(), + PodName: p.Name(), + Namespace: p.Namespace(), + PodCreated: p.Created(), + PodState: p.State(), + PodIP: p.Status.PodIP, report.ControlProbeID: probeID, - }) - if len(p.serviceIDs) > 0 { - n = n.WithLatests(map[string]string{ServiceIDs: strings.Join(p.serviceIDs, " ")}) - } + }).WithSets(report.EmptySets. + Add(PodContainerIDs, p.ContainerIDs()). + Add(ServiceIDs, p.serviceIDs), + ) for _, serviceID := range p.serviceIDs { segments := strings.SplitN(serviceID, "/", 2) if len(segments) != 2 { diff --git a/render/filters.go b/render/filters.go index 6b18ee459f..0d075ce326 100644 --- a/render/filters.go +++ b/render/filters.go @@ -275,6 +275,14 @@ func HasChildren(topology string) FilterFunc { } } +// IsNamespace checks if the node is a pod/service in the specified namespace +func IsNamespace(namespace string) FilterFunc { + return func(n report.Node) bool { + gotNamespace, ok := n.Latest.Lookup(kubernetes.Namespace) + return !ok || namespace == gotNamespace + } +} + var systemContainerNames = map[string]struct{}{ "weavescope": {}, "weavedns": {}, diff --git a/render/pod.go b/render/pod.go index c3cc7e77fa..7c4654cdc2 100644 --- a/render/pod.go +++ b/render/pod.go @@ -15,37 +15,41 @@ const ( // PodRenderer is a Renderer which produces a renderable kubernetes // graph by merging the container graph and the pods topology. -var PodRenderer = MakeFilter( - func(n report.Node) bool { - // Drop deleted or empty pods - state, ok := n.Latest.Lookup(kubernetes.PodState) - return HasChildren(report.Container)(n) && (!ok || state != kubernetes.StateDeleted) - }, - MakeReduce( - MakeFilter( - func(n report.Node) bool { - // Drop unconnected pseudo nodes (could appear due to filtering) - _, isConnected := n.Latest.Lookup(IsConnected) - return n.Topology != Pseudo || isConnected - }, - ColorConnected(MakeMap( - MapContainer2Pod, - ContainerWithImageNameRenderer, - )), +var PodRenderer = ApplyDecorators( + MakeFilter( + func(n report.Node) bool { + // Drop deleted or empty pods + state, ok := n.Latest.Lookup(kubernetes.PodState) + return HasChildren(report.Container)(n) && (!ok || state != kubernetes.StateDeleted) + }, + MakeReduce( + MakeFilter( + func(n report.Node) bool { + // Drop unconnected pseudo nodes (could appear due to filtering) + _, isConnected := n.Latest.Lookup(IsConnected) + return n.Topology != Pseudo || isConnected + }, + ColorConnected(MakeMap( + MapContainer2Pod, + ContainerWithImageNameRenderer, + )), + ), + SelectPod, ), - SelectPod, ), ) // PodServiceRenderer is a Renderer which produces a renderable kubernetes services // graph by merging the pods graph and the services topology. -var PodServiceRenderer = FilterEmpty(report.Pod, - MakeReduce( - MakeMap( - MapPod2Service, - PodRenderer, +var PodServiceRenderer = ApplyDecorators( + FilterEmpty(report.Pod, + MakeReduce( + MakeMap( + MapPod2Service, + PodRenderer, + ), + SelectService, ), - SelectService, ), ) @@ -125,13 +129,13 @@ func MapPod2Service(pod report.Node, _ report.Networks) report.Nodes { if !ok { return report.Nodes{} } - ids, ok := pod.Latest.Lookup(kubernetes.ServiceIDs) + serviceIDs, ok := pod.Sets.Lookup(kubernetes.ServiceIDs) if !ok { return report.Nodes{} } result := report.Nodes{} - for _, serviceID := range strings.Fields(ids) { + for _, serviceID := range serviceIDs { serviceName := strings.TrimPrefix(serviceID, namespace+"/") id := report.MakeServiceNodeID(namespace, serviceName) node := NewDerivedNode(id, pod).WithTopology(report.Service)