[Ingest Manager] Expose processes and their metrics (#24788)

[Ingest Manager] Expose processes and their metrics (#24788)

NOTICE.txt

@@ -10242,6 +10242,43 @@ Contents of probable licence file $GOMODCACHE/github.com/gorhill/[email protected]
    limitations under the License.
 
 
+--------------------------------------------------------------------------------
+Dependency : github.com/gorilla/mux
+Version: v1.7.2
+Licence type (autodetected): BSD-3-Clause
+--------------------------------------------------------------------------------
+
+Contents of probable licence file $GOMODCACHE/github.com/gorilla/[email protected]/LICENSE:
+
+Copyright (c) 2012-2018 The Gorilla Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+	 * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+	 * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+	 * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+
 --------------------------------------------------------------------------------
 Dependency : github.com/h2non/filetype
 Version: v1.1.1-0.20201130172452-f60988ab73d5
@@ -31247,43 +31284,6 @@ THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
---------------------------------------------------------------------------------
-Dependency : github.com/gorilla/mux
-Version: v1.7.2
-Licence type (autodetected): BSD-3-Clause
---------------------------------------------------------------------------------
-
-Contents of probable licence file $GOMODCACHE/github.com/gorilla/[email protected]/LICENSE:
-
-Copyright (c) 2012-2018 The Gorilla Authors. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-	 * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-	 * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-	 * Neither the name of Google Inc. nor the names of its
-contributors may be used to endorse or promote products derived from
-this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-
 --------------------------------------------------------------------------------
 Dependency : github.com/gorilla/websocket
 Version: v1.4.1

go.mod

@@ -95,7 +95,7 @@ require (
 	github.com/google/gopacket v1.1.18-0.20191009163724-0ad7f2610e34
 	github.com/google/uuid v1.1.2
 	github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75
-	github.com/gorilla/mux v1.7.2 // indirect
+	github.com/gorilla/mux v1.7.2
 	github.com/grpc-ecosystem/grpc-gateway v1.13.0 // indirect
 	github.com/h2non/filetype v1.1.1-0.20201130172452-f60988ab73d5
 	github.com/hashicorp/go-multierror v1.1.0

x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl

@@ -33,6 +33,15 @@ inputs:
 #   logs: true
 #   # enables metrics monitoring
 #   metrics: true
+#   # exposes agent metrics using http, by default sockets and named pipes are used
+#   http:
+#       # enables http endpoint
+#       enabled: false
+#       # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe.
+#       # When using IP addresses, it is recommended to only use localhost.
+#       host: localhost
+#       # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled.
+#       port: 0
 
 # # Allow fleet to reload its configuration locally on disk.
 # # Notes: Only specific process configuration will be reloaded.

x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl

@@ -107,6 +107,15 @@ inputs:
 #   logs: false
 #   # enables metrics monitoring
 #   metrics: false
+#   # exposes agent metrics using http, by default sockets and named pipes are used
+#   http:
+#       # enables http endpoint
+#       enabled: false
+#       # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe.
+#       # When using IP addresses, it is recommended to only use localhost.
+#       host: localhost
+#       # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled.
+#       port: 0
 
 # # Allow fleet to reload its configuration locally on disk.
 # # Notes: Only specific process configuration will be reloaded.

x-pack/elastic-agent/_meta/config/elastic-agent.docker.yml.tmpl

@@ -107,6 +107,15 @@ inputs:
 #   logs: false
 #   # enables metrics monitoring
 #   metrics: false
+#   # exposes agent metrics using http, by default sockets and named pipes are used
+#   http:
+#       # enables http endpoint
+#       enabled: false
+#       # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe.
+#       # When using IP addresses, it is recommended to only use localhost.
+#       host: localhost
+#       # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled.
+#       port: 0
 
 # # Allow fleet to reload its configuration locally on disk.
 # # Notes: Only specific process configuration will be reloaded.

x-pack/elastic-agent/_meta/elastic-agent.yml

@@ -102,6 +102,15 @@ inputs:
 #   logs: false
 #   # enables metrics monitoring
 #   metrics: false
+#   # exposes agent metrics using http, by default sockets and named pipes are used
+#   http:
+#       # enables http endpoint
+#       enabled: false
+#       # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe.
+#       # When using IP addresses, it is recommended to only use localhost.
+#       host: localhost
+#       # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled.
+#       port: 0
 
 # # Allow fleet to reload his configuration locally on disk.
 # # Notes: Only specific process configuration will be reloaded.

x-pack/elastic-agent/elastic-agent.docker.yml

@@ -107,6 +107,15 @@ inputs:
 #   logs: false
 #   # enables metrics monitoring
 #   metrics: false
+#   # exposes agent metrics using http, by default sockets and named pipes are used
+#   http:
+#       # enables http endpoint
+#       enabled: false
+#       # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe.
+#       # When using IP addresses, it is recommended to only use localhost.
+#       host: localhost
+#       # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled.
+#       port: 0
 
 # # Allow fleet to reload its configuration locally on disk.
 # # Notes: Only specific process configuration will be reloaded.

x-pack/elastic-agent/elastic-agent.reference.yml

@@ -113,6 +113,15 @@ inputs:
 #   logs: false
 #   # enables metrics monitoring
 #   metrics: false
+#   # exposes agent metrics using http, by default sockets and named pipes are used
+#   http:
+#       # enables http endpoint
+#       enabled: false
+#       # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe.
+#       # When using IP addresses, it is recommended to only use localhost.
+#       host: localhost
+#       # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled.
+#       port: 0
 
 # # Allow fleet to reload its configuration locally on disk.
 # # Notes: Only specific process configuration will be reloaded.

x-pack/elastic-agent/elastic-agent.yml

@@ -39,6 +39,15 @@ inputs:
 #   logs: true
 #   # enables metrics monitoring
 #   metrics: true
+#   # exposes agent metrics using http, by default sockets and named pipes are used
+#   http:
+#       # enables http endpoint
+#       enabled: false
+#       # The HTTP endpoint will bind to this hostname, IP address, unix socket or named pipe.
+#       # When using IP addresses, it is recommended to only use localhost.
+#       host: localhost
+#       # Port on which the HTTP endpoint will bind. Default is 0 meaning feature is disabled.
+#       port: 0
 
 # # Allow fleet to reload its configuration locally on disk.
 # # Notes: Only specific process configuration will be reloaded.

x-pack/elastic-agent/pkg/agent/application/application.go

@@ -11,6 +11,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/status"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
@@ -26,6 +27,7 @@ type Application interface {
 	Start() error
 	Stop() error
 	AgentInfo() *info.AgentInfo
+	Routes() *sorted.Set
 }
 
 type reexecManager interface {

x-pack/elastic-agent/pkg/agent/application/fleet_server_bootstrap.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/program"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted"
 	"github.com/elastic/go-sysinfo"
 
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filters"
@@ -113,6 +114,11 @@ func newFleetServerBootstrap(
 	return bootstrapApp, nil
 }
 
+// Routes returns a list of routes handled by server.
+func (b *FleetServerBootstrap) Routes() *sorted.Set {
+	return b.router.Routes()
+}
+
 // Start starts a managed elastic-agent.
 func (b *FleetServerBootstrap) Start() error {
 	b.log.Info("Agent is starting")

x-pack/elastic-agent/pkg/agent/application/local_mode.go

@@ -30,6 +30,7 @@ import (
 	acker "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi/acker/noop"
 	reporting "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter"
 	logreporter "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter/log"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted"
 )
 
 type discoverFunc func() ([]string, error)
@@ -157,6 +158,11 @@ func newLocal(
 	return localApplication, nil
 }
 
+// Routes returns a list of routes handled by agent.
+func (l *Local) Routes() *sorted.Set {
+	return l.router.Routes()
+}
+
 // Start starts a local agent.
 func (l *Local) Start() error {
 	l.log.Info("Agent is starting")

x-pack/elastic-agent/pkg/agent/application/managed_mode.go

@@ -43,6 +43,7 @@ import (
 	reporting "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter"
 	fleetreporter "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter/fleet"
 	logreporter "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/reporter/log"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted"
 )
 
 type stateStore interface {
@@ -278,6 +279,11 @@ func newManaged(
 	return managedApplication, nil
 }
 
+// Routes returns a list of routes handled by agent.
+func (m *Managed) Routes() *sorted.Set {
+	return m.router.Routes()
+}
+
 // Start starts a managed elastic-agent.
 func (m *Managed) Start() error {
 	m.log.Info("Agent is starting")

x-pack/elastic-agent/pkg/agent/application/pipeline/pipeline.go

@@ -13,6 +13,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted"
 )
 
 // ConfigHandler is capable of handling configrequest.
@@ -30,6 +31,7 @@ type RoutingKey = string
 
 // Router is an interace routes programs to correspongind stream
 type Router interface {
+	Routes() *sorted.Set
 	Route(id string, grpProg map[RoutingKey][]program.Program) error
 	Shutdown()
 }

x-pack/elastic-agent/pkg/agent/application/pipeline/router/router.go

@@ -34,6 +34,10 @@ func New(log *logger.Logger, factory pipeline.StreamFunc) (pipeline.Router, erro
 	return &router{log: log, streamFactory: factory, routes: sorted.NewSet()}, nil
 }
 
+func (r *router) Routes() *sorted.Set {
+	return r.routes
+}
+
 func (r *router) Route(id string, grpProg map[pipeline.RoutingKey][]program.Program) error {
 	s := sorted.NewSet()
 

x-pack/elastic-agent/pkg/agent/application/pipeline/stream/operator_stream.go

@@ -8,17 +8,30 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/pipeline"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/configrequest"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/state"
 )
 
 type operatorStream struct {
 	configHandler pipeline.ConfigHandler
 	log           *logger.Logger
 }
 
+type stater interface {
+	State() map[string]state.State
+}
+
 func (b *operatorStream) Close() error {
 	return b.configHandler.Close()
 }
 
+func (b *operatorStream) State() map[string]state.State {
+	if s, ok := b.configHandler.(stater); ok {
+		return s.State()
+	}
+
+	return nil
+}
+
 func (b *operatorStream) Execute(cfg configrequest.Request) error {
 	return b.configHandler.HandleConfig(cfg)
 }

x-pack/elastic-agent/pkg/agent/cmd/inspect.go

@@ -31,6 +31,7 @@ import (
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/monitoring/noop"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/status"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/sorted"
 	"github.com/elastic/go-sysinfo"
 )
 
@@ -299,6 +300,10 @@ type inmemRouter struct {
 	programs map[string][]program.Program
 }
 
+func (r *inmemRouter) Routes() *sorted.Set {
+	return nil
+}
+
 func (r *inmemRouter) Route(id string, grpProg map[pipeline.RoutingKey][]program.Program) error {
 	r.programs = grpProg
 	return nil