Skip to content

Commit

Permalink
V1 metrics monitoring for V2 (#1487)
Browse files Browse the repository at this point in the history
V1 metrics monitoring for V2 (#1487)
  • Loading branch information
michalpristas authored Oct 19, 2022
1 parent bfc490a commit ec83c2c
Show file tree
Hide file tree
Showing 23 changed files with 2,271 additions and 96 deletions.
932 changes: 888 additions & 44 deletions NOTICE.txt

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,14 @@ require (
github.com/elastic/elastic-agent-autodiscover v0.0.0-20220404145827-89887023c1ab
github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484
github.com/elastic/elastic-agent-libs v0.2.6
github.com/elastic/elastic-agent-system-metrics v0.4.4
github.com/elastic/go-licenser v0.4.0
github.com/elastic/go-sysinfo v1.7.1
github.com/elastic/go-ucfg v0.8.5
github.com/gofrs/flock v0.8.1
github.com/gofrs/uuid v4.2.0+incompatible
github.com/google/go-cmp v0.5.6
github.com/gorilla/mux v1.8.0
github.com/hashicorp/go-multierror v1.1.1
github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901
Expand All @@ -38,6 +40,7 @@ require (
github.com/spf13/cobra v1.3.0
github.com/stretchr/testify v1.7.0
github.com/tsg/go-daemon v0.0.0-20200207173439-e704b93fd89b
go.elastic.co/apm/module/apmgorilla v1.15.0
go.elastic.co/ecszap v1.0.1
go.elastic.co/go-licence-detector v0.5.0
go.uber.org/zap v1.21.0
Expand Down Expand Up @@ -69,7 +72,9 @@ require (
github.com/docker/distribution v2.8.1+incompatible // indirect
github.com/docker/docker v20.10.12+incompatible // indirect
github.com/docker/go-connections v0.4.0 // indirect
github.com/elastic/go-structform v0.0.9 // indirect
github.com/elastic/go-windows v1.0.1 // indirect
github.com/elastic/gosigar v0.14.2 // indirect
github.com/evanphx/json-patch v4.12.0+incompatible // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
Expand Down
11 changes: 11 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -383,12 +383,16 @@ github.com/elastic/elastic-agent-client/v7 v7.0.0-20220804181728-b0328d2fe484/go
github.com/elastic/elastic-agent-libs v0.0.0-20220303160015-5b4e674da3dd/go.mod h1://82M1l73IHx0wDbS2Tzkq6Fx9fkmytS1KgkIyzvNTM=
github.com/elastic/elastic-agent-libs v0.2.6 h1:DpcUcCVYZ7lNtHLUlyT1u/GtGAh49wpL15DTH7+8O5o=
github.com/elastic/elastic-agent-libs v0.2.6/go.mod h1:chO3rtcLyGlKi9S0iGVZhYCzDfdDsAQYBc+ui588AFE=
github.com/elastic/elastic-agent-system-metrics v0.4.4 h1:Br3S+TlBhijrLysOvbHscFhgQ00X/trDT5VEnOau0E0=
github.com/elastic/elastic-agent-system-metrics v0.4.4/go.mod h1:tF/f9Off38nfzTZHIVQ++FkXrDm9keFhFpJ+3pQ00iI=
github.com/elastic/elastic-package v0.32.1/go.mod h1:l1fEnF52XRBL6a5h6uAemtdViz2bjtjUtgdQcuRhEAY=
github.com/elastic/go-elasticsearch/v7 v7.16.0/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4=
github.com/elastic/go-elasticsearch/v8 v8.0.0-20210317102009-a9d74cec0186/go.mod h1:xe9a/L2aeOgFKKgrO3ibQTnMdpAeL0GC+5/HpGScSa4=
github.com/elastic/go-licenser v0.3.1/go.mod h1:D8eNQk70FOCVBl3smCGQt/lv7meBeQno2eI1S5apiHQ=
github.com/elastic/go-licenser v0.4.0 h1:jLq6A5SilDS/Iz1ABRkO6BHy91B9jBora8FwGRsDqUI=
github.com/elastic/go-licenser v0.4.0/go.mod h1:V56wHMpmdURfibNBggaSBfqgPxyT1Tldns1i87iTEvU=
github.com/elastic/go-structform v0.0.9 h1:HpcS7xljL4kSyUfDJ8cXTJC6rU5ChL1wYb6cx3HLD+o=
github.com/elastic/go-structform v0.0.9/go.mod h1:CZWf9aIRYY5SuKSmOhtXScE5uQiLZNqAFnwKR4OrIM4=
github.com/elastic/go-sysinfo v1.1.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
github.com/elastic/go-sysinfo v1.7.1 h1:Wx4DSARcKLllpKT2TnFVdSUJOsybqMYCNQZq1/wO+s0=
github.com/elastic/go-sysinfo v1.7.1/go.mod h1:i1ZYdU10oLNfRzq4vq62BEwD2fH8KaWh6eh0ikPT9F0=
Expand All @@ -398,6 +402,8 @@ github.com/elastic/go-ucfg v0.8.5/go.mod h1:4E8mPOLSUV9hQ7sgLEJ4bvt0KhMuDJa8joDT
github.com/elastic/go-windows v1.0.0/go.mod h1:TsU0Nrp7/y3+VwE82FoZF8gC/XFg/Elz6CcloAxnPgU=
github.com/elastic/go-windows v1.0.1 h1:AlYZOldA+UJ0/2nBuqWdo90GFCgG9xuyw9SYzGUtJm0=
github.com/elastic/go-windows v1.0.1/go.mod h1:FoVvqWSun28vaDQPbj2Elfc0JahhPB7WQEGa3c814Ss=
github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4=
github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs=
github.com/elastic/package-spec v1.3.0/go.mod h1:KzGTSDqCkdhmL1IFpOH2ZQNSSE9JEhNtndxU3ZrQilA=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
Expand Down Expand Up @@ -634,8 +640,10 @@ github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2c
github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw=
github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
Expand Down Expand Up @@ -1189,6 +1197,8 @@ go.elastic.co/apm v1.13.0/go.mod h1:dylGv2HKR0tiCV+wliJz1KHtDyuD8SPe69oV7VyK6WY=
go.elastic.co/apm v1.15.0 h1:uPk2g/whK7c7XiZyz/YCUnAUBNPiyNeE3ARX3G6Gx7Q=
go.elastic.co/apm v1.15.0/go.mod h1:dylGv2HKR0tiCV+wliJz1KHtDyuD8SPe69oV7VyK6WY=
go.elastic.co/apm/module/apmelasticsearch v1.10.0/go.mod h1:lwoaGDfZzfb9e6TXd3h8/KNmLAONOas7o5NLVNmv8Xk=
go.elastic.co/apm/module/apmgorilla v1.15.0 h1:1yTAksffgaFXYEIwlLRiQnxLfy3p3RtpDw8HDupIJfY=
go.elastic.co/apm/module/apmgorilla v1.15.0/go.mod h1:+23mZudYvZ9VgxCQjseLo9EF5gkKEr0KSQBupw+rzP8=
go.elastic.co/apm/module/apmgrpc v1.15.0 h1:Z7h58uuMJUoYXK6INFunlcGEXZQ18QKAhPh6NFYDNHE=
go.elastic.co/apm/module/apmgrpc v1.15.0/go.mod h1:IEbTGJzY5Xx737PkHDT3bbzh9syovK+IfAlckJsUgPE=
go.elastic.co/apm/module/apmhttp v1.10.0/go.mod h1:Y4timwcJ8sQWbWpcw3Y7Mat1OssNpGhpwyfUnpqIDew=
Expand Down Expand Up @@ -1429,6 +1439,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down
6 changes: 4 additions & 2 deletions internal/pkg/agent/application/application.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator"
"github.com/elastic/elastic-agent/internal/pkg/agent/application/info"
"github.com/elastic/elastic-agent/internal/pkg/agent/application/monitoring"
"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
"github.com/elastic/elastic-agent/internal/pkg/agent/application/upgrade"
"github.com/elastic/elastic-agent/internal/pkg/agent/configuration"
Expand Down Expand Up @@ -64,8 +65,9 @@ func New(
}

upgrader := upgrade.NewUpgrader(log, cfg.Settings.DownloadConfig, agentInfo)
monitor := monitoring.New(cfg.Settings.V1MonitoringEnabled, cfg.Settings.DownloadConfig.OS(), cfg.Settings.MonitoringConfig, agentInfo)

runtime, err := runtime.NewManager(log, cfg.Settings.GRPC.String(), agentInfo, tracer)
runtime, err := runtime.NewManager(log, cfg.Settings.GRPC.String(), agentInfo, tracer, monitor)
if err != nil {
return nil, fmt.Errorf("failed to initialize runtime manager: %w", err)
}
Expand Down Expand Up @@ -114,7 +116,7 @@ func New(
return nil, errors.New(err, "failed to initialize composable controller")
}

coord := coordinator.New(log, agentInfo, specs, reexec, upgrader, runtime, configMgr, composable, caps, compModifiers...)
coord := coordinator.New(log, agentInfo, specs, reexec, upgrader, runtime, configMgr, composable, caps, monitor, compModifiers...)
if managed != nil {
// the coordinator requires the config manager as well as in managed-mode the config manager requires the
// coordinator, so it must be set here once the coordinator is created
Expand Down
28 changes: 26 additions & 2 deletions internal/pkg/agent/application/coordinator/coordinator.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,18 @@ type UpgradeManager interface {
Ack(ctx context.Context, acker acker.Acker) error
}

// MonitorManager provides an interface to perform the monitoring action for the agent.
type MonitorManager interface {
// Enabled when configured to collect metrics/logs.
Enabled() bool

// Reload reloads the configuration for the upgrade manager.
Reload(rawConfig *config.Config) error

// InjectMonitoring injects monitoring configuration into resolved ast tree.
MonitoringConfig(map[string]interface{}, map[string]string) (map[string]interface{}, error)
}

// Runner provides interface to run a manager and receive running errors.
type Runner interface {
// Run runs the manager.
Expand Down Expand Up @@ -151,6 +163,7 @@ type Coordinator struct {

reexecMgr ReExecManager
upgradeMgr UpgradeManager
monitorMgr MonitorManager

runtimeMgr RuntimeManager
runtimeMgrErr error
Expand All @@ -166,7 +179,7 @@ type Coordinator struct {
}

// New creates a new coordinator.
func New(logger *logger.Logger, agentInfo *info.AgentInfo, specs component.RuntimeSpecs, reexecMgr ReExecManager, upgradeMgr UpgradeManager, runtimeMgr RuntimeManager, configMgr ConfigManager, varsMgr VarsManager, caps capabilities.Capability, modifiers ...ComponentsModifier) *Coordinator {
func New(logger *logger.Logger, agentInfo *info.AgentInfo, specs component.RuntimeSpecs, reexecMgr ReExecManager, upgradeMgr UpgradeManager, runtimeMgr RuntimeManager, configMgr ConfigManager, varsMgr VarsManager, caps capabilities.Capability, monitorMgr MonitorManager, modifiers ...ComponentsModifier) *Coordinator {
return &Coordinator{
logger: logger,
agentInfo: agentInfo,
Expand All @@ -181,6 +194,7 @@ func New(logger *logger.Logger, agentInfo *info.AgentInfo, specs component.Runti
state: coordinatorState{
state: agentclient.Starting,
},
monitorMgr: monitorMgr,
}
}

Expand Down Expand Up @@ -575,6 +589,10 @@ func (c *Coordinator) processConfig(ctx context.Context, cfg *config.Config) (er
return fmt.Errorf("failed to reload upgrade manager configuration: %w", err)
}

if err := c.monitorMgr.Reload(cfg); err != nil {
return fmt.Errorf("failed to reload upgrade manager configuration: %w", err)
}

c.state.config = cfg
c.state.ast = rawAst

Expand Down Expand Up @@ -640,7 +658,13 @@ func (c *Coordinator) compute() (map[string]interface{}, []component.Component,
if err != nil {
return nil, nil, fmt.Errorf("failed to convert ast to map[string]interface{}: %w", err)
}
comps, err := c.specs.ToComponents(cfg)

var configInjector component.GenerateMonitoringCfgFn
if c.monitorMgr.Enabled() {
configInjector = c.monitorMgr.MonitoringConfig
}

comps, err := c.specs.ToComponents(cfg, configInjector)
if err != nil {
return nil, nil, fmt.Errorf("failed to render components: %w", err)
}
Expand Down
70 changes: 70 additions & 0 deletions internal/pkg/agent/application/monitoring/handler.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package monitoring

import (
"encoding/json"
"fmt"
"net/http"
)

const errTypeUnexpected = "UNEXPECTED"

type apiError interface {
Status() int
}

func createHandler(fn func(w http.ResponseWriter, r *http.Request) error) *apiHandler {
return &apiHandler{
innerFn: fn,
}
}

type apiHandler struct {
innerFn func(w http.ResponseWriter, r *http.Request) error
}

// ServeHTTP sets status code based on err returned
func (h *apiHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
err := h.innerFn(w, r)
if err != nil {
switch e := err.(type) { // nolint:errorlint // Will need refactor.
case apiError:
w.WriteHeader(e.Status())
default:
w.WriteHeader(http.StatusInternalServerError)

}

writeResponse(w, unexpectedErrorWithReason(err.Error()))
}
}

func writeResponse(w http.ResponseWriter, c interface{}) {
bytes, err := json.Marshal(c)
if err != nil {
// json marshal failed
fmt.Fprintf(w, "Not valid json: %v", err)
return
}

fmt.Fprint(w, string(bytes))

}

type errResponse struct {
// Type is a type of error
Type string `json:"type"`

// Reason is a detailed error message
Reason string `json:"reason"`
}

func unexpectedErrorWithReason(reason string, args ...interface{}) errResponse {
return errResponse{
Type: errTypeUnexpected,
Reason: fmt.Sprintf(reason, args...),
}
}
86 changes: 86 additions & 0 deletions internal/pkg/agent/application/monitoring/server.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package monitoring

import (
"net/http"
"os"
"path/filepath"
"runtime"
"strings"

"github.com/gorilla/mux"
"go.elastic.co/apm"
"go.elastic.co/apm/module/apmgorilla"

"github.com/elastic/elastic-agent-libs/api"
"github.com/elastic/elastic-agent-libs/config"
"github.com/elastic/elastic-agent-libs/monitoring"
"github.com/elastic/elastic-agent/pkg/core/logger"
)

// New creates a new server exposing metrics and process information.
func NewServer(
log *logger.Logger,
endpointConfig api.Config,
ns func(string) *monitoring.Namespace,
tracer *apm.Tracer,
) (*api.Server, error) {
if err := createAgentMonitoringDrop(endpointConfig.Host); err != nil {
// log but ignore
log.Errorf("failed to create monitoring drop: %v", err)
}

cfg, err := config.NewConfigFrom(endpointConfig)
if err != nil {
return nil, err
}

return exposeMetricsEndpoint(log, cfg, ns, tracer)
}

func exposeMetricsEndpoint(
log *logger.Logger,
config *config.C,
ns func(string) *monitoring.Namespace,
tracer *apm.Tracer,
) (*api.Server, error) {
r := mux.NewRouter()
if tracer != nil {
r.Use(apmgorilla.Middleware(apmgorilla.WithTracer(tracer)))
}
statsHandler := statsHandler(ns("stats"))
r.Handle("/stats", createHandler(statsHandler))

mux := http.NewServeMux()
mux.Handle("/", r)

return api.New(log, mux, config)
}

func createAgentMonitoringDrop(drop string) error {
if drop == "" || runtime.GOOS == "windows" {
return nil
}

path := strings.TrimPrefix(drop, "unix://")
if strings.HasSuffix(path, ".sock") {
path = filepath.Dir(path)
}

_, err := os.Stat(path)
if err != nil {
if !os.IsNotExist(err) {
return err
}

// create
if err := os.MkdirAll(path, 0775); err != nil {
return err
}
}

return os.Chown(path, os.Geteuid(), os.Getegid())
}
36 changes: 36 additions & 0 deletions internal/pkg/agent/application/monitoring/stats.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
// or more contributor license agreements. Licensed under the Elastic License;
// you may not use this file except in compliance with the Elastic License.

package monitoring

import (
"encoding/json"
"fmt"
"net/http"

"github.com/elastic/elastic-agent-libs/monitoring"
)

func statsHandler(ns *monitoring.Namespace) func(http.ResponseWriter, *http.Request) error {
return func(w http.ResponseWriter, r *http.Request) error {
w.Header().Set("Content-Type", "application/json; charset=utf-8")

data := monitoring.CollectStructSnapshot(
ns.GetRegistry(),
monitoring.Full,
false,
)

bytes, err := json.Marshal(data)
var content string
if err != nil {
content = fmt.Sprintf("Not valid json: %v", err)
} else {
content = string(bytes)
}
fmt.Fprint(w, content)

return nil
}
}
Loading

0 comments on commit ec83c2c

Please sign in to comment.