Cherry-pick elastic#23736 to 7.x: [Elastic Agent] Add the ability to …

…run the Fleet Server (elastic#23785) * [Elastic Agent] Add the ability to run the Fleet Server (elastic#23736) * Add the ability to run the Fleet Server. * Add test and changelog. * Fix changelog. (cherry picked from commit d59f780) * Fixed monitoring filebeat and metricbeat not connecting to Agent over GRPC (elastic#23843) Fixed monitoring filebeat and metricbeat not connecting to Agent over GRPC (elastic#23843) Co-authored-by: Michal Pristas <[email protected]>
fearful-symmetry · Feb 10, 2021 · 2fab9a8 · 2fab9a8
1 parent 9273c58
commit 2fab9a8
Show file tree

Hide file tree

Showing 12 changed files with 171 additions and 74 deletions.
diff --git a/dev-tools/packaging/packages.yml b/dev-tools/packaging/packages.yml
@@ -106,6 +106,18 @@ shared:
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+        skip_on_missing: true
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
+        mode: 0644
+        skip_on_missing: true
+      /var/lib/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
+        mode: 0644
+        skip_on_missing: true
 
 
 
@@ -198,6 +210,18 @@ shared:
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+        skip_on_missing: true
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
+        mode: 0644
+        skip_on_missing: true
+      /etc/{{.BeatName}}/data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc:
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
+        mode: 0644
+        skip_on_missing: true
 
   - &agent_binary_files
     '{{.BeatName}}{{.BinaryExt}}':
@@ -278,6 +302,18 @@ shared:
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
         mode: 0644
         skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz'
+        mode: 0644
+        skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.sha512'
+        mode: 0644
+        skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.tar.gz.asc'
+        mode: 0644
+        skip_on_missing: true
 
   # Binary package spec (zip for windows) for community beats.
   - &agent_windows_binary_spec
@@ -326,6 +362,18 @@ shared:
         source: '{{.AgentDropPath}}/endpoint-security-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
         mode: 0644
         skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip'
+        mode: 0644
+        skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.sha512'
+        mode: 0644
+        skip_on_missing: true
+      'data/{{.BeatName}}-{{ commit_short }}/downloads/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc':
+        source: '{{.AgentDropPath}}/fleet-server-{{ beat_version }}{{if .Snapshot}}-SNAPSHOT{{end}}-{{.GOOS}}-{{.AgentArchName}}.zip.asc'
+        mode: 0644
+        skip_on_missing: true
 
   - &agent_docker_spec
     <<: *agent_binary_spec

diff --git a/x-pack/elastic-agent/CHANGELOG.asciidoc b/x-pack/elastic-agent/CHANGELOG.asciidoc
@@ -32,6 +32,7 @@
 - Fixed nil pointer during unenroll {pull}23609[23609]
 - Fixed reenroll scenario {pull}23686[23686]
 - Fixed make status readable in the log. {pull}23849[23849]
+- Fixed Monitoring filebeat and metricbeat not connecting to Agent over GRPC {pull}23843[23843]
 
 ==== New features
 

diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc
@@ -65,3 +65,4 @@
 - Log level reloadable from fleet {pull}22690[22690]
 - Push log level downstream {pull}22815[22815]
 - Add metrics collection for Agent {pull}22793[22793]
+- Add support for Fleet Server {pull}23736[23736]
diff --git a/x-pack/elastic-agent/pkg/agent/application/fleet_decorator.go b/x-pack/elastic-agent/pkg/agent/application/fleet_decorator.go
@@ -54,7 +54,13 @@ func injectFleet(cfg *config.Config, hostInfo types.HostInfo, agentInfo *info.Ag
 			transpiler.NewKey("id", transpiler.NewStrVal(hostInfo.UniqueID)),
 		}))
 
-		fleet := transpiler.NewDict([]transpiler.Node{agent, token, kbn, host})
+		nodes := []transpiler.Node{agent, token, kbn, host}
+		server, ok := transpiler.Lookup(ast, "fleet.server")
+		if ok {
+			nodes = append(nodes, server)
+		}
+		fleet := transpiler.NewDict(nodes)
+
 		err = transpiler.Insert(rootAst, fleet, "fleet")
 		if err != nil {
 			return err

diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go
@@ -138,13 +138,9 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [
 			stepID = configrequest.StepRemove
 		}
 		filebeatStep := configrequest.Step{
-			ID:      stepID,
-			Version: version,
-			ProgramSpec: program.Spec{
-				Name:     logsProcessName,
-				Cmd:      logsProcessName,
-				Artifact: fmt.Sprintf("%s/%s", artifactPrefix, logsProcessName),
-			},
+			ID:          stepID,
+			Version:     version,
+			ProgramSpec: loadSpecFromSupported(logsProcessName),
 			Meta: map[string]interface{}{
 				configrequest.MetaConfigKey: fbConfig,
 			},
@@ -160,13 +156,9 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [
 		}
 
 		metricbeatStep := configrequest.Step{
-			ID:      stepID,
-			Version: version,
-			ProgramSpec: program.Spec{
-				Name:     metricsProcessName,
-				Cmd:      metricsProcessName,
-				Artifact: fmt.Sprintf("%s/%s", artifactPrefix, metricsProcessName),
-			},
+			ID:          stepID,
+			Version:     version,
+			ProgramSpec: loadSpecFromSupported(metricsProcessName),
 			Meta: map[string]interface{}{
 				configrequest.MetaConfigKey: mbConfig,
 			},
@@ -178,6 +170,18 @@ func (o *Operator) generateMonitoringSteps(version string, output interface{}) [
 	return steps
 }
 
+func loadSpecFromSupported(processName string) program.Spec {
+	if loadedSpec, found := program.SupportedMap[strings.ToLower(processName)]; found {
+		return loadedSpec
+	}
+
+	return program.Spec{
+		Name:     processName,
+		Cmd:      processName,
+		Artifact: fmt.Sprintf("%s/%s", artifactPrefix, processName),
+	}
+}
+
 func (o *Operator) getMonitoringFilebeatConfig(output interface{}) (map[string]interface{}, bool) {
 	inputs := []interface{}{
 		map[string]interface{}{

diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go
@@ -384,8 +384,8 @@ func TestConfiguration(t *testing.T) {
 		err      bool
 	}{
 		"single_config": {
-			programs: []string{"filebeat", "heartbeat", "metricbeat", "endpoint", "packetbeat"},
-			expected: 5,
+			programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"},
+			expected: 6,
 		},
 		// "audit_config": {
 		// 	programs: []string{"auditbeat"},

diff --git a/x-pack/elastic-agent/pkg/agent/program/supported.go b/x-pack/elastic-agent/pkg/agent/program/supported.go
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-fleet-server.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config-fleet-server.yml
@@ -0,0 +1,13 @@
+fleet:
+  agent:
+    id: fleet-agent-id
+
+output:
+  elasticsearch:
+    hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ]
+    username: fleet
+    password: fleetpassword
+
+inputs:
+  - id: fleet-server-id
+    type: fleet-server
diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/single_config.yml
@@ -8,6 +8,12 @@ fleet:
     protocol: https
     host: localhost:5601
     timeout: 30s
+  server:
+    output:
+      elasticsearch:
+        hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ]
+        username: fleet
+        password: fleetpassword
 
 outputs:
   default:
@@ -25,6 +31,11 @@ outputs:
     ca_sha256: "7lHLiyp4J8m9kw38SJ7SURJP4bXRZv/BNxyyXkCcE/M="
 
 inputs:
+- id: fleet-server-id
+  type: fleet-server
+  use_output: default
+  data_stream:
+    type: default
 - type: docker/metrics
   use_output: default
   streams:

diff --git a/x-pack/elastic-agent/pkg/core/app/descriptor.go b/x-pack/elastic-agent/pkg/core/app/descriptor.go
@@ -5,7 +5,6 @@
 package app
 
 import (
-	"path"
 	"path/filepath"
 	"strings"
 
@@ -29,7 +28,7 @@ func NewDescriptor(spec program.Spec, version string, config *artifact.Config, t
 		spec:         spec,
 		directory:    dir,
 		executionCtx: NewExecutionContext(spec.ServicePort, spec.Cmd, version, tags),
-		process:      specification(dir, spec.Cmd),
+		process:      specification(dir, spec),
 	}
 }
 
@@ -78,23 +77,12 @@ func (p *Descriptor) Directory() string {
 	return p.directory
 }
 
-func defaultSpec(dir string, binaryName string) ProcessSpec {
-	if !isKnownBeat(binaryName) {
-		return ProcessSpec{
-			BinaryPath: path.Join(dir, binaryName),
-		}
-	}
-
+func specification(dir string, spec program.Spec) ProcessSpec {
 	return ProcessSpec{
-		BinaryPath: path.Join(dir, binaryName),
-		Args:       []string{},
+		BinaryPath:    filepath.Join(dir, spec.Cmd),
+		Args:          spec.Args,
+		Configuration: nil,
 	}
-
-}
-
-func specification(directory, binaryName string) ProcessSpec {
-	defaultSpec := defaultSpec(directory, binaryName)
-	return populateSpec(directory, binaryName, defaultSpec)
 }
 
 func directory(spec program.Spec, version string, config *artifact.Config) string {
@@ -114,42 +102,3 @@ func directory(spec program.Spec, version string, config *artifact.Config) strin
 
 	return strings.TrimSuffix(path, suffix)
 }
-
-func isKnownBeat(name string) bool {
-	switch name {
-	case "filebeat":
-		fallthrough
-	case "metricbeat":
-		return true
-	}
-
-	return false
-}
-
-func populateSpec(dir, binaryName string, spec ProcessSpec) ProcessSpec {
-	var programSpec program.Spec
-	var found bool
-	binaryName = strings.ToLower(binaryName)
-	for _, prog := range program.Supported {
-		if binaryName != strings.ToLower(prog.Name) {
-			continue
-		}
-		found = true
-		programSpec = prog
-		break
-	}
-
-	if !found {
-		return spec
-	}
-
-	if programSpec.Cmd != "" {
-		spec.BinaryPath = filepath.Join(dir, programSpec.Cmd)
-	}
-
-	if len(programSpec.Args) > 0 {
-		spec.Args = programSpec.Args
-	}
-
-	return spec
-}