Cherry-pick #17960 to 7.x: Use data subfolder as default for process logs

x-pack/elastic-agent/CHANGELOG.asciidoc

@@ -25,6 +25,7 @@
 - Fixed injected log path to monitoring beat {pull}17833[17833]
 - Make sure that the Elastic Agent connect over TLS in cloud. {pull}17843[17843]
 - Moved stream.* fields to top of event {pull}17858[17858]
+- Use /tmp for default monitoring endpoint location for libbeat {pull}18131[18131]
 - Use default output by default {pull}18091[18091]
 
 ==== New features
@@ -39,3 +40,4 @@
 - Monitoring configuration reloadable {pull}17855[17855]
 - Allow CLI overrides of paths {pull}17781[17781]
 - Enable Filebeat input: S3, Azureeventhub, cloudfoundry, httpjson, netflow, o365audit. {pull}17909[17909]
+- Use data subfolder as default for process logs {pull}17960[17960]

x-pack/elastic-agent/pkg/agent/application/global_config.go

@@ -5,61 +5,18 @@
 package application
 
 import (
-	"os"
-	"path/filepath"
-
-	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/config"
 )
 
-var (
-	homePath   string
-	dataPath   string
-	overwrites *common.Config
-)
-
-func init() {
-	homePath = retrieveExecutablePath()
-	dataPath = retrieveDataPath()
-	overwrites = common.NewConfig()
-	common.ConfigOverwriteFlag(nil, overwrites, "path.home", "path.home", "", "Agent root path")
-	common.ConfigOverwriteFlag(nil, overwrites, "path.data", "path.data", "", "Data path contains Agent managed binaries")
-}
-
-// HomePath returns home path where.
-func HomePath() string {
-	if val, err := overwrites.String("path.home", -1); err == nil {
-		return val
-	}
-
-	return homePath
-}
-
-// DataPath returns data path where.
-func DataPath() string {
-	if val, err := overwrites.String("path.data", -1); err == nil {
-		return val
-	}
-
-	return dataPath
-}
-
 // InjectAgentConfig injects config to a provided configuration.
 func InjectAgentConfig(c *config.Config) error {
 	globalConfig := agentGlobalConfig()
 	if err := c.Merge(globalConfig); err != nil {
 		return errors.New("failed to inject agent global config", err, errors.TypeConfig)
 	}
 
-	return injectOverwrites(c)
-}
-
-func injectOverwrites(c *config.Config) error {
-	if err := c.Merge(overwrites); err != nil {
-		return errors.New("failed to inject agent overwrites", err, errors.TypeConfig)
-	}
-
 	return nil
 }
 
@@ -68,24 +25,8 @@ func injectOverwrites(c *config.Config) error {
 func agentGlobalConfig() map[string]interface{} {
 	return map[string]interface{}{
 		"path": map[string]interface{}{
-			"data": dataPath,
-			"home": homePath,
+			"data": paths.Data(),
+			"home": paths.Home(),
 		},
 	}
 }
-
-// retrieveExecutablePath returns a directory where binary lives
-// Executable is not supported on nacl.
-func retrieveExecutablePath() string {
-	execPath, err := os.Executable()
-	if err != nil {
-		panic(err)
-	}
-
-	return filepath.Dir(execPath)
-}
-
-// retrieveHomePath returns a home directory of current user
-func retrieveDataPath() string {
-	return filepath.Join(retrieveExecutablePath(), "data")
-}

x-pack/elastic-agent/pkg/agent/application/paths/paths.go

@@ -0,0 +1,45 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package paths
+
+import (
+	"flag"
+	"os"
+	"path/filepath"
+)
+
+var (
+	homePath string
+	dataPath string
+)
+
+func init() {
+	exePath := retrieveExecutablePath()
+
+	fs := flag.CommandLine
+	fs.StringVar(&homePath, "path.home", exePath, "Agent root path")
+	fs.StringVar(&dataPath, "path.data", filepath.Join(exePath, "data"), "Data path contains Agent managed binaries")
+}
+
+// Home returns a directory where binary lives
+// Executable is not supported on nacl.
+func Home() string {
+	return homePath
+}
+
+// Data returns a home directory of current user
+func Data() string {
+	return dataPath
+}
+
+func retrieveExecutablePath() string {
+
+	execPath, err := os.Executable()
+	if err != nil {
+		panic(err)
+	}
+
+	return filepath.Dir(execPath)
+}

x-pack/elastic-agent/pkg/agent/cmd/common.go

@@ -12,7 +12,7 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/basecmd"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
 )
@@ -27,8 +27,8 @@ type globalFlags struct {
 
 // Config returns path which identifies configuration file.
 func (f *globalFlags) Config() string {
-	if len(f.PathConfigFile) == 0 {
-		return filepath.Join(application.HomePath(), defaultConfig)
+	if len(f.PathConfigFile) == 0 || f.PathConfigFile == defaultConfig {
+		return filepath.Join(paths.Home(), defaultConfig)
 	}
 	return f.PathConfigFile
 }

x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/drop_test.go

@@ -15,22 +15,22 @@ type testCase struct {
 
 func TestMonitoringDrops(t *testing.T) {
 	cases := []testCase{
-		testCase{`/var/lib/drop/abc.sock`, "/var/lib/drop"},
-		testCase{`npipe://drop`, ""},
-		testCase{`http+npipe://drop`, ""},
-		testCase{`\\.\pipe\drop`, ""},
-		testCase{`unix:///var/lib/drop/abc.sock`, "/var/lib/drop"},
-		testCase{`http+unix:///var/lib/drop/abc.sock`, "/var/lib/drop"},
-		testCase{`file:///var/lib/drop/abc.sock`, "/var/lib/drop"},
-		testCase{`http://localhost/stats`, ""},
-		testCase{`localhost/stats`, ""},
-		testCase{`http://localhost:8080/stats`, ""},
-		testCase{`localhost:8080/stats`, ""},
-		testCase{`http://1.2.3.4/stats`, ""},
-		testCase{`http://1.2.3.4:5678/stats`, ""},
-		testCase{`1.2.3.4:5678/stats`, ""},
-		testCase{`http://hithere.com:5678/stats`, ""},
-		testCase{`hithere.com:5678/stats`, ""},
+		{`/var/lib/drop/abc.sock`, "/var/lib/drop"},
+		{`npipe://drop`, ""},
+		{`http+npipe://drop`, ""},
+		{`\\.\pipe\drop`, ""},
+		{`unix:///var/lib/drop/abc.sock`, "/var/lib/drop"},
+		{`http+unix:///var/lib/drop/abc.sock`, "/var/lib/drop"},
+		{`file:///var/lib/drop/abc.sock`, "/var/lib/drop"},
+		{`http://localhost/stats`, ""},
+		{`localhost/stats`, ""},
+		{`http://localhost:8080/stats`, ""},
+		{`localhost:8080/stats`, ""},
+		{`http://1.2.3.4/stats`, ""},
+		{`http://1.2.3.4:5678/stats`, ""},
+		{`1.2.3.4:5678/stats`, ""},
+		{`http://hithere.com:5678/stats`, ""},
+		{`hithere.com:5678/stats`, ""},
 	}
 
 	for _, c := range cases {

x-pack/elastic-agent/pkg/core/plugin/app/monitoring/beats/monitoring.go

@@ -6,16 +6,18 @@ package beats
 
 import (
 	"fmt"
+
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 )
 
 const (
-	// args: pipeline name, application name
-	logFileFormat = "/var/log/elastic-agent/%s/%s"
-	// args: install path, pipeline name, application name
-	logFileFormatWin = "%s\\logs\\elastic-agent\\%s\\%s"
+	// args: data path, pipeline name, application name
+	logFileFormat = "%s/logs/%s/%s"
+	// args: data path, install path, pipeline name, application name
+	logFileFormatWin = "%s\\logs\\%s\\%s"
 
 	// args: pipeline name, application name
-	mbEndpointFileFormat = "unix:///var/run/elastic-agent/%s/%s/%s.sock"
+	mbEndpointFileFormat = "unix:///tmp/elastic-agent/%s/%s/%s.sock"
 	// args: pipeline name, application name
 	mbEndpointFileFormatWin = `npipe:///%s-%s`
 )
@@ -30,8 +32,8 @@ func getMonitoringEndpoint(program, operatingSystem, pipelineID string) string {
 
 func getLoggingFile(program, operatingSystem, installPath, pipelineID string) string {
 	if operatingSystem == "windows" {
-		return fmt.Sprintf(logFileFormatWin, installPath, pipelineID, program)
+		return fmt.Sprintf(logFileFormatWin, paths.Data(), pipelineID, program)
 	}
 
-	return fmt.Sprintf(logFileFormat, pipelineID, program)
+	return fmt.Sprintf(logFileFormat, paths.Data(), pipelineID, program)
 }

x-pack/elastic-agent/pkg/core/plugin/app/start.go

@@ -15,6 +15,7 @@ import (
 
 	"gopkg.in/yaml.v2"
 
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/plugin/authority"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/plugin/process"
@@ -210,12 +211,7 @@ func (a *Application) checkGrpcHTTP(ctx context.Context, address string, ca *aut
 }
 
 func injectDataPath(args []string, pipelineID, id string) []string {
-	wd := ""
-	if w, err := os.Getwd(); err == nil {
-		wd = w
-	}
-
-	dataPath := filepath.Join(wd, "data", pipelineID, id)
+	dataPath := filepath.Join(paths.Data(), pipelineID, id)
 	return append(args, "-E", "path.data="+dataPath)
 }