V2 Runtime Component Manager (#645)

* Add runtime for command v2 components.

* Fix imports.

* Add tests for watching checkins.

* Fix lint and move checkin period to a configurable timeout.

* Fix tests now that checkin timeout needs to be defined.

* Fix code review and lint.

.gitignore

@@ -57,8 +57,11 @@ internal/pkg/agent/operation/tests/scripts/configurable-1.0-darwin-x86/configura
 internal/pkg/agent/operation/tests/scripts/servicable-1.0-darwin-x86/configurable
 internal/pkg/agent/operation/tests/scripts/configurable-1.0-darwin-x86_64/configurable
 internal/pkg/agent/operation/tests/scripts/serviceable-1.0-darwin-x86_64/serviceable
+internal/pkg/agent/operation/tests/scripts/configurable
+internal/pkg/agent/operation/tests/scripts/serviceable
 internal/pkg/agent/application/fleet.yml
 internal/pkg/agent/transpiler/tests/exec-1.0-darwin-x86_64/exec
+pkg/component/fake/fake
 
 # VSCode
 /.vscode

internal/pkg/agent/cmd/container.go

@@ -36,9 +36,9 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/artifact/install/tar"
 	"github.com/elastic/elastic-agent/internal/pkg/cli"
 	"github.com/elastic/elastic-agent/internal/pkg/config"
-	"github.com/elastic/elastic-agent/internal/pkg/core/process"
 	"github.com/elastic/elastic-agent/internal/pkg/release"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 	"github.com/elastic/elastic-agent/version"
 )
 
@@ -727,10 +727,6 @@ func runLegacyAPMServer(streams *cli.IOStreams, path string) (*process.Info, err
 	apmDir := filepath.Join(path, files[0].Name())
 	// Start apm-server process respecting path ENVs
 	apmBinary := filepath.Join(apmDir, spec.Cmd)
-	log, err := logger.New("apm-server", false)
-	if err != nil {
-		return nil, err
-	}
 	// add APM Server specific configuration
 	var args []string
 	addEnv := func(arg, env string) {
@@ -751,7 +747,7 @@ func runLegacyAPMServer(streams *cli.IOStreams, path string) (*process.Info, err
 	addEnv("--httpprof", "HTTPPROF")
 	addSettingEnv("gc_percent", "APMSERVER_GOGC")
 	logInfo(streams, "Starting legacy apm-server daemon as a subprocess.")
-	return process.Start(log, apmBinary, nil, os.Geteuid(), os.Getegid(), args)
+	return process.Start(apmBinary, os.Geteuid(), os.Getegid(), args, nil)
 }
 
 func logToStderr(cfg *configuration.Configuration) {

internal/pkg/agent/cmd/enroll_cmd.go

@@ -37,12 +37,12 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/core/authority"
 	"github.com/elastic/elastic-agent/internal/pkg/core/backoff"
 	monitoringConfig "github.com/elastic/elastic-agent/internal/pkg/core/monitoring/config"
-	"github.com/elastic/elastic-agent/internal/pkg/core/process"
 	"github.com/elastic/elastic-agent/internal/pkg/fleetapi"
 	fleetclient "github.com/elastic/elastic-agent/internal/pkg/fleetapi/client"
 	"github.com/elastic/elastic-agent/internal/pkg/release"
 	"github.com/elastic/elastic-agent/internal/pkg/remote"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 )
 
 const (
@@ -600,9 +600,10 @@ func (c *enrollCmd) startAgent(ctx context.Context) (<-chan *os.ProcessState, er
 		args = append(args, "--path.home.unversioned")
 	}
 	proc, err := process.StartContext(
-		ctx, c.log, cmd, nil, os.Geteuid(), os.Getegid(), args, func(c *exec.Cmd) {
+		ctx, cmd, os.Geteuid(), os.Getegid(), args, nil, func(c *exec.Cmd) error {
 			c.Stdout = os.Stdout
 			c.Stderr = os.Stderr
+			return nil
 		})
 	if err != nil {
 		return nil, err

internal/pkg/agent/configuration/settings.go

@@ -9,9 +9,9 @@ import (
 
 	"github.com/elastic/elastic-agent/internal/pkg/artifact"
 	monitoringCfg "github.com/elastic/elastic-agent/internal/pkg/core/monitoring/config"
-	"github.com/elastic/elastic-agent/internal/pkg/core/process"
 	"github.com/elastic/elastic-agent/internal/pkg/core/retry"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 	"github.com/elastic/elastic-agent/pkg/core/server"
 )
 

internal/pkg/agent/operation/common_test.go

@@ -25,10 +25,10 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/artifact/uninstall"
 	"github.com/elastic/elastic-agent/internal/pkg/core/app"
 	"github.com/elastic/elastic-agent/internal/pkg/core/monitoring/noop"
-	"github.com/elastic/elastic-agent/internal/pkg/core/process"
 	"github.com/elastic/elastic-agent/internal/pkg/core/retry"
 	"github.com/elastic/elastic-agent/internal/pkg/core/status"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 	"github.com/elastic/elastic-agent/pkg/core/server"
 )
 

internal/pkg/agent/operation/monitoring_test.go

@@ -27,10 +27,10 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/core/app"
 	"github.com/elastic/elastic-agent/internal/pkg/core/monitoring"
 	monitoringConfig "github.com/elastic/elastic-agent/internal/pkg/core/monitoring/config"
-	"github.com/elastic/elastic-agent/internal/pkg/core/process"
 	"github.com/elastic/elastic-agent/internal/pkg/core/retry"
 	"github.com/elastic/elastic-agent/internal/pkg/core/state"
 	"github.com/elastic/elastic-agent/internal/pkg/core/status"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 	"github.com/elastic/elastic-agent/pkg/core/server"
 )
 

internal/pkg/agent/storage/encrypted_disk_storage_windows_linux_test.go

@@ -16,8 +16,9 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/elastic/elastic-agent/internal/pkg/agent/application/secret"
 	"github.com/google/go-cmp/cmp"
+
+	"github.com/elastic/elastic-agent/internal/pkg/agent/application/secret"
 )
 
 const (

internal/pkg/artifact/install/atomic/atomic_installer.go

@@ -11,8 +11,9 @@ import (
 	"path/filepath"
 	"runtime"
 
-	"github.com/elastic/elastic-agent/internal/pkg/agent/program"
 	"github.com/hashicorp/go-multierror"
+
+	"github.com/elastic/elastic-agent/internal/pkg/agent/program"
 )
 
 type embeddedInstaller interface {

internal/pkg/core/monitoring/server/processes_test.go

@@ -15,9 +15,9 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 
-	"github.com/elastic/elastic-agent/internal/pkg/core/process"
 	"github.com/elastic/elastic-agent/internal/pkg/core/state"
 	"github.com/elastic/elastic-agent/internal/pkg/sorted"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 )
 
 func TestProcesses(t *testing.T) {

internal/pkg/core/plugin/process/app.go

@@ -19,11 +19,11 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/agent/program"
 	"github.com/elastic/elastic-agent/internal/pkg/core/app"
 	"github.com/elastic/elastic-agent/internal/pkg/core/monitoring"
-	"github.com/elastic/elastic-agent/internal/pkg/core/process"
 	"github.com/elastic/elastic-agent/internal/pkg/core/state"
 	"github.com/elastic/elastic-agent/internal/pkg/core/status"
 	"github.com/elastic/elastic-agent/internal/pkg/tokenbucket"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 	"github.com/elastic/elastic-agent/pkg/core/server"
 )
 

internal/pkg/core/plugin/process/start.go

@@ -18,8 +18,8 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/agent/application/paths"
 	"github.com/elastic/elastic-agent/internal/pkg/agent/errors"
 	"github.com/elastic/elastic-agent/internal/pkg/core/app"
-	"github.com/elastic/elastic-agent/internal/pkg/core/process"
 	"github.com/elastic/elastic-agent/internal/pkg/core/state"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 	"github.com/elastic/elastic-agent/pkg/core/server"
 )
 
@@ -128,14 +128,13 @@ func (a *Application) start(ctx context.Context, t app.Taggable, cfg map[string]
 	spec.Args = injectDataPath(spec.Args, a.pipelineID, a.id)
 
 	a.state.ProcessInfo, err = process.Start(
-		a.logger,
 		spec.BinaryPath,
-		a.processConfig,
 		a.uid,
 		a.gid,
-		spec.Args, func(c *exec.Cmd) {
+		spec.Args, nil, func(c *exec.Cmd) error {
 			c.Stdout = newLoggerWriter(a.Name(), logStdOut, a.logger)
 			c.Stderr = newLoggerWriter(a.Name(), logStdErr, a.logger)
+			return nil
 		})
 	if err != nil {
 		return fmt.Errorf("%q failed to start %q: %w",

internal/pkg/core/plugin/process/status.go

@@ -13,8 +13,8 @@ import (
 
 	"github.com/elastic/elastic-agent-client/v7/pkg/proto"
 
-	"github.com/elastic/elastic-agent/internal/pkg/core/process"
 	"github.com/elastic/elastic-agent/internal/pkg/core/state"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 	"github.com/elastic/elastic-agent/pkg/core/server"
 )
 

internal/pkg/core/plugin/service/app.go

@@ -23,11 +23,11 @@ import (
 	"github.com/elastic/elastic-agent/internal/pkg/core/app"
 	"github.com/elastic/elastic-agent/internal/pkg/core/monitoring"
 	"github.com/elastic/elastic-agent/internal/pkg/core/plugin"
-	"github.com/elastic/elastic-agent/internal/pkg/core/process"
 	"github.com/elastic/elastic-agent/internal/pkg/core/state"
 	"github.com/elastic/elastic-agent/internal/pkg/core/status"
 	"github.com/elastic/elastic-agent/internal/pkg/tokenbucket"
 	"github.com/elastic/elastic-agent/pkg/core/logger"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 	"github.com/elastic/elastic-agent/pkg/core/server"
 )
 

internal/pkg/core/state/state.go

@@ -10,7 +10,7 @@ import (
 
 	"github.com/elastic/elastic-agent-client/v7/pkg/proto"
 
-	"github.com/elastic/elastic-agent/internal/pkg/core/process"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 )
 
 // Status describes the current status of the application process.

magefile.go

@@ -272,18 +272,22 @@ func (Build) Clean() {
 func (Build) TestBinaries() error {
 	p := filepath.Join("internal", "pkg", "agent", "operation", "tests", "scripts")
 	p2 := filepath.Join("internal", "pkg", "agent", "transpiler", "tests")
+	p3 := filepath.Join("pkg", "component")
 	configurableName := "configurable"
 	serviceableName := "serviceable"
 	execName := "exec"
+	fakeName := "fake"
 	if runtime.GOOS == "windows" {
 		configurableName += ".exe"
 		serviceableName += ".exe"
 		execName += ".exe"
+		fakeName += ".exe"
 	}
 	return combineErr(
 		RunGo("build", "-o", filepath.Join(p, configurableName), filepath.Join(p, "configurable-1.0-darwin-x86_64", "main.go")),
 		RunGo("build", "-o", filepath.Join(p, serviceableName), filepath.Join(p, "serviceable-1.0-darwin-x86_64", "main.go")),
 		RunGo("build", "-o", filepath.Join(p2, "exec-1.0-darwin-x86_64", execName), filepath.Join(p2, "exec-1.0-darwin-x86_64", "main.go")),
+		RunGo("build", "-o", filepath.Join(p3, "fake", fakeName), filepath.Join(p3, "fake", "main.go")),
 	)
 }
 

main.go

@@ -11,7 +11,7 @@ import (
 	"time"
 
 	"github.com/elastic/elastic-agent/internal/pkg/agent/cmd"
-	"github.com/elastic/elastic-agent/internal/pkg/agent/cmd/proc"
+	"github.com/elastic/elastic-agent/pkg/core/process"
 )
 
 // Setups and Runs agent.
@@ -21,7 +21,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	pj, err := proc.CreateJobObject()
+	pj, err := process.CreateJobObject()
 	if err != nil {
 		fmt.Fprintf(os.Stderr, "Failed to initialize process job object: %v\n", err)
 		os.Exit(1)

pkg/component/fake/README.md

@@ -0,0 +1,3 @@
+# Fake Component
+
+Controllable through GRPC control protocol with actions. Allows unit tests to simulate control and communication with a running sub-process.