Cherry-pick #24387 to 7.x: Fix failing installation on windows 7 (#24409

)

libbeat/common/backoff/exponential.go

@@ -21,8 +21,8 @@ import (
 	"time"
 )
 
-// ExpBackoff exponential backoff, will wait an initial time and exponentialy
-// increases the wait time up to a predefined maximun. Resetting Backoff will reset the next sleep
+// ExpBackoff exponential backoff, will wait an initial time and exponentially
+// increases the wait time up to a predefined maximum. Resetting Backoff will reset the next sleep
 // timer to the initial backoff duration.
 type ExpBackoff struct {
 	duration time.Duration

libbeat/common/file/helper_windows.go

@@ -19,6 +19,7 @@ package file
 
 import (
 	"os"
+	"path/filepath"
 )
 
 // SafeFileRotate safely rotates an existing file under path and replaces it with the tempfile
@@ -40,5 +41,13 @@ func SafeFileRotate(path, tempfile string) error {
 	if e = os.Rename(tempfile, path); e != nil {
 		return e
 	}
+
+	// sync all files
+	parent := filepath.Dir(path)
+	if f, err := os.OpenFile(parent, os.O_SYNC|os.O_RDWR, 0755); err == nil {
+		f.Sync()
+		f.Close()
+	}
+
 	return nil
 }

x-pack/elastic-agent/CHANGELOG.asciidoc

@@ -37,6 +37,7 @@
 - Fix reloading of log level for services {pull}[24055]24055
 - Fix: Successfully installed and enrolled agent running standalone{pull}[24128]24128
 - Make installer atomic on windows {pull}[24253]24253
+- Fix failing installation on windows 7 {pull}[24387]24387
 - Fix capabilities resolution in inspect command {pull}[24346]24346
 
 ==== New features

x-pack/elastic-agent/pkg/agent/application/enroll_cmd.go

@@ -15,29 +15,29 @@ import (
 	"os"
 	"time"
 
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/process"
-
 	"gopkg.in/yaml.v2"
 
 	"github.com/elastic/beats/v7/libbeat/common/backoff"
-
 	"github.com/elastic/beats/v7/libbeat/common/transport/tlscommon"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filelock"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/client"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/control/proto"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/authority"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/process"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/fleetapi"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/kibana"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/release"
 )
 
 const (
-	waitingForAgent        = "waiting for Elastic Agent to start"
-	waitingForFleetServer  = "waiting for Elastic Agent to start Fleet Server"
-	defaultFleetServerPort = 8220
+	maxRetriesstoreAgentInfo = 5
+	waitingForAgent          = "waiting for Elastic Agent to start"
+	waitingForFleetServer    = "waiting for Elastic Agent to start Fleet Server"
+	defaultFleetServerPort   = 8220
 )
 
 var (
@@ -233,8 +233,9 @@ func (c *EnrollCmd) fleetServerBootstrap(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	if err := c.configStore.Save(reader); err != nil {
-		return errors.New(err, "could not save fleet server bootstrap information", errors.TypeFilesystem)
+
+	if err := safelyStoreAgentInfo(c.configStore, reader); err != nil {
+		return err
 	}
 
 	if agentRunning {
@@ -404,11 +405,7 @@ func (c *EnrollCmd) enroll(ctx context.Context) error {
 		return err
 	}
 
-	if err := c.configStore.Save(reader); err != nil {
-		return errors.New(err, "could not save enrollment information", errors.TypeFilesystem)
-	}
-
-	if _, err := info.NewAgentInfo(); err != nil {
+	if err := safelyStoreAgentInfo(c.configStore, reader); err != nil {
 		return err
 	}
 
@@ -556,3 +553,34 @@ func getAppFromStatus(status *client.AgentStatus, name string) *client.Applicati
 	}
 	return nil
 }
+
+func safelyStoreAgentInfo(s store, reader io.Reader) error {
+	var err error
+	signal := make(chan struct{})
+	backExp := backoff.NewExpBackoff(signal, 100*time.Millisecond, 3*time.Second)
+
+	for i := 0; i <= maxRetriesstoreAgentInfo; i++ {
+		backExp.Wait()
+		err = storeAgentInfo(s, reader)
+		if err != filelock.ErrAppAlreadyRunning {
+			break
+		}
+	}
+
+	close(signal)
+	return err
+}
+
+func storeAgentInfo(s store, reader io.Reader) error {
+	fileLock := info.AgentConfigFileLock()
+	if err := fileLock.TryLock(); err != nil {
+		return err
+	}
+	defer fileLock.Unlock()
+
+	if err := s.Save(reader); err != nil {
+		return errors.New(err, "could not save enrollment information", errors.TypeFilesystem)
+	}
+
+	return nil
+}

x-pack/elastic-agent/pkg/agent/application/locker.go → x-pack/elastic-agent/pkg/agent/application/filelock/locker.go

@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package application
+package filelock
 
 import (
 	"fmt"

x-pack/elastic-agent/pkg/agent/application/locker_test.go → x-pack/elastic-agent/pkg/agent/application/filelock/locker_test.go

@@ -2,7 +2,7 @@
 // or more contributor license agreements. Licensed under the Elastic License;
 // you may not use this file except in compliance with the Elastic License.
 
-package application
+package filelock
 
 import (
 	"io/ioutil"

x-pack/elastic-agent/pkg/agent/application/info/agent_id.go

@@ -9,10 +9,13 @@ import (
 	"fmt"
 	"io"
 	"path/filepath"
+	"time"
 
 	"github.com/gofrs/uuid"
 	"gopkg.in/yaml.v2"
 
+	"github.com/elastic/beats/v7/libbeat/common/backoff"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filelock"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/storage"
@@ -29,6 +32,7 @@ const defaultAgentActionStoreFile = "action_store.yml"
 const defaultAgentStateStoreFile = "state.yml"
 
 const defaultLogLevel = "info"
+const maxRetriesloadAgentInfo = 5
 
 type persistentAgentInfo struct {
 	ID       string `json:"id" yaml:"id" config:"id"`
@@ -45,6 +49,14 @@ func AgentConfigFile() string {
 	return filepath.Join(paths.Config(), defaultAgentConfigFile)
 }
 
+// AgentConfigFileLock is a locker for agent config file updates.
+func AgentConfigFileLock() *filelock.AppLocker {
+	return filelock.NewAppLocker(
+		paths.Config(),
+		fmt.Sprintf("%s.lock", defaultAgentConfigFile),
+	)
+}
+
 // AgentCapabilitiesPath is a name of file used to store agent capabilities
 func AgentCapabilitiesPath() string {
 	return filepath.Join(paths.Config(), defaultAgentCapabilitiesFile)
@@ -62,7 +74,7 @@ func AgentStateStoreFile() string {
 
 // updateLogLevel updates log level and persists it to disk.
 func updateLogLevel(level string) error {
-	ai, err := loadAgentInfo(false, defaultLogLevel)
+	ai, err := loadAgentInfoWithBackoff(false, defaultLogLevel)
 	if err != nil {
 		return err
 	}
@@ -88,31 +100,6 @@ func generateAgentID() (string, error) {
 	return uid.String(), nil
 }
 
-func loadAgentInfo(forceUpdate bool, logLevel string) (*persistentAgentInfo, error) {
-	agentConfigFile := AgentConfigFile()
-	s := storage.NewDiskStore(agentConfigFile)
-
-	agentinfo, err := getInfoFromStore(s, logLevel)
-	if err != nil {
-		return nil, err
-	}
-
-	if agentinfo != nil && !forceUpdate && agentinfo.ID != "" {
-		return agentinfo, nil
-	}
-
-	agentinfo.ID, err = generateAgentID()
-	if err != nil {
-		return nil, err
-	}
-
-	if err := updateAgentInfo(s, agentinfo); err != nil {
-		return nil, errors.New(err, "storing generated agent id", errors.TypeFilesystem)
-	}
-
-	return agentinfo, nil
-}
-
 func getInfoFromStore(s ioStore, logLevel string) (*persistentAgentInfo, error) {
 	agentConfigFile := AgentConfigFile()
 	reader, err := s.Load()
@@ -178,6 +165,19 @@ func updateAgentInfo(s ioStore, agentInfo *persistentAgentInfo) error {
 		return errors.New(err, "failed to unpack stored config to map")
 	}
 
+	// best effort to keep the ID
+	if agentInfoSubMap, found := configMap[agentInfoKey]; found {
+		if cc, err := config.NewConfigFrom(agentInfoSubMap); err == nil {
+			pid := &persistentAgentInfo{}
+			err := cc.Unpack(&pid)
+			if err == nil && pid.ID != agentInfo.ID {
+				// if our id is different (we just generated it)
+				// keep the one present in the file
+				agentInfo.ID = pid.ID
+			}
+		}
+	}
+
 	configMap[agentInfoKey] = agentInfo
 
 	r, err := yamlToReader(configMap)
@@ -195,3 +195,62 @@ func yamlToReader(in interface{}) (io.Reader, error) {
 	}
 	return bytes.NewReader(data), nil
 }
+
+func loadAgentInfoWithBackoff(forceUpdate bool, logLevel string) (*persistentAgentInfo, error) {
+	var err error
+	var ai *persistentAgentInfo
+
+	signal := make(chan struct{})
+	backExp := backoff.NewExpBackoff(signal, 100*time.Millisecond, 3*time.Second)
+
+	for i := 0; i <= maxRetriesloadAgentInfo; i++ {
+		backExp.Wait()
+		ai, err = loadAgentInfo(forceUpdate, logLevel)
+		if err != filelock.ErrAppAlreadyRunning {
+			break
+		}
+	}
+
+	close(signal)
+	return ai, err
+}
+
+func loadAgentInfo(forceUpdate bool, logLevel string) (*persistentAgentInfo, error) {
+	idLock := AgentConfigFileLock()
+	if err := idLock.TryLock(); err != nil {
+		return nil, err
+	}
+	defer idLock.Unlock()
+
+	agentConfigFile := AgentConfigFile()
+	s := storage.NewDiskStore(agentConfigFile)
+
+	agentinfo, err := getInfoFromStore(s, logLevel)
+	if err != nil {
+		return nil, err
+	}
+
+	if agentinfo != nil && !forceUpdate && agentinfo.ID != "" {
+		return agentinfo, nil
+	}
+
+	if err := updateID(agentinfo, s); err != nil {
+		return nil, err
+	}
+
+	return agentinfo, nil
+}
+
+func updateID(agentInfo *persistentAgentInfo, s ioStore) error {
+	var err error
+	agentInfo.ID, err = generateAgentID()
+	if err != nil {
+		return err
+	}
+
+	if err := updateAgentInfo(s, agentInfo); err != nil {
+		return errors.New(err, "storing generated agent id", errors.TypeFilesystem)
+	}
+
+	return nil
+}

x-pack/elastic-agent/pkg/agent/application/info/agent_info.go

@@ -21,7 +21,7 @@ type AgentInfo struct {
 // If agent config file does not exist it gets created.
 // Initiates log level to predefined value.
 func NewAgentInfoWithLog(level string) (*AgentInfo, error) {
-	agentInfo, err := loadAgentInfo(false, level)
+	agentInfo, err := loadAgentInfoWithBackoff(false, level)
 	if err != nil {
 		return nil, err
 	}
@@ -51,6 +51,16 @@ func (i *AgentInfo) LogLevel(level string) error {
 	return nil
 }
 
+// ReloadID reloads agent info ID from configuration file.
+func (i *AgentInfo) ReloadID() error {
+	newInfo, err := NewAgentInfoWithLog(i.logLevel)
+	if err != nil {
+		return err
+	}
+	i.agentID = newInfo.agentID
+	return nil
+}
+
 // AgentID returns an agent identifier.
 func (i *AgentInfo) AgentID() string {
 	return i.agentID

x-pack/elastic-agent/pkg/agent/application/managed_mode.go

@@ -280,6 +280,11 @@ func (m *Managed) Start() error {
 		return nil
 	}
 
+	// reload ID because of win7 sync issue
+	if err := m.agentInfo.ReloadID(); err != nil {
+		return err
+	}
+
 	err := m.upgrader.Ack(m.bgContext)
 	if err != nil {
 		m.log.Warnf("failed to ack update %v", err)

x-pack/elastic-agent/pkg/agent/cmd/install.go

@@ -9,12 +9,11 @@ import (
 	"os"
 	"os/exec"
 
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application"
-	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
-
 	"github.com/spf13/cobra"
 
 	c "github.com/elastic/beats/v7/libbeat/common/cli"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filelock"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/install"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/warn"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/cli"
@@ -58,9 +57,9 @@ func installCmd(streams *cli.IOStreams, cmd *cobra.Command, flags *globalFlags,
 	}
 
 	// check the lock to ensure that elastic-agent is not already running in this directory
-	locker := application.NewAppLocker(paths.Data(), agentLockFileName)
+	locker := filelock.NewAppLocker(paths.Data(), agentLockFileName)
 	if err := locker.TryLock(); err != nil {
-		if err == application.ErrAppAlreadyRunning {
+		if err == filelock.ErrAppAlreadyRunning {
 			return fmt.Errorf("cannot perform installation as Elastic Agent is already running from this directory")
 		}
 		return err

x-pack/elastic-agent/pkg/agent/cmd/run.go

@@ -26,6 +26,7 @@ import (
 	"github.com/elastic/beats/v7/libbeat/monitoring"
 	"github.com/elastic/beats/v7/libbeat/service"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application"
+	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/filelock"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/info"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/paths"
 	"github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/application/reexec"
@@ -63,7 +64,7 @@ func run(flags *globalFlags, streams *cli.IOStreams) error { // Windows: Mark se
 	// This must be the first deferred cleanup task (last to execute).
 	defer service.NotifyTermination()
 
-	locker := application.NewAppLocker(paths.Data(), agentLockFileName)
+	locker := filelock.NewAppLocker(paths.Data(), agentLockFileName)
 	if err := locker.TryLock(); err != nil {
 		return err
 	}