From 86f37b286e58d4283884f6dbe1ee56caeb2e2cd2 Mon Sep 17 00:00:00 2001
From: stefanprodan <stefan.prodan@gmail.com>
Date: Thu, 26 Sep 2019 17:54:58 +0300
Subject: [PATCH] Make sync operations timeout configurable

---
 cmd/fluxd/main.go         |  2 ++
 pkg/daemon/daemon.go      | 13 ++-----------
 pkg/daemon/daemon_test.go |  2 +-
 pkg/daemon/loop.go        |  1 +
 pkg/daemon/sync_test.go   |  2 +-
 5 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/cmd/fluxd/main.go b/cmd/fluxd/main.go
index 9fdea0e30c..df485cba18 100644
--- a/cmd/fluxd/main.go
+++ b/cmd/fluxd/main.go
@@ -135,6 +135,7 @@ func main() {
 
 		// syncing
 		syncInterval = fs.Duration("sync-interval", 5*time.Minute, "apply config in git to cluster at least this often, even if there are no new commits")
+		syncTimeout  = fs.Duration("sync-timeout", 1*time.Minute, "duration after which sync operations time out")
 		syncGC       = fs.Bool("sync-garbage-collection", false, "experimental; delete resources that were created by fluxd, but are no longer in the git repo")
 		dryGC        = fs.Bool("sync-garbage-collection-dry", false, "experimental; only log what would be garbage collected, rather than deleting. Implies --sync-garbage-collection")
 		syncState    = fs.String("sync-state", fluxsync.GitTagStateMode, fmt.Sprintf("method used by flux for storing state (one of {%s})", strings.Join([]string{fluxsync.GitTagStateMode, fluxsync.NativeStateMode}, ",")))
@@ -689,6 +690,7 @@ func main() {
 		ManifestGenerationEnabled: *manifestGeneration,
 		LoopVars: &daemon.LoopVars{
 			SyncInterval:        *syncInterval,
+			SyncTimeout:         *syncTimeout,
 			SyncState:           syncProvider,
 			AutomationInterval:  *automationInterval,
 			GitTimeout:          *gitTimeout,
diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go
index d05a8ca327..1e5b3b6c98 100644
--- a/pkg/daemon/daemon.go
+++ b/pkg/daemon/daemon.go
@@ -31,15 +31,6 @@ import (
 	"github.com/fluxcd/flux/pkg/update"
 )
 
-const (
-	// This is set to be in sympathy with the request / RPC timeout (i.e., empirically)
-	defaultHandlerTimeout = 10 * time.Second
-	// A job can take an arbitrary amount of time but we want to have
-	// a (generous) threshold for considering a job stuck and
-	// abandoning it
-	defaultJobTimeout = 60 * time.Second
-)
-
 // Daemon is the fully-functional state of a daemon (compare to
 // `NotReadyDaemon`).
 type Daemon struct {
@@ -276,7 +267,7 @@ func (d *Daemon) makeJobFromUpdate(update updateFunc) jobFunc {
 // executeJob runs a job func and keeps track of its status, so the
 // daemon can report it when asked.
 func (d *Daemon) executeJob(id job.ID, do jobFunc, logger log.Logger) (job.Result, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), defaultJobTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), d.SyncTimeout)
 	defer cancel()
 	d.JobStatusCache.SetStatus(id, job.Status{StatusString: job.StatusRunning})
 	result, err := do(ctx, id, logger)
@@ -371,7 +362,7 @@ func (d *Daemon) UpdateManifests(ctx context.Context, spec update.Spec) (job.ID,
 func (d *Daemon) sync() jobFunc {
 	return func(ctx context.Context, jobID job.ID, logger log.Logger) (job.Result, error) {
 		var result job.Result
-		ctx, cancel := context.WithTimeout(ctx, defaultJobTimeout)
+		ctx, cancel := context.WithTimeout(ctx, d.SyncTimeout)
 		defer cancel()
 		err := d.Repo.Refresh(ctx)
 		if err != nil {
diff --git a/pkg/daemon/daemon_test.go b/pkg/daemon/daemon_test.go
index ad55a0b93b..c7891877b9 100644
--- a/pkg/daemon/daemon_test.go
+++ b/pkg/daemon/daemon_test.go
@@ -741,7 +741,7 @@ func mockDaemon(t *testing.T) (*Daemon, func(), func(), *mock.Mock, *mockEventWr
 		JobStatusCache: &job.StatusCache{Size: 100},
 		EventWriter:    events,
 		Logger:         logger,
-		LoopVars:       &LoopVars{GitTimeout: timeout, SyncState: gitSync},
+		LoopVars:       &LoopVars{SyncTimeout: timeout, GitTimeout: timeout, SyncState: gitSync},
 	}
 
 	start := func() {
diff --git a/pkg/daemon/loop.go b/pkg/daemon/loop.go
index 3a6dfe692c..467e1175f8 100644
--- a/pkg/daemon/loop.go
+++ b/pkg/daemon/loop.go
@@ -15,6 +15,7 @@ import (
 
 type LoopVars struct {
 	SyncInterval        time.Duration
+	SyncTimeout         time.Duration
 	AutomationInterval  time.Duration
 	GitTimeout          time.Duration
 	GitVerifySignatures bool
diff --git a/pkg/daemon/sync_test.go b/pkg/daemon/sync_test.go
index 39525a025c..d159c35feb 100644
--- a/pkg/daemon/sync_test.go
+++ b/pkg/daemon/sync_test.go
@@ -75,7 +75,7 @@ func daemon(t *testing.T) (*Daemon, func()) {
 		JobStatusCache: &job.StatusCache{Size: 100},
 		EventWriter:    events,
 		Logger:         log.NewLogfmtLogger(os.Stdout),
-		LoopVars:       &LoopVars{GitTimeout: timeout},
+		LoopVars:       &LoopVars{SyncTimeout: timeout, GitTimeout: timeout},
 	}
 	return d, func() {
 		close(shutdown)