Make plugin start timeout configurable. (#4321)

pkg/cmdconfig/viper.go

@@ -2,10 +2,11 @@ package cmdconfig
 
 import (
 	"fmt"
-	"github.com/turbot/steampipe/pkg/filepaths"
 	"log"
 	"os"
 
+	"github.com/turbot/steampipe/pkg/filepaths"
+
 	filehelpers "github.com/turbot/go-kit/files"
 	"github.com/turbot/steampipe/pkg/steampipeconfig"
 
@@ -175,6 +176,7 @@ func setDefaultsFromEnv() {
 		constants.EnvCacheMaxTTL:           {[]string{constants.ArgCacheMaxTtl}, Int},
 		constants.EnvMemoryMaxMb:           {[]string{constants.ArgMemoryMaxMb}, Int},
 		constants.EnvMemoryMaxMbPlugin:     {[]string{constants.ArgMemoryMaxMbPlugin}, Int},
+		constants.EnvPluginStartTimeout:    {[]string{constants.ArgPluginStartTimeout}, Int},
 
 		// we need this value to go into different locations
 		constants.EnvCacheEnabled: {[]string{

pkg/constants/args.go

@@ -71,6 +71,7 @@ const (
 	ArgDatabaseSSLPassword     = "database-ssl-password"
 	ArgMemoryMaxMb             = "memory-max-mb"
 	ArgMemoryMaxMbPlugin       = "memory-max-mb-plugin"
+	ArgPluginStartTimeout      = "plugin-start-timeout"
 )
 
 // metaquery mode arguments

pkg/constants/default_options.go

@@ -33,5 +33,6 @@ const DefaultConnectionConfigContent = `
 
 # options "plugin" {
 #   memory_max_mb    = "1024"	# the default maximum memory to allow a plugin process - used if there is not max memory specified in the 'plugin' block' for that plugin
+#   start_timeout    = 30       # maximum time (in seconds) to wait for a plugin to start up
 # }
 `

pkg/constants/duration.go

@@ -9,4 +9,5 @@ var (
 	DBRecoveryTimeout        = 24 * time.Hour
 	DBRecoveryRetryBackoff   = 200 * time.Millisecond
 	ServicePingInterval      = 50 * time.Millisecond
+	PluginStartTimeout       = 30 * time.Second
 )

pkg/constants/env.go

@@ -43,4 +43,6 @@ const (
 
 	EnvMemoryMaxMb       = "STEAMPIPE_MEMORY_MAX_MB"
 	EnvMemoryMaxMbPlugin = "STEAMPIPE_PLUGIN_MEMORY_MAX_MB"
+
+	EnvPluginStartTimeout = "STEAMPIPE_PLUGIN_START_TIMEOUT"
 )

pkg/pluginmanager_service/plugin_manager.go

@@ -668,9 +668,24 @@ func (m *PluginManager) notifyNewDynamicSchemas(pluginClient *sdkgrpc.PluginClie
 }
 
 func (m *PluginManager) waitForPluginLoad(p *runningPlugin, req *pb.GetRequest) error {
-	log.Printf("[TRACE] waitForPluginLoad (%p)", req)
-	// TODO make this configurable
-	pluginStartTimeoutSecs := 30
+
+	pluginConfig := m.plugins[p.pluginInstance]
+	if pluginConfig == nil {
+		// not expected
+		return sperr.New("plugin manager has no config for plugin instance %s", p.pluginInstance)
+	}
+	pluginStartTimeoutSecs := pluginConfig.GetStartTimeout()
+	if pluginStartTimeoutSecs == 0 {
+		if viper.IsSet(constants.ArgMemoryMaxMbPlugin) {
+			pluginStartTimeoutSecs = viper.GetInt64(constants.ArgPluginStartTimeout)
+		}
+	}
+	if pluginStartTimeoutSecs == 0 {
+		// if we don't have any timeout set use 30 seconds
+		pluginStartTimeoutSecs = int64(30)
+	}
+
+	log.Printf("[TRACE] waitForPluginLoad: waiting %d seconds (%p)", pluginStartTimeoutSecs, req)
 
 	// wait for the plugin to be initialized
 	select {

pkg/steampipeconfig/modconfig/plugin.go

@@ -13,6 +13,7 @@ type Plugin struct {
 	Instance        string         `hcl:"name,label" db:"plugin_instance"`
 	Alias           string         `hcl:"source,optional"`
 	MemoryMaxMb     *int           `hcl:"memory_max_mb,optional" db:"memory_max_mb"`
+	StartTimeout    *int           `hcl:"start_timeout,optional"`
 	Limiters        []*RateLimiter `hcl:"limiter,block" db:"limiters"`
 	FileName        *string        `db:"file_name"`
 	StartLineNumber *int           `db:"start_line_number"`
@@ -61,6 +62,15 @@ func (l *Plugin) GetMaxMemoryBytes() int64 {
 	}
 	return int64(1024 * 1024 * memoryMaxMb)
 }
+
+func (l *Plugin) GetStartTimeout() int64 {
+	startTimout := 0
+	if l.StartTimeout != nil {
+		startTimout = *l.StartTimeout
+	}
+	return int64(startTimout)
+}
+
 func (l *Plugin) GetLimiterMap() map[string]*RateLimiter {
 	res := make(map[string]*RateLimiter, len(l.Limiters))
 	for _, l := range l.Limiters {
@@ -74,6 +84,7 @@ func (l *Plugin) Equals(other *Plugin) bool {
 	return l.Instance == other.Instance &&
 		l.Alias == other.Alias &&
 		l.GetMaxMemoryBytes() == other.GetMaxMemoryBytes() &&
+		l.GetStartTimeout() == other.GetStartTimeout() &&
 		l.Plugin == other.Plugin &&
 		// compare limiters ignoring order
 		maps.EqualFunc(l.GetLimiterMap(), other.GetLimiterMap(), func(l, r *RateLimiter) bool { return l.Equals(r) })
@@ -91,4 +102,3 @@ func ResolvePluginImageRef(pluginAlias string) string {
 	// ok so there is no plugin block reference - build the plugin image ref from the PluginAlias field
 	return ociinstaller.NewSteampipeImageRef(pluginAlias).DisplayImageRef()
 }
-

pkg/steampipeconfig/options/plugin.go

@@ -8,7 +8,8 @@ import (
 )
 
 type Plugin struct {
-	MemoryMaxMb *int `hcl:"memory_max_mb"`
+	MemoryMaxMb  *int `hcl:"memory_max_mb"`
+	StartTimeout *int `hcl:"start_timeout"`
 }
 
 // ConfigMap creates a config map that can be merged with viper
@@ -18,6 +19,11 @@ func (t *Plugin) ConfigMap() map[string]interface{} {
 	if t.MemoryMaxMb != nil {
 		res[constants.ArgMemoryMaxMbPlugin] = t.MemoryMaxMb
 	}
+	if t.StartTimeout != nil {
+		res[constants.ArgPluginStartTimeout] = t.StartTimeout
+	} else {
+		res[constants.ArgPluginStartTimeout] = constants.PluginStartTimeout.Seconds()
+	}
 
 	return res
 }
@@ -30,6 +36,9 @@ func (t *Plugin) Merge(otherOptions Options) {
 		if o.MemoryMaxMb != nil {
 			t.MemoryMaxMb = o.MemoryMaxMb
 		}
+		if o.StartTimeout != nil {
+			t.StartTimeout = o.StartTimeout
+		}
 	}
 }
 
@@ -43,6 +52,11 @@ func (t *Plugin) String() string {
 	} else {
 		str = append(str, fmt.Sprintf("  MemoryMaxMb: %d", *t.MemoryMaxMb))
 	}
+	if t.StartTimeout == nil {
+		str = append(str, "  PluginStartTimeout: nil")
+	} else {
+		str = append(str, fmt.Sprintf("  PluginStartTimeout: %d", *t.StartTimeout))
+	}
 
 	return strings.Join(str, "\n")
 }