setup genesis,init edges, concise leader/validator population, db oth… (

#1325)

* setup genesis,init edges, concise leader/validator population, db other eth address

unconfigurize admin.{cert,key}

* node ID from kwild key info cmd

* rename KWILD_ env prefix, early root dir handling for config.toml loading

* setup init: add key-file flag

* fix preRunBindConfigFile and test when no other root bind precedes

app/key/cmd.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/kwilteam/kwil-db/app/shared/display"
 	"github.com/kwilteam/kwil-db/core/crypto"
+	"github.com/kwilteam/kwil-db/core/crypto/auth"
+	authExt "github.com/kwilteam/kwil-db/extensions/auth"
 )
 
 func KeyCmd() *cobra.Command {
@@ -35,12 +37,20 @@ func privKeyInfo(priv crypto.PrivateKey) *PrivateKeyInfo {
 		keyText = hex.EncodeToString(priv.Bytes())
 		keyFmt = "hex"
 	}
+	var address string
+	if priv.Type() == crypto.KeyTypeSecp256k1 {
+		if s := auth.GetUserSigner(priv); s != nil {
+			address, _ = authExt.GetIdentifierFromSigner(s)
+		}
+
+	}
 	return &PrivateKeyInfo{
 		KeyType:        priv.Type().String(),
 		PrivateKeyText: keyText,
 		privKeyFmt:     keyFmt,
 		PublicKeyHex:   hex.EncodeToString(priv.Public().Bytes()),
 		NodeID:         hex.EncodeToString(priv.Public().Bytes()) + "#" + priv.Type().String(),
+		Address:        address,
 	}
 }
 
@@ -50,6 +60,9 @@ type PrivateKeyInfo struct {
 	privKeyFmt     string `json:"-"`
 	PublicKeyHex   string `json:"public_key_hex"`
 	NodeID         string `json:"node_id,omitempty"`
+	// Address is an optional field that may be set for certain key types that
+	// can generate an address depending on the signature (auth) type used.
+	Address string `json:"user_address,omitempty"`
 }
 
 func (p *PrivateKeyInfo) MarshalJSON() ([]byte, error) {
@@ -58,12 +71,28 @@ func (p *PrivateKeyInfo) MarshalJSON() ([]byte, error) {
 }
 
 func (p *PrivateKeyInfo) MarshalText() ([]byte, error) {
+	if p.Address != "" {
+		return []byte(fmt.Sprintf(`Key type: %s
+Private key (%s): %s
+Public key (plain hex): %v
+Node ID: %s
+Equivalent User Address: %s`,
+			p.KeyType,
+			p.privKeyFmt,
+			p.PrivateKeyText,
+			p.PublicKeyHex,
+			p.NodeID,
+			p.Address,
+		)), nil
+	}
 	return []byte(fmt.Sprintf(`Key type: %s
 Private key (%s): %s
-Public key (plain hex): %v`,
+Public key (plain hex): %v
+Node ID: %s`,
 		p.KeyType,
 		p.privKeyFmt,
 		p.PrivateKeyText,
 		p.PublicKeyHex,
+		p.NodeID,
 	)), nil
 }

app/key/info.go

@@ -18,7 +18,7 @@ var (
 The private key can either be passed as a key file path, or as a hex-encoded string.`
 
 	infoExample = `# Using a key file
-kwild key info --key-file ~/.kwild/private_key
+kwild key info --key-file ~/.kwild/nodekey.json
 
 # Using a hex-encoded string
 kwild key info 381d28cf348c9efbf7d26ea54b647e2cb646d3b98cdeec0f1053a5ff599a036a0aa381bd4aad1670a39977d5416bfac7bd060765adc58a4bb16bbbafeefbae34`

app/node/build.go

@@ -18,6 +18,7 @@ import (
 	"github.com/kwilteam/kwil-db/common"
 	"github.com/kwilteam/kwil-db/config"
 	"github.com/kwilteam/kwil-db/core/crypto/auth"
+	"github.com/kwilteam/kwil-db/core/rpc/transport"
 	"github.com/kwilteam/kwil-db/extensions/precompiles"
 	"github.com/kwilteam/kwil-db/node"
 	"github.com/kwilteam/kwil-db/node/accounts"
@@ -597,7 +598,7 @@ func buildJRPCAdminServer(d *coreDependencies) *rpcserver.Server {
 	if err != nil {
 		if strings.Contains(err.Error(), "missing port in address") {
 			host = addr
-			port = "8484"
+			port = "8584"
 		} else if strings.Contains(err.Error(), "too many colons in address") {
 			u, err := url.Parse(addr)
 			if err != nil {
@@ -639,8 +640,8 @@ func buildJRPCAdminServer(d *coreDependencies) *rpcserver.Server {
 				d.logger.Warn("disabling TLS on non-loopback admin service listen address",
 					"addr", addr, "with_password", adminPass != "")
 			} else {
-				withClientAuth := adminPass == "" // no basic http auth => use transport layer auth
-				opts = append(opts, rpcserver.WithTLS(tlsConfig(d, withClientAuth)))
+				withTransportClientAuth := adminPass == "" // no basic http auth => use transport layer auth
+				opts = append(opts, rpcserver.WithTLS(tlsConfig(d, withTransportClientAuth)))
 			}
 		}
 	}
@@ -686,13 +687,14 @@ func loadTLSCertificate(keyFile, certFile, hostname string) (*tls.Certificate, e
 }
 
 // tlsConfig returns a tls.Config to be used with the admin RPC service. If
-// withClientAuth is true, the config will require client authentication (mutual
-// TLS), otherwise it is standard TLS for encryption and server authentication.
-func tlsConfig(d *coreDependencies, withClientAuth bool) *tls.Config {
+// withTransportClientAuth is true, the config will require client
+// authentication (mutual TLS), otherwise it is standard TLS for encryption and
+// server authentication.
+func tlsConfig(d *coreDependencies, withTransportClientAuth bool) *tls.Config {
 	if d.adminKey == nil {
 		return nil
 	}
-	if !withClientAuth {
+	if !withTransportClientAuth {
 		// TLS only for encryption and authentication of server to client.
 		return &tls.Config{
 			Certificates: []tls.Certificate{*d.adminKey},
@@ -708,24 +710,24 @@ func tlsConfig(d *coreDependencies, withClientAuth bool) *tls.Config {
 		if err != nil {
 			failBuild(err, "failed to load client CAs file")
 		}
-	} else /*else if d.autogen {
-		clientCredsFileBase := filepath.Join(d.rootDir, "auth")
+		d.logger.Infoln("loaded client CAs from", clientsFile)
+	} else if d.autogen {
+		clientCredsFileBase := filepath.Join(d.rootDir, "adminclient")
 		clientCertFile, clientKeyFile := clientCredsFileBase+".cert", clientCredsFileBase+".key"
 		err = transport.GenTLSKeyPair(clientCertFile, clientKeyFile, "local kwild CA", nil)
 		if err != nil {
 			failBuild(err, "failed to generate admin client credentials")
 		}
-		d.logger.Info("generated admin service client key pair", log.String("cert", clientCertFile), log.String("key", clientKeyFile))
+		d.logger.Info("generated admin service client key pair", "cert", clientCertFile, "key", clientKeyFile)
 		if clientsCerts, err = os.ReadFile(clientCertFile); err != nil {
 			failBuild(err, "failed to read auto-generate client certificate")
 		}
 		if err = os.WriteFile(clientsFile, clientsCerts, 0644); err != nil {
 			failBuild(err, "failed to write client CAs file")
 		}
-		d.logger.Info("generated admin service client CAs file", log.String("file", clientsFile))
-	} */
-	{
-		d.logger.Info("No admin client CAs file. Use kwild's admin gen-auth-key command to generate")
+		d.logger.Info("generated admin service client CAs file", "file", clientsFile)
+	} else {
+		d.logger.Info("No admin client CAs file. Use 'kwild admin gen-auth-key' to generate.")
 	}
 
 	if len(clientsCerts) > 0 && !caCertPool.AppendCertsFromPEM(clientsCerts) {

app/node/conf/conf.go

@@ -127,7 +127,41 @@ func PreRunBindFlags(cmd *cobra.Command, args []string) error {
 // preceded env, they are loaded assuming every "_" is a section delimiter,
 // which may be incorrect for multi-word keys like KWIL_SECTION_SOME_KEY.
 func PreRunBindEnvMatching(cmd *cobra.Command, args []string) error {
-	return bind.PreRunBindEnvMatchingTo(cmd, args, "KWIL_", k)
+	return bind.PreRunBindEnvMatchingTo(cmd, args, "KWILD_", k)
+}
+
+// PreRunBindEarlyRootDirEnv updates the active config's root directory from
+// KWILD_ROOT. This allows downstream sources, such as config file loading, to
+// use the root directory, while having the other environment variables override
+// config file values.
+func PreRunBindEarlyRootDirEnv(cmd *cobra.Command, args []string) error {
+	const rootEnvVar = "KWILD_ROOT"
+	return k.Load(env.ProviderWithValue(rootEnvVar, ".", func(s, v string) (string, any) {
+		if len(s) != len(rootEnvVar) { // KWILD_ROOT_OTHER
+			return "", nil // ignore this variable
+		}
+		bind.Debugf("root env var changed: %s", v)
+		return bind.RootFlagName, v // "root" = value_of_var
+	}), nil)
+}
+
+// PreRunBindEarlyRootDirFlag is like PreRunBindEarlyRootDirEnv, but for the
+// root flag. These two "early" preruns are separate from each other as well as
+// from the other preruns such as PreRunBindConfigFile.
+func PreRunBindEarlyRootDirFlag(cmd *cobra.Command, args []string) error {
+	rootFlag := cmd.Flag(bind.RootFlagName)
+	if rootFlag == nil {
+		return fmt.Errorf("root flag not found: %s", bind.RootFlagName)
+	}
+	if !rootFlag.Changed {
+		return nil
+	}
+
+	rootDir := rootFlag.Value.String()
+	bind.Debugf("root flag changed: %v", rootDir)
+	k.Set(bind.RootFlagName, rootDir)
+
+	return nil
 }
 
 // PreRunBindEnvAllSections treats all underscores as section delimiters. With
@@ -141,24 +175,27 @@ func PreRunBindEnvMatching(cmd *cobra.Command, args []string) error {
 // To merge all, k.Load from each source should merge by standardizing the key
 // names into "twowords", AND the `koanf` tag should match.
 func PreRunBindEnvAllSections(cmd *cobra.Command, args []string) error {
-	k.Load(env.Provider("KWIL_", ".", func(s string) string {
+	k.Load(env.Provider("KWILD_", ".", func(s string) string {
 		// The following , not the above goal.
 		// KWIL_SECTION_SUBSECTION_SOMEVALUE => section.subsection.somevalue
 		// Values cannot have underscores in this convention!
-		s, _ = strings.CutPrefix(s, "KWIL_")
+		s, _ = strings.CutPrefix(s, "KWILD_")
 		return strings.ReplaceAll(strings.ToLower(s), "_", ".")
 	}), nil)
 	return nil
 }
 
 func preRunBindConfigFile(cmd *cobra.Command, args []string, parser koanf.Parser) error {
-	// Grab the root directory like other commands that will access it via
-	// persistent flags from the root command.
-	rootDir, err := bind.RootDir(cmd)
-	if err != nil {
-		return err // a parent command needs to have a persistent flag named "root"
+	rootDir := RootDir() // from k, requires and "early" bind
+	if rootDir == "" {
+		// The command did not configure a root dir prerun, so we'll take the
+		// liberty of assuming we can try to import it from pflags.
+		var err error
+		rootDir, err = bind.RootDir(cmd)
+		if err != nil {
+			return err // a parent command needs to have a persistent flag named "root"
+		}
 	}
-	// rootDir := RootDir() // from k, requires BindDefaultsWithRootDir or a root field of the config struct
 
 	// If we want to instead have space placeholders removed (to match
 	// PreRunBindEnvAllSections) rather than having them be standardized to
@@ -175,7 +212,7 @@ func preRunBindConfigFile(cmd *cobra.Command, args []string, parser koanf.Parser
 
 	if err := k.Load(file.Provider(confPath), parser /*, mergeFn*/); err != nil {
 		if !errors.Is(err, os.ErrNotExist) {
-			return fmt.Errorf("error loading config: %v", err)
+			return fmt.Errorf("error loading config from %s: %v", confPath, err)
 		}
 		// Not an error, just no config file present.
 		bind.Debugf("No config file present at %v", confPath)

app/node/conf/conf_test.go

@@ -33,6 +33,8 @@ string_value = "nested-string"
 	// Create test command with root flag
 	cmd := &cobra.Command{Use: "test"}
 	cmd.Flags().String("root", tmpDir, "root directory")
+	rootFlag := cmd.Flag("root")
+	rootFlag.Changed = true
 
 	err = PreRunBindConfigFile(cmd, []string{})
 	assert.NoError(t, err)

app/node/deps.go

@@ -64,6 +64,7 @@ type coreDependencies struct {
 	closers *closeFuncs // for clean close on failBuild
 
 	adminKey *tls.Certificate
+	autogen  bool
 
 	logger           log.Logger
 	dbOpener         dbOpener

app/node/node.go

@@ -98,17 +98,15 @@ func runNode(ctx context.Context, rootDir string, cfg *config.Config, autogen bo
 
 	logger.Infof("Node public key: %x (%s)", privKey.Public().Bytes(), privKey.Public().Type())
 
+	logger.Info("loading TLS key pair for the admin server if TLS enabled",
+		"key_file", config.AdminServerKeyName, "cert_file", config.AdminServerCertName)
+	customHostname := "" // cfg TODO
+	keyFile := rootedPath(config.AdminServerKeyName, rootDir)
+	certFile := rootedPath(config.AdminServerCertName, rootDir)
 	var tlsKeyPair *tls.Certificate
-	logger.Info("loading TLS key pair for the admin server", "key_file", cfg.Admin.TLSKeyFile,
-		"cert_file", cfg.Admin.TLSKeyFile)
-	if cfg.Admin.TLSKeyFile != "" || cfg.Admin.TLSCertFile != "" {
-		customHostname := "" // cfg TODO
-		keyFile := rootedPath(cfg.Admin.TLSKeyFile, rootDir)
-		certFile := rootedPath(cfg.Admin.TLSCertFile, rootDir)
-		tlsKeyPair, err = loadTLSCertificate(keyFile, certFile, customHostname)
-		if err != nil {
-			return err
-		}
+	tlsKeyPair, err = loadTLSCertificate(keyFile, certFile, customHostname)
+	if err != nil {
+		return err
 	}
 
 	host, port, user, pass := cfg.DB.Host, cfg.DB.Port, cfg.DB.User, cfg.DB.Pass
@@ -162,6 +160,7 @@ func runNode(ctx context.Context, rootDir string, cfg *config.Config, autogen bo
 		genesisCfg:       genConfig,
 		privKey:          privKey,
 		logger:           logger,
+		autogen:          autogen,
 		dbOpener:         newDBOpener(host, port, user, pass, nsmgr.Filter),
 		namespaceManager: nsmgr,
 		poolOpener:       newPoolBOpener(host, port, user, pass),

app/node/printcfg.go

@@ -16,9 +16,8 @@ func PrintConfigCmd() *cobra.Command {
 		Short: "Print the node configuration",
 		Long:  `The print-config command shows the parsed node configuration based on the combination of the default configuration, configuration file, flags,and environment variables. The configuration is printed to stdout in TOML format. All flags available to the start command are recognized by this command.`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			_, err := bind.RootDir(cmd)
-			if err != nil {
-				return err // the parent command needs to set a persistent flag named "root"
+			if conf.RootDir() == "" {
+				return fmt.Errorf("root directory not set") // bug, parent command did not set default
 			}
 
 			cfg := conf.ActiveConfig()

app/node/start.go

@@ -31,10 +31,7 @@ func StartCmd() *cobra.Command {
 		Version: version.KwilVersion,
 		Example: custom.BinaryConfig.NodeCmd + " start -r .testnet",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			rootDir, err := bind.RootDir(cmd)
-			if err != nil {
-				return err // the parent command needs to set a persistent flag named "root"
-			}
+			rootDir := conf.RootDir()
 
 			cfg := conf.ActiveConfig()
 

app/root.go

@@ -46,10 +46,13 @@ func RootCmd() *cobra.Command {
 			DisableDefaultCmd: true,
 		},
 		Version: version.KwilVersion,
-		Example: custom.BinaryConfig.NodeCmd + " -r ~/.kwild",
 		// PersistentPreRunE so k has all the settings in all (sub)command's RunE funcs
-		PersistentPreRunE: bind.ChainPreRuns(bind.MaybeEnableCLIDebug, conf.PreRunBindConfigFileStrict[config.Config],
-			conf.PreRunBindFlags, conf.PreRunBindEnvMatching, conf.PreRunPrintEffectiveConfig),
+		PersistentPreRunE: bind.ChainPreRuns(bind.MaybeEnableCLIDebug,
+			conf.PreRunBindEarlyRootDirEnv, conf.PreRunBindEarlyRootDirFlag, // bind root from env and flag first
+			conf.PreRunBindConfigFileStrict[config.Config], // then config file
+			conf.PreRunBindFlags,                           // then flags
+			conf.PreRunBindEnvMatching,                     // then env vars
+			conf.PreRunPrintEffectiveConfig),
 	}
 
 	bind.BindDebugFlag(cmd) // --debug enabled CLI debug mode (shared.Debugf output)

app/rpc/admin.go

@@ -28,8 +28,8 @@ func BindRPCFlags(cmd *cobra.Command) {
 
 	cmd.PersistentFlags().String("authrpc-cert", "", "kwild's TLS server certificate, required for HTTPS server")
 	cmd.PersistentFlags().String("pass", "", "admin server password (alternative to mTLS with tlskey/tlscert)")
-	cmd.PersistentFlags().String("tlskey", "auth.key", "kwild's TLS client key file to establish a mTLS (authenticated) connection")
-	cmd.PersistentFlags().String("tlscert", "auth.cert", "kwild's TLS client certificate file for server to authenticate us")
+	cmd.PersistentFlags().String("tlskey", "auth.key", "TLS client key file to establish a mTLS (authenticated) connection")
+	cmd.PersistentFlags().String("tlscert", "auth.cert", "TLS client certificate file for server to authenticate us")
 }
 
 // GetRPCServerFlag returns the RPC flag from the given command.
@@ -127,7 +127,7 @@ func (nc *nodeCert) tlsFiles(rootDir string) (nodeCert, ourKey, ourCert string,
 	if nodeCert != "" {
 		nodeCert = fullPath(nodeCert, rootDir)
 	} else {
-		nodeCert = fullPath(config.AdminCertName, rootDir) // ~/.kwild/admin.cert
+		nodeCert = fullPath(config.AdminServerCertName, rootDir) // ~/.kwild/admin.cert
 	}
 	if nodeCert == "" || !fileExists(nodeCert) {
 		err = fmt.Errorf("kwild cert file not found, checked %v", nodeCert)

app/rpc/gen-auth-key.go

@@ -14,10 +14,10 @@ import (
 var (
 	genAuthKeyLong = `The ` + "`gen-auth-key`" + `command generates a new key pair for use with an authenticated admin RPC service.
 	
-The key pair is generated and stored in the node's configuration directory, in the files ` + "`auth.key`" + ` and ` + "`auth.cert`" + `. The key pair is used to authenticate the admin tool to the node.`
+The key pair is generated and stored in the node's configuration directory, in the files ` + "`adminclient.key`" + ` and ` + "`adminclient.cert`" + `. The key pair is used to authenticate the admin tool to the node.`
 	genAuthKeyExample = `# Generate a new TLS key pair to talk to the node
 kwild admin gen-auth-key
-# kwild admin commands uses auth.{key,cert}, while kwild uses clients.pem`
+# kwild admin commands uses adminclient.{key,cert}, while kwild uses clients.pem`
 )
 
 func genAuthKeyCmd() *cobra.Command {
@@ -56,16 +56,19 @@ func genAuthKeyCmd() *cobra.Command {
 			if err != nil {
 				return display.PrintErr(cmd, fmt.Errorf("failed to generate TLS key pair: %v", err))
 			}
-			certText, err := os.ReadFile(certFile)
+
+			display.PrintCmd(cmd, display.RespString(fmt.Sprintf("TLS key pair generated in %v and %v\n", keyFile, certFile)))
+
+			certText, err := os.ReadFile(certFile + "x")
 			if err != nil {
 				return display.PrintErr(cmd, fmt.Errorf("failed to read cert file: %v", err))
 			}
 			return appendToFile(filepath.Join(rootDir, "clients.pem"), certText)
 		},
 	}
 
-	cmd.Flags().StringVar(&keyFile, "tlskey", "auth.key", "output path for the new client key file")
-	cmd.Flags().StringVar(&certFile, "tlscert", "auth.cert", "output path for the new client certificate")
+	cmd.Flags().StringVar(&keyFile, "tlskey", "adminclient.key", "output path for the new client key file")
+	cmd.Flags().StringVar(&certFile, "tlscert", "adminclient.cert", "output path for the new client certificate")
 
 	return cmd
 }