Agents platform and new CLI

Dockerfile

@@ -43,7 +43,7 @@ RUN mv /tmp/qed/c-deps/* c-deps/
 # Build QED, Storage binary and riot
 RUN go build -o /usr/local/bin/qed                                   &&\
     go build -o /usr/local/bin/riot tests/riot.go                    &&\
-    go build -o /usr/local/bin/storage tests/gossip/test_service.go
+    go build -o /usr/local/bin/storage testutils/notifierstore.go
 
 # Clean
 RUN rm -rf /var/lib/apt/lists/* /tmp/qed

api/apihttp/apihttp.go

@@ -253,13 +253,13 @@ func Incremental(balloon raftwal.RaftBalloonApi) http.HandlerFunc {
 		// Wait for the response
 		proof, err := balloon.QueryConsistency(request.Start, request.End)
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
+			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
 
 		out, err := json.Marshal(protocol.ToIncrementalResponse(proof))
 		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
+			http.Error(w, err.Error(), http.StatusBadRequest)
 			return
 		}
 

balloon/balloon.go

@@ -288,10 +288,7 @@ func (b Balloon) QueryConsistency(start, end uint64) (*IncrementalProof, error)
 	stats.AddFloat("QueryConsistency", 1)
 	var proof IncrementalProof
 
-	if start >= b.version ||
-		end >= b.version ||
-		start >= end {
-
+	if start >= b.version || end >= b.version || start > end {
 		return nil, errors.New("unable to process proof from history tree: invalid range")
 	}
 

client/config.go

@@ -89,52 +89,56 @@ const (
 
 // Config sets the HTTP client configuration
 type Config struct {
+	// Log level
+	Log string `desc:"Set log level to info, error or debug"`
+
 	// Endpoints [host:port,host:port,...] to ask for QED cluster-topology.
-	Endpoints []string
+	Endpoints []string `desc:"REST QED Log service endpoint list http://ip1:port1,http://ip2:port2... "`
 
 	// ApiKey to query the server endpoint.
-	APIKey string
+	APIKey string `desc:"Set API Key to talk to QED Log service"`
 
 	// Insecure enables the verification of the server's certificate chain
 	// and host name, allowing MiTM vector attacks.
-	Insecure bool
+	Insecure bool `desc:"Set it to true to disable the verification of the server's certificate chain"`
 
-	// Timeout is the number of seconds to wait for a request to QED.
-	Timeout time.Duration
+	// Timeout is the time to wait for a request to QED.
+	Timeout time.Duration `desc:"Time to wait for a request to QED"`
 
-	// DialTimeout is the number of seconds to wait for the connection to be established.
-	DialTimeout time.Duration
+	// DialTimeout is the time to wait for the connection to be established.
+	DialTimeout time.Duration `desc:"Time to wait for the connection to be established"`
 
-	// HandshakeTimeout is the number of seconds to wait for a handshake negotiation.
-	HandshakeTimeout time.Duration
+	// HandshakeTimeout is the time to wait for a handshake negotiation.
+	HandshakeTimeout time.Duration `desc:"Time to wait for a handshake negotiation"`
 
 	// Controls how the client will route all queries to members of the cluster.
-	ReadPreference ReadPref
+	ReadPreference ReadPref `flag:"-"`
 
 	// MaxRetries sets the maximum number of retries before giving up
 	// when performing an HTTP request to QED.
-	MaxRetries int
+	MaxRetries int `desc:"Sets the maximum number of retries before giving up"`
 
 	// EnableTopologyDiscovery enables the process of discovering the cluster
 	// topology when requests fail.
-	EnableTopologyDiscovery bool
+	EnableTopologyDiscovery bool `desc:"Enables the process of discovering the cluster topology when requests fail"`
 
 	// EnableHealthChecks enables helthchecks of all endpoints in the current cluster topology.
-	EnableHealthChecks bool
+	EnableHealthChecks bool `desc:"Enables helthchecks of all endpoints in the current cluster topology"`
 
-	// HealthCheckTimeout is the timeout in seconds the healthcheck waits for a response
+	// HealthCheckTimeout is the time the healthcheck waits for a response
 	// from a QED server.
-	HealthCheckTimeout time.Duration
+
+	HealthCheckTimeout time.Duration `desc:"Time the healthcheck waits for a response from QED"`
 
 	// AttemptToReviveEndpoints sets if dead endpoints will be marked alive again after a
 	// round-robin round. This way, they will be picked up in the next try.
-	AttemptToReviveEndpoints bool
+	AttemptToReviveEndpoints bool `desc:"Set if dead endpoints will be marked alive again after a round-robin round"`
 }
 
 // DefaultConfig creates a Config structures with default values.
 func DefaultConfig() *Config {
 	return &Config{
-		Endpoints:                []string{"127.0.0.1:8800"},
+		Endpoints:                []string{"http://127.0.0.1:8800"},
 		APIKey:                   "my-key",
 		Insecure:                 DefaultInsecure,
 		Timeout:                  DefaultTimeout,

cmd/agent.go

@@ -17,76 +17,45 @@
 package cmd
 
 import (
-	"regexp"
-
-	"github.com/spf13/cobra"
-	v "github.com/spf13/viper"
+	"context"
 
 	"github.com/bbva/qed/gossip"
+	"github.com/bbva/qed/log"
+	"github.com/octago/sflags/gen/gpflag"
+	"github.com/spf13/cobra"
 )
 
-func newAgentCommand(cmdCtx *cmdContext, args []string) *cobra.Command {
-
-	config := gossip.DefaultConfig()
-
-	cmd := &cobra.Command{
-		Use:   "agent",
-		Short: "Start a gossip agent for the verifiable log QED",
-	}
-
-	f := cmd.PersistentFlags()
-	f.StringVar(&config.NodeName, "node", "", "Unique name for node. If not set, fallback to hostname")
-	f.StringVar(&config.BindAddr, "bind", "", "Bind address for TCP/UDP gossip on (host:port)")
-	f.StringVar(&config.AdvertiseAddr, "advertise", "", "Address to advertise to cluster")
-	f.StringVar(&config.MetricsAddr, "metrics", "", "Address to bind metrics endpoint")
-	f.StringSliceVar(&config.StartJoin, "join", []string{}, "Comma-delimited list of nodes ([host]:port), through which a cluster can be joined")
-	f.StringSliceVar(&config.AlertsUrls, "alertsUrls", []string{}, "Comma-delimited list of Alert servers ([host]:port), through which an agent can post alerts")
-
-	// Lookups
-	v.BindPFlag("agent.node", f.Lookup("node"))
-	v.BindPFlag("agent.bind", f.Lookup("bind"))
-	v.BindPFlag("agent.advertise", f.Lookup("advertise"))
-	v.BindPFlag("agent.metrics", f.Lookup("metrics"))
-	v.BindPFlag("agent.join", f.Lookup("join"))
-	v.BindPFlag("agent.alerts_urls", f.Lookup("alertsUrls"))
-
-	agentPreRun := func(config gossip.Config) gossip.Config {
-		config.EnableCompression = true
-		config.NodeName = v.GetString("agent.node")
-		config.BindAddr = v.GetString("agent.bind")
-		config.AdvertiseAddr = v.GetString("agent.advertise")
-		config.MetricsAddr = v.GetString("agent.metrics")
-		config.StartJoin = v.GetStringSlice("agent.join")
-		config.AlertsUrls = v.GetStringSlice("agent.alerts_urls")
-
-		markStringRequired(config.NodeName, "node")
-		markStringRequired(config.BindAddr, "bind")
-		markSliceStringRequired(config.StartJoin, "join")
-		markSliceStringRequired(config.AlertsUrls, "alertsUrls")
-
-		return config
-	}
-
-	var kind string
-	re := regexp.MustCompile("^monitor$|^auditor$|^publisher$")
-	for _, arg := range args {
-		if re.MatchString(arg) {
-			kind = arg
-			break
-		}
-	}
+var agentCmd *cobra.Command = &cobra.Command{
+	Use:   "agent",
+	Short: "Provides access to the QED gossip agents",
+	Long: `QED provides standalone agents to help maintain QED security. We have included
+three agents into the distribution:
+	* Monitor agent: checks the lag of the system between the QED Log and the
+	  Snapshot Store as seen by the gossip network
+	* Auditor agent: verifies QED membership proofs of the snapshots received
+	  throught the  gossip network
+	* Publisher agent: publish snapshots to the snapshot store`,
+	TraverseChildren: true,
+}
 
-	switch kind {
-	case "publisher":
-		cmd.AddCommand(newAgentPublisherCommand(cmdCtx, *config, agentPreRun))
+var agentCtx context.Context = configAgent()
 
-	case "auditor":
-		cmd.AddCommand(newAgentAuditorCommand(cmdCtx, *config, agentPreRun))
+func init() {
+	agentCmd.MarkFlagRequired("bind-addr")
+	agentCmd.MarkFlagRequired("metrics-addr")
+	agentCmd.MarkFlagRequired("node-name")
+	agentCmd.MarkFlagRequired("role")
+	agentCmd.MarkFlagRequired("log")
+	Root.AddCommand(agentCmd)
+}
 
-	case "monitor":
-		cmd.AddCommand(newAgentMonitorCommand(cmdCtx, *config, agentPreRun))
+func configAgent() context.Context {
+	conf := gossip.DefaultConfig()
+	err := gpflag.ParseTo(conf, agentCmd.PersistentFlags())
+	if err != nil {
+		log.Fatalf("err: %v", err)
 	}
 
-	return cmd
-
+	return context.WithValue(Ctx, k("agent.config"), conf)
 }
+