WIP: Make revad start on a per directory basis (#416)

* make revad start on a per directory basis

* rename dir flag to dev-dir

cmd/revad/internal/grace/grace.go

@@ -35,7 +35,7 @@ import (
 )
 
 // Watcher watches a process for a graceful restart
-// preserving open network sockets to avoid packets.
+// preserving open network sockets to avoid packet loss.
 type Watcher struct {
 	log       zerolog.Logger
 	graceful  bool

cmd/revad/main.go

@@ -22,8 +22,13 @@ import (
 	"flag"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"log"
+	"net"
 	"os"
+	"os/signal"
 	"path"
+	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
@@ -51,8 +56,8 @@ var (
 	signalFlag  = flag.String("s", "", "send signal to a master process: stop, quit, reload")
 	configFlag  = flag.String("c", "/etc/revad/revad.toml", "set configuration file")
 	pidFlag     = flag.String("p", "", "pid file. If empty defaults to a random file in the OS temporary directory")
-
-	// Compile time variables initialez with gcc flags.
+	dirFlag     = flag.String("dev-dir", "", "runs any toml file in the specified directory. Intended for development use only")
+	// Compile time variables initialized with gcc flags.
 	gitCommit, buildDate, version, goVersion string
 )
 
@@ -67,6 +72,7 @@ type coreConf struct {
 func main() {
 	flag.Parse()
 
+	handleDirFlag()
 	handleVersionFlag()
 	handleSignalFlag()
 	handleTestFlag()
@@ -75,25 +81,45 @@ func main() {
 	coreConf := parseCoreConfOrDie(mainConf["core"])
 	logConf := parseLogConfOrDie(mainConf["log"])
 
-	log, err := newLogger(logConf)
+	run(mainConf, coreConf, logConf, "")
+}
+
+func run(mainConf map[string]interface{}, coreConf *coreConf, logConf *logConf, filename string) {
+	logger := initLogger(logConf)
+
+	initTracing(coreConf, logger)
+	initCPUCount(coreConf, logger)
+
+	servers := initServers(mainConf, logger)
+	watcher, err := initWatcher(logger, filename)
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "error creating logger, exiting ...")
-		os.Exit(1)
+		log.Panic(err)
 	}
+	listeners := initListeners(watcher, servers, logger)
 
-	if err := setupOpenCensus(coreConf); err != nil {
-		log.Error().Err(err).Msg("error configuring open census stats and tracing")
-		os.Exit(1)
+	start(mainConf, servers, listeners, logger, watcher)
+}
+
+func initListeners(watcher *grace.Watcher, servers map[string]grace.Server, log *zerolog.Logger) map[string]net.Listener {
+	listeners, err := watcher.GetListeners(servers)
+	if err != nil {
+		log.Error().Err(err).Msg("error getting sockets")
+		watcher.Exit(1)
 	}
+	return listeners
+}
 
-	ncpus, err := adjustCPU(coreConf.MaxCPUs)
+func initWatcher(log *zerolog.Logger, filename string) (*grace.Watcher, error) {
+	watcher, err := handlePIDFlag(log, filename)
+	// TODO(labkode): maybe pidfile can be created later on? like once a server is going to be created?
 	if err != nil {
-		log.Error().Err(err).Msg("error adjusting number of cpus")
+		log.Error().Err(err).Msg("error creating grace watcher")
 		os.Exit(1)
 	}
-	log.Info().Msgf("%s", getVersionString())
-	log.Info().Msgf("running on %d cpus", ncpus)
+	return watcher, err
+}
 
+func initServers(mainConf map[string]interface{}, log *zerolog.Logger) map[string]grace.Server {
 	servers := map[string]grace.Server{}
 	if isEnabledHTTP(mainConf) {
 		s, err := getHTTPServer(mainConf["http"], log)
@@ -118,19 +144,36 @@ func main() {
 		log.Info().Msg("nothing to do, no grpc/http enabled_services declared in config")
 		os.Exit(1)
 	}
+	return servers
+}
+
+func initTracing(conf *coreConf, log *zerolog.Logger) {
+	if err := setupOpenCensus(conf); err != nil {
+		log.Error().Err(err).Msg("error configuring open census stats and tracing")
+		os.Exit(1)
+	}
+}
 
-	watcher, err := handlePIDFlag(log) // TODO(labkode): maybe pidfile can be created later on? like once a server is going to be created?
+func initCPUCount(conf *coreConf, log *zerolog.Logger) {
+	ncpus, err := adjustCPU(conf.MaxCPUs)
 	if err != nil {
-		log.Error().Err(err).Msg("error creating grace watcher")
+		log.Error().Err(err).Msg("error adjusting number of cpus")
 		os.Exit(1)
 	}
+	log.Info().Msgf("%s", getVersionString())
+	log.Info().Msgf("running on %d cpus", ncpus)
+}
 
-	listeners, err := watcher.GetListeners(servers)
+func initLogger(conf *logConf) *zerolog.Logger {
+	log, err := newLogger(conf)
 	if err != nil {
-		log.Error().Err(err).Msg("error getting sockets")
-		watcher.Exit(1)
+		fmt.Fprintf(os.Stderr, "error creating logger, exiting ...")
+		os.Exit(1)
 	}
+	return log
+}
 
+func start(mainConf map[string]interface{}, servers map[string]grace.Server, listeners map[string]net.Listener, log *zerolog.Logger, watcher *grace.Watcher) {
 	if isEnabledHTTP(mainConf) {
 		go func() {
 			if err := servers["http"].(*rhttp.Server).Start(listeners["http"]); err != nil {
@@ -139,7 +182,6 @@ func main() {
 			}
 		}()
 	}
-
 	if isEnabledGRPC(mainConf) {
 		go func() {
 			if err := servers["grpc"].(*rgrpc.Server).Start(listeners["grpc"]); err != nil {
@@ -148,9 +190,10 @@ func main() {
 			}
 		}()
 	}
-
-	// wait for signal to close servers
-	watcher.TrapSignals()
+	// wait for signal to close servers when running on single process mode
+	if *dirFlag == "" {
+		watcher.TrapSignals()
+	}
 }
 
 func newLogger(conf *logConf) (*zerolog.Logger, error) {
@@ -196,6 +239,48 @@ func getVersionString() string {
 	return fmt.Sprintf(msg, version, gitCommit, goVersion, buildDate)
 }
 
+func handleDirFlag() {
+	var configFiles []string
+	if *dirFlag != "" {
+		files, err := ioutil.ReadDir(*dirFlag)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		for _, value := range files {
+			if !value.IsDir() {
+				expr := regexp.MustCompile(`[\w].toml`)
+
+				if expr.Match([]byte(value.Name())) {
+					configFiles = append(configFiles, value.Name())
+				}
+			}
+		}
+
+		stop := make(chan os.Signal, 1)
+		defer close(stop)
+
+		for _, file := range configFiles {
+			f := file
+			mainConf := parseConfigFlagOrDie(f)
+			coreConf := parseCoreConfOrDie(mainConf["core"])
+			logConf := parseLogConfOrDie(mainConf["log"])
+
+			go run(mainConf, coreConf, logConf, f+".pid")
+		}
+
+		signal.Notify(stop, os.Interrupt)
+		for range stop {
+			for i := 0; i < len(configFiles); i++ {
+				fname := configFiles[i] + ".pid"
+				fmt.Printf("removing pid file: %v\n", fname)
+				os.Remove(fname)
+			}
+			os.Exit(0)
+		}
+	}
+}
+
 func handleVersionFlag() {
 	if *versionFlag {
 		fmt.Fprintf(os.Stderr, "%s\n", getVersionString())
@@ -245,9 +330,12 @@ func handleTestFlag() {
 	}
 }
 
-func handlePIDFlag(l *zerolog.Logger) (*grace.Watcher, error) {
-	// if pid is empty, we store it in the OS temporary folder with random name
-	if *pidFlag == "" {
+func handlePIDFlag(l *zerolog.Logger, filename string) (*grace.Watcher, error) {
+	// if a filename is provided use this instead
+	if filename != "" {
+		*pidFlag = filename
+	} else if *pidFlag == "" {
+		// if pid is empty, we store it in the OS temporary folder with random name
 		uuid := uuid.Must(uuid.NewV4())
 		*pidFlag = path.Join(os.TempDir(), "revad-"+uuid.String()+".pid")
 	}
@@ -301,6 +389,28 @@ func handleConfigFlagOrDie() map[string]interface{} {
 	return v
 }
 
+func parseConfigFlagOrDie(dst string) map[string]interface{} {
+	var path string
+	if *dirFlag != "" && *dirFlag != "." {
+		path = *dirFlag
+	}
+
+	fd, err := os.Open(path + dst)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error opening file: %+s\n", err.Error())
+		os.Exit(1)
+	}
+	defer fd.Close()
+
+	v, err := config.Read(fd)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "error reading config: %s\n", err.Error())
+		os.Exit(1)
+	}
+
+	return v
+}
+
 func setupOpenCensus(conf *coreConf) error {
 	if err := view.Register(ochttp.DefaultServerViews...); err != nil {
 		return err

examples/separate/users.toml

@@ -40,13 +40,13 @@ iframe_ui_provider = "http://localhost:19500/iframeui"
 auth_manager = "json"
 
 [grpc.services.authprovider.auth_managers.json]
-users = "./examples/separate/users.demo.json"
+users = "users.demo.json"
 
 [grpc.services.userprovider]
 driver = "json"
 
 [grpc.services.userprovider.drivers.json]
-users = "./examples/separate/users.demo.json"
+users = "users.demo.json"
 
 
 # TODO bring back iframe app ui demo

go.mod

@@ -12,7 +12,6 @@ require (
 	github.com/fatih/color v1.7.0 // indirect
 	github.com/go-openapi/strfmt v0.19.2 // indirect
 	github.com/gofrs/uuid v3.2.0+incompatible
-	github.com/gogo/protobuf v1.2.0 // indirect
 	github.com/golang/protobuf v1.3.2
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4
@@ -25,20 +24,17 @@ require (
 	github.com/pkg/errors v0.8.1
 	github.com/pkg/xattr v0.4.1
 	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
-	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 // indirect
 	github.com/rs/cors v1.7.0
 	github.com/rs/zerolog v1.17.2
 	go.opencensus.io v0.22.1
 	golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a
-	golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 // indirect
 	golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 // indirect
 	golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421
 	google.golang.org/grpc v1.25.1
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
 	gopkg.in/cheggaaa/pb.v1 v1.0.27 // indirect
 	gopkg.in/ldap.v2 v2.5.1
 	gopkg.in/square/go-jose.v2 v2.2.2 // indirect
-	honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc // indirect
 )
 
 go 1.13