Merge pull request #8 from jmpsec/queries-logging-fix

Fix for on-demand queries logging in splunk/graylog
jmpsec · Sep 3, 2019 · 9cd21fd · 9cd21fd
2 parents f69c109 + b89f732
commit 9cd21fd
Show file tree

Hide file tree

Showing 9 changed files with 48 additions and 50 deletions.
diff --git a/cmd/admin/json-queries.go b/cmd/admin/json-queries.go
@@ -71,6 +71,7 @@ func jsonQueryHandler(w http.ResponseWriter, r *http.Request) {
 			status = queries.StatusComplete
 		}
 		progress := make(QueryProgress)
+		progress["expected"] = q.Expected
 		progress["executions"] = q.Executions
 		progress["errors"] = q.Errors
 		data := make(QueryData)

diff --git a/cmd/admin/settings.go b/cmd/admin/settings.go
@@ -44,7 +44,7 @@ func loadingSettings() {
 		if err != nil {
 			log.Fatalf("Failed to initialize metrics (port): %v", err)
 		}
-		_metrics, err = metrics.CreateMetrics(mProtocol, mHost, int(mPort), settings.ServiceAdmin)
+		_metrics, err = metrics.CreateMetrics(mProtocol, mHost, int(mPort), serviceName)
 		if err != nil {
 			log.Fatalf("Failed to initialize metrics: %v", err)
 		}

diff --git a/cmd/admin/templates/queries.html b/cmd/admin/templates/queries.html
@@ -147,8 +147,9 @@
               data: 'progress',
               render: function (data, type, row, meta) {
                 if (type === 'display') {
-                  return  '<span style="color:green;">'+data.executions+'</span>/' +
-                          '<span style="color:red;">'+data.errors+'</span>';
+                  return  '<span style="color:black;">'+data.expected+'</span>/' +
+                          '<b><span style="color:green;">'+data.executions+'</span></b>/' +
+                          '<b><span style="color:red;">'+data.errors+'</span></b>';
                 } else {
                   return data;
                 }

diff --git a/cmd/tls/settings.go b/cmd/tls/settings.go
@@ -34,7 +34,7 @@ func loadingSettings() {
 		if err != nil {
 			log.Fatalf("Failed to initialize metrics (port): %v", err)
 		}
-		_metrics, err = metrics.CreateMetrics(mProtocol, mHost, int(mPort), settings.ServiceTLS)
+		_metrics, err = metrics.CreateMetrics(mProtocol, mHost, int(mPort), serviceName)
 		if err != nil {
 			log.Fatalf("Failed to initialize metrics: %v", err)
 		}

diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go
@@ -7,12 +7,13 @@ import (
 	"log"
 	"net/http"
 	"net/http/httputil"
+	"strings"
 )
 
 // SendRequest - Helper function to send HTTP requests
-func SendRequest(secure bool, reqType, url string, params io.Reader, headers map[string]string) (int, []byte, error) {
+func SendRequest(reqType, url string, params io.Reader, headers map[string]string) (int, []byte, error) {
 	var client *http.Client
-	if secure {
+	if strings.HasPrefix(url, "https") {
 		tr := &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		}

diff --git a/plugins/graylog_logging/plugin.go b/plugins/graylog_logging/plugin.go
@@ -74,7 +74,7 @@ func GraylogSend(logType string, data []byte, environment, uuid, url string, deb
 		log.Printf("Sending %d bytes to Graylog for %s - %s", len(data), environment, uuid)
 	}
 	// Send log with a POST to the Graylog URL
-	resp, body, err := utils.SendRequest(true, graylogMethod, url, jsonParam, headers)
+	resp, body, err := utils.SendRequest(graylogMethod, url, jsonParam, headers)
 	if err != nil {
 		log.Printf("Error sending request %s", err)
 		return

diff --git a/plugins/logging_dispatcher/control.go b/plugins/logging_dispatcher/control.go
diff --git a/plugins/logging_dispatcher/logging.go b/plugins/logging_dispatcher/logging.go
@@ -20,42 +20,37 @@ var (
 // Initialization of the plugin
 func init() {
 	var err error
-	if graylogEnabled {
-		graylogCfg, err = loadGraylogConfiguration()
-		if err != nil {
+	// Check if Graylog is ready to load
+	graylogCfg, err = loadGraylogConfiguration()
+	if err != nil {
+		graylogReady = false
+	} else {
+		if err := loadGraylogPlugin(); err != nil {
 			graylogReady = false
-			log.Printf("Failed to load graylog json - %v", err)
+			log.Printf("Failed to load graylog plugin - %v", err)
 		} else {
-			if err := loadGraylogPlugin(); err != nil {
-				graylogReady = false
-				log.Printf("Failed to load graylog plugin - %v", err)
-			} else {
-				graylogReady = true
-			}
+			graylogReady = true
 		}
 	}
-	if splunkEnabled {
-		splunkCfg, err = loadSplunkConfiguration()
-		if err != nil {
+	// Check if Splunk is ready to load
+	splunkCfg, err = loadSplunkConfiguration()
+	if err != nil {
+		splunkReady = false
+	} else {
+		if err := loadSplunkPlugin(); err != nil {
 			splunkReady = false
-			log.Printf("Failed to load splunk json - %v", err)
+			log.Printf("Failed to load splunk plugin - %v", err)
 		} else {
-			if err := loadSplunkPlugin(); err != nil {
-				splunkReady = false
-				log.Printf("Failed to load splunk plugin - %v", err)
-			} else {
-				splunkReady = true
-			}
+			splunkReady = true
 		}
 	}
-	if dbEnabled {
-		err = loadDBPlugin()
-		if err != nil {
-			dbReady = false
-			log.Printf("Failed to load db plugin - %v", err)
-		} else {
-			dbReady = true
-		}
+	// Loading DB plugin regardless
+	err = loadDBPlugin()
+	if err != nil {
+		dbReady = false
+		log.Printf("Failed to load db plugin - %v", err)
+	} else {
+		dbReady = true
 	}
 }
 
@@ -67,18 +62,28 @@ func LogsDispatcher(logging, logType string, params ...interface{}) {
 	uuid := params[3].(string)
 	switch logging {
 	case settings.LoggingGraylog:
-		debug := params[4].(bool)
 		if graylogReady {
+			var debug bool
+			if logType == types.QueryLog {
+				debug = params[6].(bool)
+			} else {
+				debug = params[4].(bool)
+			}
 			graylogSend(logType, data, environment, uuid, graylogCfg.URL, debug)
 		} else {
-			log.Printf("Logging with %s isn't ready - Dropping %d bytes", graylogName, len(data))
+			log.Printf("Logging with %s isn't ready [%s] - Dropping %d bytes", graylogName, graylogCfg.URL, len(data))
 		}
 	case settings.LoggingSplunk:
-		debug := params[4].(bool)
 		if splunkReady {
+			var debug bool
+			if logType == types.QueryLog {
+				debug = params[6].(bool)
+			} else {
+				debug = params[4].(bool)
+			}
 			splunkSend(logType, data, environment, uuid, splunkCfg.URL, splunkCfg.Token, debug)
 		} else {
-			log.Printf("Logging with %s isn't ready - Dropping %d bytes", splunkName, len(data))
+			log.Printf("Logging with %s isn't ready [%s] - Dropping %d bytes", splunkName, splunkCfg.URL, len(data))
 		}
 	case settings.LoggingDB:
 		if dbReady {

diff --git a/plugins/splunk_logging/plugin.go b/plugins/splunk_logging/plugin.go
@@ -84,7 +84,7 @@ func SplunkSend(logType string, data []byte, environment, uuid, url, token strin
 		log.Printf("Sending %d bytes to Splunk for %s - %s", len(data), environment, uuid)
 	}
 	// Send log with a POST to the Splunk URL
-	resp, body, err := utils.SendRequest(true, splunkMethod, url, jsonParam, headers)
+	resp, body, err := utils.SendRequest(splunkMethod, url, jsonParam, headers)
 	if err != nil {
 		log.Printf("Error sending request %s", err)
 	}