From 1f3bc4a3fc6fccaef4a21e2e1dc2007d0d6a1caa Mon Sep 17 00:00:00 2001 From: Javier Marcos Date: Fri, 30 Aug 2019 09:39:46 -0700 Subject: [PATCH 1/2] Fixed issue with logging plugins --- cmd/admin/handlers-get.go | 4 +- cmd/admin/handlers-post.go | 21 +++++++- cmd/admin/templates/environments.html | 4 +- cmd/tls/handlers-tls.go | 27 +++++----- cmd/tls/main.go | 2 +- pkg/environments/environments.go | 2 +- pkg/environments/flags.go | 17 +++--- plugins/logging_dispatcher/control.go | 10 ++++ plugins/logging_dispatcher/db.go | 5 ++ plugins/logging_dispatcher/logging.go | 77 ++++++++++++++++++--------- 10 files changed, 116 insertions(+), 53 deletions(-) create mode 100644 plugins/logging_dispatcher/control.go diff --git a/cmd/admin/handlers-get.go b/cmd/admin/handlers-get.go index a6373a81..5383dd1b 100644 --- a/cmd/admin/handlers-get.go +++ b/cmd/admin/handlers-get.go @@ -139,7 +139,7 @@ func environmentHandler(w http.ResponseWriter, r *http.Request) { return } if settingsmgr.DebugService(settings.ServiceAdmin) { - log.Println("DebugService: Environment table template served") + log.Println("DebugService: Environment table template served") } incMetric(metricAdminOK) } @@ -963,7 +963,7 @@ func envsGETHandler(w http.ResponseWriter, r *http.Request) { return } if settingsmgr.DebugService(settings.ServiceAdmin) { - log.Println("DebugService: Environments template served") + log.Println("DebugService: Environments template served") } incMetric(metricAdminOK) } diff --git a/cmd/admin/handlers-post.go b/cmd/admin/handlers-post.go index 9361a8c6..a5ee7d58 100644 --- a/cmd/admin/handlers-post.go +++ b/cmd/admin/handlers-post.go @@ -933,6 +933,7 @@ func envsPOSTHandler(w http.ResponseWriter, r *http.Request) { if settingsmgr.DebugService(settings.ServiceAdmin) { log.Printf("DebugService: %s %v", responseMessage, err) } + goto response } else { // Check CSRF Token if checkCSRFToken(ctx["csrftoken"], c.CSRFToken) { @@ -946,6 +947,19 @@ func envsPOSTHandler(w http.ResponseWriter, r *http.Request) { if env.Configuration == "" { env.Configuration = environments.ReadExternalFile(emptyConfiguration) } + if env.Flags == "" { + // Generate flags + flags, err := environments.GenerateFlags(env, "", "") + if err != nil { + responseMessage = "error creating environment" + responseCode = http.StatusInternalServerError + if settingsmgr.DebugService(settings.ServiceAdmin) { + log.Printf("DebugService: %s %v", responseMessage, err) + } + goto response + } + env.Flags = flags + } err := envs.Create(env) if err != nil { responseMessage = "error creating environment" @@ -953,6 +967,7 @@ func envsPOSTHandler(w http.ResponseWriter, r *http.Request) { if settingsmgr.DebugService(settings.ServiceAdmin) { log.Printf("DebugService: %s %v", responseMessage, err) } + goto response } else { responseMessage = "Environment created successfully" } @@ -969,6 +984,7 @@ func envsPOSTHandler(w http.ResponseWriter, r *http.Request) { if settingsmgr.DebugService(settings.ServiceAdmin) { log.Printf("DebugService: %s %v", responseMessage, err) } + goto response } else { responseMessage = "Environment deleted successfully" } @@ -983,6 +999,7 @@ func envsPOSTHandler(w http.ResponseWriter, r *http.Request) { if settingsmgr.DebugService(settings.ServiceAdmin) { log.Printf("DebugService: %s %v", responseMessage, err) } + goto response } else { responseMessage = "DebugHTTP changed successfully" } @@ -994,8 +1011,10 @@ func envsPOSTHandler(w http.ResponseWriter, r *http.Request) { if settingsmgr.DebugService(settings.ServiceAdmin) { log.Printf("DebugService: %s %v", responseMessage, err) } + goto response } } +response: // Prepare response response, err := json.Marshal(AdminResponse{Message: responseMessage}) if err != nil { @@ -1011,7 +1030,7 @@ func envsPOSTHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(responseCode) _, _ = w.Write(response) if settingsmgr.DebugService(settings.ServiceAdmin) { - log.Println("DebugService: Environments response sent") + log.Println("DebugService: Environments response sent") } } diff --git a/cmd/admin/templates/environments.html b/cmd/admin/templates/environments.html index f657f28a..92dfd9b4 100644 --- a/cmd/admin/templates/environments.html +++ b/cmd/admin/templates/environments.html @@ -20,13 +20,13 @@
- All TLS Environments + All TLS Environments
diff --git a/cmd/tls/handlers-tls.go b/cmd/tls/handlers-tls.go index 4ccbc106..164607f4 100644 --- a/cmd/tls/handlers-tls.go +++ b/cmd/tls/handlers-tls.go @@ -340,8 +340,15 @@ func processLogs(data json.RawMessage, logType, environment, ipaddress string) { // Helper to dispatch logs func dispatchLogs(data []byte, uuid, ipaddress, user, osqueryuser, hostname, localname, hash, dhash, osqueryversion, logType, environment string) { + // Use metadata to update record + if err := nodesmgr.UpdateMetadataByUUID(user, osqueryuser, hostname, localname, ipaddress, hash, dhash, osqueryversion, uuid); err != nil { + log.Printf("error updating metadata %s", err) + } // Send data to storage // FIXME allow multiple types of logging + if envsmap[environment].DebugHTTP { + log.Printf("dispatching logs to %s", tlsConfig.Logging) + } logsDispatcher( tlsConfig.Logging, logType, @@ -350,11 +357,6 @@ func dispatchLogs(data []byte, uuid, ipaddress, user, osqueryuser, hostname, loc environment, uuid, envsmap[environment].DebugHTTP) - // Use metadata to update record - err := nodesmgr.UpdateMetadataByUUID(user, osqueryuser, hostname, localname, ipaddress, hash, dhash, osqueryversion, uuid) - if err != nil { - log.Printf("error updating metadata %s", err) - } // Refresh last logging request if logType == types.StatusLog { err := nodesmgr.RefreshLastStatus(uuid) @@ -363,8 +365,7 @@ func dispatchLogs(data []byte, uuid, ipaddress, user, osqueryuser, hostname, loc } } if logType == types.ResultLog { - err := nodesmgr.RefreshLastResult(uuid) - if err != nil { + if err := nodesmgr.RefreshLastResult(uuid); err != nil { log.Printf("error refreshing last result %v", err) } } @@ -377,8 +378,15 @@ func dispatchQueries(queryData types.QueryWriteData, node nodes.OsqueryNode) { if err != nil { log.Printf("error preparing data %v", err) } + // Refresh last query write request + if err := nodesmgr.RefreshLastQueryWrite(node.UUID); err != nil { + log.Printf("error refreshing last query write %v", err) + } // Send data to storage // FIXME allow multiple types of logging + if envsmap[node.Environment].DebugHTTP { + log.Printf("dispatching queries to %s", tlsConfig.Logging) + } logsDispatcher( tlsConfig.Logging, types.QueryLog, @@ -389,11 +397,6 @@ func dispatchQueries(queryData types.QueryWriteData, node nodes.OsqueryNode) { queryData.Name, queryData.Status, envsmap[node.Environment].DebugHTTP) - // Refresh last query write request - err = nodesmgr.RefreshLastQueryWrite(node.UUID) - if err != nil { - log.Printf("error refreshing last query write %v", err) - } } // Function to handle on-demand queries to osquery nodes diff --git a/cmd/tls/main.go b/cmd/tls/main.go index ed64e46b..accd165f 100644 --- a/cmd/tls/main.go +++ b/cmd/tls/main.go @@ -187,7 +187,7 @@ func main() { // FIXME Implement Redis cache // FIXME splay this? if settingsmgr.DebugService(settings.ServiceTLS) { - log.Println("DebugService: Environments ticker") + log.Println("DebugService: Environments ticker") } // Refresh environments as soon as service starts go refreshEnvironments() diff --git a/pkg/environments/environments.go b/pkg/environments/environments.go index bd2980d3..8ff1b932 100644 --- a/pkg/environments/environments.go +++ b/pkg/environments/environments.go @@ -74,7 +74,7 @@ type TLSEnvironment struct { // MapEnvironments to hold the TLS environments by name type MapEnvironments map[string]TLSEnvironment -// Environment keeps all TLS Environments +// Environment keeps all TLS Environments type Environment struct { DB *gorm.DB } diff --git a/pkg/environments/flags.go b/pkg/environments/flags.go index 320310b8..11a9a8eb 100644 --- a/pkg/environments/flags.go +++ b/pkg/environments/flags.go @@ -17,6 +17,7 @@ const ( --config_plugin=tls --config_tls_endpoint=/{{ .Environment.Name }}/{{ .Environment.ConfigPath }} --config_tls_refresh={{ .Environment.ConfigInterval }} +--config_tls_max_attempts=5 --logger_plugin=tls --logger_tls_compress=true --logger_tls_endpoint=/{{ .Environment.Name }}/{{ .Environment.LogPath }} @@ -28,7 +29,7 @@ const ( --disable_distributed=false --distributed_interval={{ .Environment.QueryInterval }} --distributed_plugin=tls ---distributed_tls_max_attempts=3 +--distributed_tls_max_attempts=5 --distributed_tls_read_endpoint=/{{ .Environment.Name }}/{{ .Environment.QueryReadPath }} --distributed_tls_write_endpoint=/{{ .Environment.Name }}/{{ .Environment.QueryWritePath }} --tls_dump=true @@ -49,17 +50,17 @@ type flagData struct { } // GenerateFlags to generate flags -func GenerateFlags(env TLSEnvironment, secret, certificate string) (string, error) { +func GenerateFlags(env TLSEnvironment, secretPath, certificatePath string) (string, error) { t, err := template.New("flags").Parse(FlagsTemplate) if err != nil { return "", err } - flagSecret := secret - if secret == "" { + flagSecret := secretPath + if secretPath == "" { flagSecret = emptyFlagSecret } - flagCertificate := certificate - if certificate == "" { + flagCertificate := certificatePath + if certificatePath == "" { flagCertificate = emptyFlagCert } data := flagData{ @@ -75,10 +76,10 @@ func GenerateFlags(env TLSEnvironment, secret, certificate string) (string, erro } // GenerateFlagsEnv to generate flags by environment name -func (environment *Environment) GenerateFlagsEnv(name string, secret, certificate string) (string, error) { +func (environment *Environment) GenerateFlagsEnv(name string, secretPath, certificatePath string) (string, error) { env, err := environment.Get(name) if err != nil { return "", fmt.Errorf("error getting environment %v", err) } - return GenerateFlags(env, secret, certificate) + return GenerateFlags(env, secretPath, certificatePath) } diff --git a/plugins/logging_dispatcher/control.go b/plugins/logging_dispatcher/control.go new file mode 100644 index 00000000..c02a8df7 --- /dev/null +++ b/plugins/logging_dispatcher/control.go @@ -0,0 +1,10 @@ +package main + +const ( + // splunkEnabled + splunkEnabled bool = false + // graylogEnabled + graylogEnabled bool = false + // dbEnabled + dbEnabled bool = true +) diff --git a/plugins/logging_dispatcher/db.go b/plugins/logging_dispatcher/db.go index e6b93061..44794a6b 100644 --- a/plugins/logging_dispatcher/db.go +++ b/plugins/logging_dispatcher/db.go @@ -8,6 +8,11 @@ import ( "github.com/jinzhu/gorm" ) +const ( + // Graylog value + dbName string = "DB" +) + var ( dbLog func(string, *gorm.DB, []byte, string, string, bool) dbQuery func(*gorm.DB, []byte, string, string, string, int, bool) diff --git a/plugins/logging_dispatcher/logging.go b/plugins/logging_dispatcher/logging.go index 8b8a61f9..e52fc1b1 100644 --- a/plugins/logging_dispatcher/logging.go +++ b/plugins/logging_dispatcher/logging.go @@ -8,40 +8,53 @@ import ( "github.com/jmpsec/osctrl/pkg/types" ) -const ( - // Graylog enabled - graylogOn bool = false - // Splunk enabled - splunkOn bool = false - // DB enabled - dbOn bool = true -) - // Variables for configuration from JSON files var ( - graylogCfg GraylogConfiguration - splunkCfg SlunkConfiguration + graylogCfg GraylogConfiguration + graylogReady bool + splunkCfg SlunkConfiguration + splunkReady bool + dbReady bool ) // Initialization of the plugin func init() { var err error - if graylogOn { + if graylogEnabled { graylogCfg, err = loadGraylogConfiguration() if err != nil { - log.Fatalf("Failed to load graylog json - %v", err) + graylogReady = false + log.Printf("Failed to load graylog json - %v", err) + } else { + if err := loadGraylogPlugin(); err != nil { + graylogReady = false + log.Printf("Failed to load graylog plugin - %v", err) + } else { + graylogReady = true + } } } - if splunkOn { + if splunkEnabled { splunkCfg, err = loadSplunkConfiguration() if err != nil { - log.Fatalf("Failed to load splunk json - %v", err) + splunkReady = false + log.Printf("Failed to load splunk json - %v", err) + } else { + if err := loadSplunkPlugin(); err != nil { + splunkReady = false + log.Printf("Failed to load splunk plugin - %v", err) + } else { + splunkReady = true + } } } - if dbOn { + if dbEnabled { err = loadDBPlugin() if err != nil { - log.Fatalf("Failed to load db plugin - %v", err) + dbReady = false + log.Printf("Failed to load db plugin - %v", err) + } else { + dbReady = true } } } @@ -55,19 +68,31 @@ func LogsDispatcher(logging, logType string, params ...interface{}) { switch logging { case settings.LoggingGraylog: debug := params[4].(bool) - graylogSend(logType, data, environment, uuid, graylogCfg.URL, debug) + if graylogReady { + graylogSend(logType, data, environment, uuid, graylogCfg.URL, debug) + } else { + log.Printf("Logging with %s isn't ready - Dropping %d bytes", graylogName, len(data)) + } case settings.LoggingSplunk: debug := params[4].(bool) - splunkSend(logType, data, environment, uuid, splunkCfg.URL, splunkCfg.Token, debug) + if splunkReady { + splunkSend(logType, data, environment, uuid, splunkCfg.URL, splunkCfg.Token, debug) + } else { + log.Printf("Logging with %s isn't ready - Dropping %d bytes", splunkName, len(data)) + } case settings.LoggingDB: - if logType == types.QueryLog { - name := params[4].(string) - status := params[5].(int) - debug := params[6].(bool) - dbQuery(db, data, environment, uuid, name, status, debug) + if dbReady { + if logType == types.QueryLog { + name := params[4].(string) + status := params[5].(int) + debug := params[6].(bool) + dbQuery(db, data, environment, uuid, name, status, debug) + } else { + debug := params[4].(bool) + dbLog(logType, db, data, environment, uuid, debug) + } } else { - debug := params[4].(bool) - dbLog(logType, db, data, environment, uuid, debug) + log.Printf("Logging with %s isn't ready - Dropping %d bytes", dbName, len(data)) } } } From 5edc575d16c7df4ab22508bd254d2ec705eb5ae3 Mon Sep 17 00:00:00 2001 From: Javier Marcos Date: Fri, 30 Aug 2019 13:37:13 -0700 Subject: [PATCH 2/2] Removing tls_dump from flags --- pkg/environments/flags.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/environments/flags.go b/pkg/environments/flags.go index 11a9a8eb..fda2574f 100644 --- a/pkg/environments/flags.go +++ b/pkg/environments/flags.go @@ -32,7 +32,6 @@ const ( --distributed_tls_max_attempts=5 --distributed_tls_read_endpoint=/{{ .Environment.Name }}/{{ .Environment.QueryReadPath }} --distributed_tls_write_endpoint=/{{ .Environment.Name }}/{{ .Environment.QueryWritePath }} ---tls_dump=true --tls_hostname={{ .Environment.Hostname }} --tls_server_certs={{ .CertFile }} `