From 654ce21532b801b6c98b6107b1c1746fc43742d0 Mon Sep 17 00:00:00 2001 From: Javier Marcos <1271349+javuto@users.noreply.github.com> Date: Thu, 26 Sep 2024 12:58:37 +0200 Subject: [PATCH] Migrating native log to zerolog in osctrl-cli and other packages --- carves/go.mod | 6 ++++++ carves/go.sum | 13 +++++++++++++ carves/s3.go | 18 +++++++++--------- cli/api-carve.go | 4 ++-- cli/api-node.go | 3 +-- cli/api-query.go | 6 +++--- cli/api.go | 11 +++-------- logging/db.go | 22 +++++++++++----------- logging/dispatch.go | 16 ++++++++-------- logging/file.go | 7 +++---- logging/graylog.go | 22 +++++++++++----------- logging/kinesis.go | 12 ++++++------ logging/logging.go | 35 +++++++++++++++++------------------ logging/logstash.go | 34 +++++++++++++++++----------------- logging/none.go | 13 ++++++------- logging/process.go | 20 ++++++++++---------- logging/s3.go | 12 ++++++------ logging/splunk.go | 22 +++++++++++----------- logging/stdout.go | 11 +++++------ metrics/go.mod | 3 +++ metrics/go.sum | 14 ++++++++++++++ metrics/metrics.go | 10 +++++----- 22 files changed, 170 insertions(+), 144 deletions(-) diff --git a/carves/go.mod b/carves/go.mod index 56bcdeab..099dc17f 100644 --- a/carves/go.mod +++ b/carves/go.mod @@ -31,6 +31,11 @@ require ( gorm.io/gorm v1.25.11 ) +require ( + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect +) + require ( github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.4 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.13 // indirect @@ -55,6 +60,7 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect + github.com/rs/zerolog v1.33.0 github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/segmentio/ksuid v1.0.4 // indirect diff --git a/carves/go.sum b/carves/go.sum index 7d5f6279..f076711f 100644 --- a/carves/go.sum +++ b/carves/go.sum @@ -400,6 +400,7 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20191124224453-732737034ffd/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= @@ -416,6 +417,7 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -502,6 +504,11 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= @@ -520,6 +527,9 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/sagikazarmark/locafero v0.3.0 h1:zT7VEGWC2DTflmccN/5T1etyKvxSxpHsjb9cJvm4SvQ= github.com/sagikazarmark/locafero v0.3.0/go.mod h1:w+v7UsPNFwzF1cHuOajOOzoq4U7v/ig1mpRjqV+Bu1U= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= @@ -725,6 +735,9 @@ golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= diff --git a/carves/s3.go b/carves/s3.go index 48cff25d..222c6de4 100644 --- a/carves/s3.go +++ b/carves/s3.go @@ -6,13 +6,13 @@ import ( "encoding/base64" "fmt" "io" - "log" "net/http" "net/url" "time" "github.com/jmpsec/osctrl/settings" "github.com/jmpsec/osctrl/types" + "github.com/rs/zerolog/log" "github.com/spf13/viper" "github.com/aws/aws-sdk-go-v2/aws" @@ -78,7 +78,7 @@ func CreateCarverS3(s3Config types.S3Configuration) (*CarverS3, error) { // LoadS3 - Function to load the S3 configuration from JSON file func LoadS3(file string) (types.S3Configuration, error) { var _s3Cfg types.S3Configuration - log.Printf("Loading %s", file) + log.Info().Msgf("Loading %s", file) // Load file and read config viper.SetConfigFile(file) if err := viper.ReadInConfig(); err != nil { @@ -94,14 +94,14 @@ func LoadS3(file string) (types.S3Configuration, error) { // Settings - Function to prepare settings for the logger func (carveS3 *CarverS3) Settings(mgr *settings.Settings) { - log.Printf("No s3 logging settings\n") + log.Info().Msg("No s3 logging settings") } // Upload - Function that sends data from carves to S3 func (carveS3 *CarverS3) Upload(block CarvedBlock, uuid, data string) error { ctx := context.Background() if carveS3.Debug { - log.Printf("DebugService: Sending %d bytes to S3 for %s - %s", block.Size, block.Environment, uuid) + log.Debug().Msgf("DebugService: Sending %d bytes to S3 for %s - %s", block.Size, block.Environment, uuid) } // Decode before upload toUpload, err := base64.StdEncoding.DecodeString(data) @@ -120,7 +120,7 @@ func (carveS3 *CarverS3) Upload(block CarvedBlock, uuid, data string) error { return fmt.Errorf("error sending data to s3 - %s", err) } if carveS3.Debug { - log.Printf("DebugService: S3 Upload %+v", uploadOutput) + log.Debug().Msgf("DebugService: S3 Upload %+v", uploadOutput) } return nil } @@ -192,7 +192,7 @@ func (carveS3 *CarverS3) Archive(carve CarvedFile, blocks []CarvedBlock) (*Carve return nil, fmt.Errorf("CompleteMultipartUpload - %s", err) } if carveS3.Debug { - log.Printf("DebugService: S3 Archived %s [%d bytes] - %s", res.File, res.Size, *multiOutput.Key) + log.Debug().Msgf("DebugService: S3 Archived %s [%d bytes] - %s", res.File, res.Size, *multiOutput.Key) } return res, nil } @@ -201,7 +201,7 @@ func (carveS3 *CarverS3) Archive(carve CarvedFile, blocks []CarvedBlock) (*Carve func (carveS3 *CarverS3) Download(carve CarvedFile) (io.WriterAt, error) { ctx := context.Background() if carveS3.Debug { - log.Printf("DebugService: Downloading %s from S3", carve.ArchivePath) + log.Debug().Msgf("DebugService: Downloading %s from S3", carve.ArchivePath) } downloader := manager.NewDownloader(carveS3.Client) var fileReader io.WriterAt @@ -215,7 +215,7 @@ func (carveS3 *CarverS3) Download(carve CarvedFile) (io.WriterAt, error) { return nil, fmt.Errorf("Download - %s", err) } if carveS3.Debug { - log.Printf("DebugService: S3 Downloaded %s [%d bytes]", carve.ArchivePath, downloadedBytes) + log.Debug().Msgf("DebugService: S3 Downloaded %s [%d bytes]", carve.ArchivePath, downloadedBytes) } return fileReader, nil } @@ -224,7 +224,7 @@ func (carveS3 *CarverS3) Download(carve CarvedFile) (io.WriterAt, error) { func (carveS3 *CarverS3) GetDownloadLink(carve CarvedFile) (string, error) { ctx := context.Background() if carveS3.Debug { - log.Printf("DebugService: Downloading link %s from S3", carve.ArchivePath) + log.Debug().Msgf("DebugService: Downloading link %s from S3", carve.ArchivePath) } preClient := s3.NewPresignClient(carveS3.Client) lnk, err := preClient.PresignGetObject(ctx, &s3.GetObjectInput{ diff --git a/cli/api-carve.go b/cli/api-carve.go index ad80d861..515f58aa 100644 --- a/cli/api-carve.go +++ b/cli/api-carve.go @@ -3,11 +3,11 @@ package main import ( "encoding/json" "fmt" - "log" "strings" "github.com/jmpsec/osctrl/carves" "github.com/jmpsec/osctrl/types" + "github.com/rs/zerolog/log" ) // GetCarves to retrieve carves from osctrl @@ -58,7 +58,7 @@ func (api *OsctrlAPI) RunCarve(env, uuid, path string) (types.ApiQueriesResponse reqURL := fmt.Sprintf("%s%s%s/%s", api.Configuration.URL, APIPath, APICarves, env) jsonMessage, err := json.Marshal(c) if err != nil { - log.Printf("error marshaling data %s", err) + log.Err(err).Msg("error marshaling data") } jsonParam := strings.NewReader(string(jsonMessage)) rawC, err := api.PostGeneric(reqURL, jsonParam) diff --git a/cli/api-node.go b/cli/api-node.go index 624fdeab..53e4be9f 100644 --- a/cli/api-node.go +++ b/cli/api-node.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "fmt" - "log" "strings" "github.com/jmpsec/osctrl/nodes" @@ -47,7 +46,7 @@ func (api *OsctrlAPI) DeleteNode(env, identifier string) error { reqURL := fmt.Sprintf("%s%s%s/%s/delete", api.Configuration.URL, APIPath, APINodes, env) jsonMessage, err := json.Marshal(n) if err != nil { - log.Printf("error marshaling data %s", err) + return fmt.Errorf("error marshaling data - %v", err) } jsonParam := strings.NewReader(string(jsonMessage)) rawN, err := api.PostGeneric(reqURL, jsonParam) diff --git a/cli/api-query.go b/cli/api-query.go index f8ff4800..765e095a 100644 --- a/cli/api-query.go +++ b/cli/api-query.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "fmt" - "log" "strings" "github.com/jmpsec/osctrl/queries" @@ -51,7 +50,7 @@ func (api *OsctrlAPI) CompleteQuery(env, identifier string) error { // RunQuery to initiate a query in osctrl func (api *OsctrlAPI) RunQuery(env, uuid, query string, hidden bool) (types.ApiQueriesResponse, error) { q := types.ApiDistributedQueryRequest{ - UUIDs: []string{uuid}, + UUIDs: []string{uuid}, Query: query, Hidden: hidden, } @@ -59,7 +58,8 @@ func (api *OsctrlAPI) RunQuery(env, uuid, query string, hidden bool) (types.ApiQ reqURL := fmt.Sprintf("%s%s%s/%s", api.Configuration.URL, APIPath, APIQueries, env) jsonMessage, err := json.Marshal(q) if err != nil { - log.Printf("error marshaling data %s", err) + return r, fmt.Errorf("error marshaling data - %v", err) + } jsonParam := strings.NewReader(string(jsonMessage)) rawQ, err := api.PostGeneric(reqURL, jsonParam) diff --git a/cli/api.go b/cli/api.go index 8b9811a1..7e70b37b 100644 --- a/cli/api.go +++ b/cli/api.go @@ -6,10 +6,10 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "log" "net/http" "net/url" + "os" "github.com/jmpsec/osctrl/version" "github.com/spf13/viper" @@ -88,7 +88,7 @@ func writeAPIConfiguration(file string, apiConf JSONConfigurationAPI) error { if err != nil { return fmt.Errorf("error serializing data %s", err) } - if err := ioutil.WriteFile(file, confByte, 0644); err != nil { + if err := os.WriteFile(file, confByte, 0644); err != nil { return fmt.Errorf("error writing to file %s", err) } return nil @@ -154,12 +154,7 @@ func (api *OsctrlAPI) ReqGeneric(reqType string, url string, body io.Reader) ([] if err != nil { return []byte{}, fmt.Errorf("Client.Do - %v", err) } - //defer resp.Body.Close() - defer func() { - if err := resp.Body.Close(); err != nil { - log.Printf("failed to close body %v", err) - } - }() + defer resp.Body.Close() // Read body bodyBytes, err := io.ReadAll(resp.Body) if err != nil { diff --git a/logging/db.go b/logging/db.go index 388e24cc..ae483e5f 100644 --- a/logging/db.go +++ b/logging/db.go @@ -3,7 +3,6 @@ package logging import ( "encoding/json" "fmt" - "log" "strconv" "strings" "time" @@ -14,6 +13,7 @@ import ( "github.com/jmpsec/osctrl/queries" "github.com/jmpsec/osctrl/settings" "github.com/jmpsec/osctrl/types" + "github.com/rs/zerolog/log" ) const ( @@ -89,28 +89,28 @@ func CreateLoggerDB(backend *backend.DBManager) (*LoggerDB, error) { } // table osquery_status_data if err := backend.Conn.AutoMigrate(&OsqueryStatusData{}); err != nil { - log.Fatalf("Failed to AutoMigrate table (osquery_status_data): %v", err) + log.Fatal().Msgf("Failed to AutoMigrate table (osquery_status_data): %v", err) } // table osquery_result_data if err := backend.Conn.AutoMigrate(&OsqueryResultData{}); err != nil { - log.Fatalf("Failed to AutoMigrate table (osquery_result_data): %v", err) + log.Fatal().Msgf("Failed to AutoMigrate table (osquery_result_data): %v", err) } // table osquery_query_data if err := backend.Conn.AutoMigrate(&OsqueryQueryData{}); err != nil { - log.Fatalf("Failed to AutoMigrate table (osquery_query_data): %v", err) + log.Fatal().Msgf("Failed to AutoMigrate table (osquery_query_data): %v", err) } return l, nil } // Settings - Function to prepare settings for the logger func (logDB *LoggerDB) Settings(mgr *settings.Settings) { - //log.Printf("Setting DB logging settings\n") + log.Info().Msg("Setting DB logging settings") } // Log - Function that sends JSON result/status/query logs to the configured DB func (logDB *LoggerDB) Log(logType string, data []byte, environment, uuid string, debug bool) { if debug { - log.Printf("Sending %d bytes to DB for %s - %s", len(data), environment, uuid) + log.Debug().Msgf("Sending %d bytes to DB for %s - %s", len(data), environment, uuid) } switch logType { case types.StatusLog: @@ -125,7 +125,7 @@ func (logDB *LoggerDB) Status(data []byte, environment, uuid string, debug bool) // Parse JSON var logs []types.LogStatusData if err := json.Unmarshal(data, &logs); err != nil { - log.Printf("error parsing logs %s %v", string(data), err) + log.Err(err).Msgf("error parsing logs %s %v", string(data), err) } // Iterate and insert in DB for _, l := range logs { @@ -139,7 +139,7 @@ func (logDB *LoggerDB) Status(data []byte, environment, uuid string, debug bool) Severity: strconv.Itoa(int(l.Severity)), } if err := logDB.Database.Conn.Create(&entry).Error; err != nil { - log.Printf("Error creating status log entry %s", err) + log.Err(err).Msg("Error creating status log entry") } } } @@ -149,7 +149,7 @@ func (logDB *LoggerDB) Result(data []byte, environment, uuid string, debug bool) // Parse JSON var logs []types.LogResultData if err := json.Unmarshal(data, &logs); err != nil { - log.Printf("error parsing logs %s %v", string(data), err) + log.Err(err).Msgf("error parsing logs %s", string(data)) } // Iterate and insert in DB for _, l := range logs { @@ -163,7 +163,7 @@ func (logDB *LoggerDB) Result(data []byte, environment, uuid string, debug bool) Counter: l.Counter, } if err := logDB.Database.Conn.Create(&entry).Error; err != nil { - log.Printf("Error creating result log entry %s", err) + log.Err(err).Msg("Error creating result log entry") } } } @@ -180,7 +180,7 @@ func (logDB *LoggerDB) Query(data []byte, environment, uuid, name string, status } // Insert in DB if err := logDB.Database.Conn.Create(&entry).Error; err != nil { - log.Printf("Error creating query log %s", err) + log.Err(err).Msg("Error creating query log") } } diff --git a/logging/dispatch.go b/logging/dispatch.go index c8c5c53b..5770ef30 100644 --- a/logging/dispatch.go +++ b/logging/dispatch.go @@ -2,35 +2,35 @@ package logging import ( "encoding/json" - "log" "github.com/jmpsec/osctrl/nodes" "github.com/jmpsec/osctrl/types" + "github.com/rs/zerolog/log" ) // DispatchLogs - Helper to dispatch logs func (l *LoggerTLS) DispatchLogs(data []byte, uuid, logType, environment string, metadata nodes.NodeMetadata, debug bool) { // Use metadata to update record if err := l.Nodes.UpdateMetadataByUUID(uuid, metadata); err != nil { - log.Printf("error updating metadata %s", err) + log.Err(err).Msg("error updating metadata") } // Send data to storage // FIXME allow multiple types of logging if debug { - log.Printf("dispatching logs to %s", l.Logging) + log.Debug().Msgf("dispatching logs to %s", l.Logging) } l.Log(logType, data, environment, uuid, debug) // Refresh last logging request if logType == types.StatusLog { // Update metadata for node if err := l.Nodes.RefreshLastStatus(uuid); err != nil { - log.Printf("error refreshing last status %v", err) + log.Err(err).Msg("error refreshing last status") } } if logType == types.ResultLog { // Update metadata for node if err := l.Nodes.RefreshLastResult(uuid); err != nil { - log.Printf("error refreshing last result %v", err) + log.Err(err).Msg("error refreshing last result") } } } @@ -40,16 +40,16 @@ func (l *LoggerTLS) DispatchQueries(queryData types.QueryWriteData, node nodes.O // Prepare data to send data, err := json.Marshal(queryData) if err != nil { - log.Printf("error preparing data %v", err) + log.Err(err).Msg("error preparing data") } // Refresh last query write request if err := l.Nodes.RefreshLastQueryWrite(node.UUID); err != nil { - log.Printf("error refreshing last query write %v", err) + log.Err(err).Msg("error refreshing last query write") } // Send data to storage // FIXME allow multiple types of logging if debug { - log.Printf("dispatching queries to %s", l.Logging) + log.Debug().Msgf("dispatching queries to %s", l.Logging) } l.QueryLog( types.QueryLog, diff --git a/logging/file.go b/logging/file.go index 04b52a85..d851066d 100644 --- a/logging/file.go +++ b/logging/file.go @@ -1,11 +1,10 @@ package logging import ( - "log" - "github.com/jmpsec/osctrl/settings" "github.com/jmpsec/osctrl/types" "github.com/rs/zerolog" + "github.com/rs/zerolog/log" lumberjack "gopkg.in/natefinch/lumberjack.v2" ) @@ -48,13 +47,13 @@ func CreateLoggerFile(filename string, cfg LumberjackConfig) (*LoggerFile, error // Settings - Function to prepare settings for the logger func (logFile *LoggerFile) Settings(mgr *settings.Settings) { - log.Printf("No file logging settings\n") + log.Info().Msg("No file logging settings") } // Log - Function that sends JSON result/status/query logs to stdout func (logFile *LoggerFile) Log(logType string, data []byte, environment, uuid string, debug bool) { if debug { - log.Printf("Sending %d bytes to stdout for %s - %s", len(data), environment, uuid) + log.Debug().Msgf("Sending %d bytes to stdout for %s - %s", len(data), environment, uuid) } switch logType { case types.StatusLog: diff --git a/logging/graylog.go b/logging/graylog.go index ffb14118..a1acc61e 100644 --- a/logging/graylog.go +++ b/logging/graylog.go @@ -2,13 +2,13 @@ package logging import ( "encoding/json" - "log" "strings" "time" "github.com/jmpsec/osctrl/settings" "github.com/jmpsec/osctrl/types" "github.com/jmpsec/osctrl/utils" + "github.com/rs/zerolog/log" "github.com/spf13/viper" ) @@ -24,7 +24,7 @@ type GraylogConfiguration struct { // LoadGraylog - Function to load the Graylog configuration from JSON file func LoadGraylog(file string) (GraylogConfiguration, error) { var _graylogCfg GraylogConfiguration - log.Printf("Loading %s", file) + log.Info().Msgf("Loading %s", file) // Load file and read config viper.SetConfigFile(file) err := viper.ReadInConfig() @@ -86,13 +86,13 @@ type GraylogMessage struct { // Settings - Function to prepare settings for the logger func (logGL *LoggerGraylog) Settings(mgr *settings.Settings) { - log.Printf("No Graylog logging settings\n") + log.Info().Msg("No Graylog logging settings") } // Send - Function that sends JSON logs to Graylog func (logGL *LoggerGraylog) Send(logType string, data []byte, environment, uuid string, debug bool) { if debug { - log.Printf("DebugService: Send %s via graylog", logType) + log.Debug().Msgf("DebugService: Send %s via graylog", logType) } // Convert the array in an array of multiple message var logs []interface{} @@ -101,20 +101,20 @@ func (logGL *LoggerGraylog) Send(logType string, data []byte, environment, uuid var result interface{} err := json.Unmarshal(data, &result) if err != nil { - log.Printf("error parsing data %s %v", string(data), err) + log.Err(err).Msgf("error parsing data %s", string(data)) } logs = append(logs, result) } else { err := json.Unmarshal(data, &logs) if err != nil { - log.Printf("error parsing logs %s %v", string(data), err) + log.Err(err).Msgf("error parsing logs %s", string(data)) } } // Prepare data to send for _, l := range logs { logMessage, err := json.Marshal(l) if err != nil { - log.Printf("error parsing log %s", err) + log.Err(err).Msg("error parsing log") continue } messsageData := GraylogMessage{ @@ -130,20 +130,20 @@ func (logGL *LoggerGraylog) Send(logType string, data []byte, environment, uuid // Serialize data using GELF jsonMessage, err := json.Marshal(messsageData) if err != nil { - log.Printf("error marshaling data %s", err) + log.Err(err).Msg("error marshaling data") } jsonParam := strings.NewReader(string(jsonMessage)) if debug { - log.Printf("DebugService: Sending %d bytes to Graylog for %s - %s", len(data), environment, uuid) + log.Debug().Msgf("DebugService: Sending %d bytes to Graylog for %s - %s", len(data), environment, uuid) } // Send log with a POST to the Graylog URL resp, body, err := utils.SendRequest(GraylogMethod, logGL.Configuration.URL, jsonParam, logGL.Headers) if err != nil { - log.Printf("error sending request %s", err) + log.Err(err).Msg("error sending request") return } if debug { - log.Printf("DebugService: HTTP %d %s", resp, body) + log.Debug().Msgf("DebugService: HTTP %d %s", resp, body) } } } diff --git a/logging/kinesis.go b/logging/kinesis.go index 9eddf9ab..5de8dd11 100644 --- a/logging/kinesis.go +++ b/logging/kinesis.go @@ -2,9 +2,9 @@ package logging import ( "fmt" - "log" "github.com/jmpsec/osctrl/settings" + "github.com/rs/zerolog/log" "github.com/spf13/viper" "github.com/aws/aws-sdk-go/aws" @@ -58,7 +58,7 @@ func CreateLoggerKinesis(kinesisFile string) (*LoggerKinesis, error) { // LoadKinesis - Function to load the Kinesis configuration from JSON file func LoadKinesis(file string) (KinesisConfiguration, error) { var _kinesisCfg KinesisConfiguration - log.Printf("Loading %s", file) + log.Info().Msgf("Loading %s", file) // Load file and read config viper.SetConfigFile(file) if err := viper.ReadInConfig(); err != nil { @@ -74,13 +74,13 @@ func LoadKinesis(file string) (KinesisConfiguration, error) { // Settings - Function to prepare settings for the logger func (logSK *LoggerKinesis) Settings(mgr *settings.Settings) { - log.Printf("No kinesis logging settings\n") + log.Info().Msg("No kinesis logging settings") } // Send - Function that sends JSON logs to Splunk HTTP Event Collector func (logSK *LoggerKinesis) Send(logType string, data []byte, environment, uuid string, debug bool) { if debug { - log.Printf("DebugService: Sending %d bytes to Kinesis for %s - %s", len(data), environment, uuid) + log.Debug().Msgf("DebugService: Sending %d bytes to Kinesis for %s - %s", len(data), environment, uuid) } streamName := aws.String(logSK.Configuration.Stream) putOutput, err := logSK.KinesisClient.PutRecord(&kinesis.PutRecordInput{ @@ -89,9 +89,9 @@ func (logSK *LoggerKinesis) Send(logType string, data []byte, environment, uuid PartitionKey: aws.String(logType + ":" + environment + ":" + uuid), }) if err != nil { - log.Printf("Error sending kinesis stream %s", err) + log.Err(err).Msg("Error sending kinesis stream") } if debug { - log.Printf("DebugService: PutRecordOutput %s", putOutput.String()) + log.Debug().Msgf("DebugService: PutRecordOutput %s", putOutput.String()) } } diff --git a/logging/logging.go b/logging/logging.go index 4964784c..80528156 100644 --- a/logging/logging.go +++ b/logging/logging.go @@ -1,13 +1,12 @@ package logging import ( - "log" - "github.com/jmpsec/osctrl/backend" "github.com/jmpsec/osctrl/nodes" "github.com/jmpsec/osctrl/queries" "github.com/jmpsec/osctrl/settings" "github.com/jmpsec/osctrl/types" + "github.com/rs/zerolog/log" ) const ( @@ -139,7 +138,7 @@ func (logTLS *LoggerTLS) Log(logType string, data []byte, environment, uuid stri case settings.LoggingSplunk: l, ok := logTLS.Logger.(*LoggerSplunk) if !ok { - log.Printf("error casting logger to %s", settings.LoggingSplunk) + log.Error().Msgf("error casting logger to %s", settings.LoggingSplunk) } if l.Enabled { l.Send(logType, data, environment, uuid, debug) @@ -147,7 +146,7 @@ func (logTLS *LoggerTLS) Log(logType string, data []byte, environment, uuid stri case settings.LoggingGraylog: l, ok := logTLS.Logger.(*LoggerGraylog) if !ok { - log.Printf("error casting logger to %s", settings.LoggingGraylog) + log.Error().Msgf("error casting logger to %s", settings.LoggingGraylog) } if l.Enabled { l.Send(logType, data, environment, uuid, debug) @@ -155,7 +154,7 @@ func (logTLS *LoggerTLS) Log(logType string, data []byte, environment, uuid stri case settings.LoggingDB: l, ok := logTLS.Logger.(*LoggerDB) if !ok { - log.Printf("error casting logger to %s", settings.LoggingDB) + log.Error().Msgf("error casting logger to %s", settings.LoggingDB) } if l.Enabled { l.Log(logType, data, environment, uuid, debug) @@ -163,7 +162,7 @@ func (logTLS *LoggerTLS) Log(logType string, data []byte, environment, uuid stri case settings.LoggingStdout: l, ok := logTLS.Logger.(*LoggerStdout) if !ok { - log.Printf("error casting logger to %s", settings.LoggingStdout) + log.Error().Msgf("error casting logger to %s", settings.LoggingStdout) } if l.Enabled { l.Log(logType, data, environment, uuid, debug) @@ -171,7 +170,7 @@ func (logTLS *LoggerTLS) Log(logType string, data []byte, environment, uuid stri case settings.LoggingFile: l, ok := logTLS.Logger.(*LoggerFile) if !ok { - log.Printf("error casting logger to %s", settings.LoggingFile) + log.Error().Msgf("error casting logger to %s", settings.LoggingFile) } if l.Enabled { l.Log(logType, data, environment, uuid, debug) @@ -179,7 +178,7 @@ func (logTLS *LoggerTLS) Log(logType string, data []byte, environment, uuid stri case settings.LoggingNone: l, ok := logTLS.Logger.(*LoggerNone) if !ok { - log.Printf("error casting logger to %s", settings.LoggingNone) + log.Error().Msgf("error casting logger to %s", settings.LoggingNone) } if l.Enabled { l.Log(logType, data, environment, uuid, debug) @@ -187,7 +186,7 @@ func (logTLS *LoggerTLS) Log(logType string, data []byte, environment, uuid stri case settings.LoggingKinesis: l, ok := logTLS.Logger.(*LoggerKinesis) if !ok { - log.Printf("error casting logger to %s", settings.LoggingKinesis) + log.Error().Msgf("error casting logger to %s", settings.LoggingKinesis) } if l.Enabled { l.Send(logType, data, environment, uuid, debug) @@ -195,7 +194,7 @@ func (logTLS *LoggerTLS) Log(logType string, data []byte, environment, uuid stri case settings.LoggingS3: l, ok := logTLS.Logger.(*LoggerS3) if !ok { - log.Printf("error casting logger to %s", settings.LoggingS3) + log.Error().Msgf("error casting logger to %s", settings.LoggingS3) } if l.Enabled { l.Send(logType, data, environment, uuid, debug) @@ -223,7 +222,7 @@ func (logTLS *LoggerTLS) QueryLog(logType string, data []byte, environment, uuid case settings.LoggingSplunk: l, ok := logTLS.Logger.(*LoggerSplunk) if !ok { - log.Printf("error casting logger to %s", settings.LoggingSplunk) + log.Error().Msgf("error casting logger to %s", settings.LoggingSplunk) } if l.Enabled { l.Send(logType, data, environment, uuid, debug) @@ -231,7 +230,7 @@ func (logTLS *LoggerTLS) QueryLog(logType string, data []byte, environment, uuid case settings.LoggingGraylog: l, ok := logTLS.Logger.(*LoggerGraylog) if !ok { - log.Printf("error casting logger to %s", settings.LoggingGraylog) + log.Error().Msgf("error casting logger to %s", settings.LoggingGraylog) } if l.Enabled { l.Send(logType, data, environment, uuid, debug) @@ -239,7 +238,7 @@ func (logTLS *LoggerTLS) QueryLog(logType string, data []byte, environment, uuid case settings.LoggingDB: l, ok := logTLS.Logger.(*LoggerDB) if !ok { - log.Printf("error casting logger to %s", settings.LoggingDB) + log.Error().Msgf("error casting logger to %s", settings.LoggingDB) } if l.Enabled { l.Query(data, environment, uuid, name, status, debug) @@ -247,7 +246,7 @@ func (logTLS *LoggerTLS) QueryLog(logType string, data []byte, environment, uuid case settings.LoggingStdout: l, ok := logTLS.Logger.(*LoggerStdout) if !ok { - log.Printf("error casting logger to %s", settings.LoggingStdout) + log.Error().Msgf("error casting logger to %s", settings.LoggingStdout) } if l.Enabled { l.Query(data, environment, uuid, name, status, debug) @@ -255,7 +254,7 @@ func (logTLS *LoggerTLS) QueryLog(logType string, data []byte, environment, uuid case settings.LoggingFile: l, ok := logTLS.Logger.(*LoggerFile) if !ok { - log.Printf("error casting logger to %s", settings.LoggingFile) + log.Error().Msgf("error casting logger to %s", settings.LoggingFile) } if l.Enabled { l.Query(data, environment, uuid, name, status, debug) @@ -263,7 +262,7 @@ func (logTLS *LoggerTLS) QueryLog(logType string, data []byte, environment, uuid case settings.LoggingNone: l, ok := logTLS.Logger.(*LoggerNone) if !ok { - log.Printf("error casting logger to %s", settings.LoggingNone) + log.Error().Msgf("error casting logger to %s", settings.LoggingNone) } if l.Enabled { l.Query(data, environment, uuid, name, status, debug) @@ -271,7 +270,7 @@ func (logTLS *LoggerTLS) QueryLog(logType string, data []byte, environment, uuid case settings.LoggingKinesis: l, ok := logTLS.Logger.(*LoggerKinesis) if !ok { - log.Printf("error casting logger to %s", settings.LoggingKinesis) + log.Error().Msgf("error casting logger to %s", settings.LoggingKinesis) } if l.Enabled { l.Send(logType, data, environment, uuid, debug) @@ -279,7 +278,7 @@ func (logTLS *LoggerTLS) QueryLog(logType string, data []byte, environment, uuid case settings.LoggingS3: l, ok := logTLS.Logger.(*LoggerS3) if !ok { - log.Printf("error casting logger to %s", settings.LoggingS3) + log.Error().Msgf("error casting logger to %s", settings.LoggingS3) } if l.Enabled { l.Send(logType, data, environment, uuid, debug) diff --git a/logging/logstash.go b/logging/logstash.go index 3f725365..f9b7ebe8 100644 --- a/logging/logstash.go +++ b/logging/logstash.go @@ -2,12 +2,12 @@ package logging import ( "fmt" - "log" "net" "strings" "github.com/jmpsec/osctrl/settings" "github.com/jmpsec/osctrl/utils" + "github.com/rs/zerolog/log" "github.com/spf13/viper" ) @@ -54,7 +54,7 @@ func CreateLoggerLogstash(logstashFile string) (*LoggerLogstash, error) { // LoadLogstash - Function to load the Logstash configuration from JSON file func LoadLogstash(file string) (LogstashConfiguration, error) { var _logstashCfg LogstashConfiguration - log.Printf("Loading %s", file) + log.Info().Msgf("Loading %s", file) // Load file and read config viper.SetConfigFile(file) if err := viper.ReadInConfig(); err != nil { @@ -88,71 +88,71 @@ type LogstashMessage struct { // Settings - Function to prepare settings for the logger func (logLS *LoggerLogstash) Settings(mgr *settings.Settings) { - log.Printf("Setting Logstash logging settings\n") + log.Info().Msg("Setting Logstash logging settings") } // SendHTTP - Function that sends JSON logs to Logstash via HTTP func (logLS *LoggerLogstash) SendHTTP(logType string, data []byte, environment, uuid string, debug bool) { if debug { - log.Printf("DebugService: Send %s via Logstash HTTP", logType) + log.Debug().Msgf("DebugService: Send %s via Logstash HTTP", logType) } jsonData := strings.NewReader(string(data)) if debug { - log.Printf("DebugService: Sending %d bytes to Logstash HTTP for %s - %s", len(data), environment, uuid) + log.Debug().Msgf("DebugService: Sending %d bytes to Logstash HTTP for %s - %s", len(data), environment, uuid) } httpURL := fmt.Sprintf("http://%s:%s", logLS.Configuration.Host, logLS.Configuration.Port) // Send log with a POST to the Splunk URL resp, body, err := utils.SendRequest(LogstashMethod, httpURL, jsonData, logLS.Headers) if err != nil { - log.Printf("Error sending request %s", err) + log.Err(err).Msg("Error sending request") } if debug { - log.Printf("DebugService: HTTP %d %s", resp, body) + log.Debug().Msgf("DebugService: HTTP %d %s", resp, body) } } // SendUDP - Function that sends data to Logstash via UDP func (logLS *LoggerLogstash) SendUDP(logType string, data []byte, environment, uuid string, debug bool) { if debug { - log.Printf("DebugService: Send %s via Logstash TCP", logType) + log.Debug().Msgf("DebugService: Send %s via Logstash TCP", logType) } if debug { - log.Printf("DebugService: Sending %d bytes to Logstash TCP for %s - %s", len(data), environment, uuid) + log.Debug().Msgf("DebugService: Sending %d bytes to Logstash TCP for %s - %s", len(data), environment, uuid) } connAddr := fmt.Sprintf(LogstashConnStr, logLS.Configuration.Host, logLS.Configuration.Port) conn, err := net.Dial("udp", connAddr) if err != nil { - log.Printf("Error connecting to Logstash %s", err) + log.Err(err).Msg("Error connecting to Logstash") } defer conn.Close() _, err = conn.Write(data) if err != nil { - log.Printf("Error writing to Logstash %s", err) + log.Err(err).Msg("Error writing to Logstash") } if debug { - log.Printf("DebugService: Sent data to Logstash TCP") + log.Debug().Msg("DebugService: Sent data to Logstash TCP") } } // SendTCP - Function that sends data to Logstash via TCP func (logLS *LoggerLogstash) SendTCP(logType string, data []byte, environment, uuid string, debug bool) { if debug { - log.Printf("DebugService: Send %s via Logstash UDP", logType) + log.Debug().Msgf("DebugService: Send %s via Logstash UDP", logType) } if debug { - log.Printf("DebugService: Sending %d bytes to Logstash UDP for %s - %s", len(data), environment, uuid) + log.Debug().Msgf("DebugService: Sending %d bytes to Logstash UDP for %s - %s", len(data), environment, uuid) } connAddr := fmt.Sprintf(LogstashConnStr, logLS.Configuration.Host, logLS.Configuration.Port) conn, err := net.Dial("tcp", connAddr) if err != nil { - log.Printf("Error connecting to Logstash %s", err) + log.Err(err).Msg("Error connecting to Logstash") } defer conn.Close() _, err = conn.Write(data) if err != nil { - log.Printf("Error writing to Logstash %s", err) + log.Err(err).Msg("Error writing to Logstash") } if debug { - log.Printf("DebugService: Sent data to Logstash UDP") + log.Debug().Msg("DebugService: Sent data to Logstash UDP") } } diff --git a/logging/none.go b/logging/none.go index c7e2e243..4ae0cd85 100644 --- a/logging/none.go +++ b/logging/none.go @@ -1,10 +1,9 @@ package logging import ( - "log" - "github.com/jmpsec/osctrl/settings" "github.com/jmpsec/osctrl/types" + "github.com/rs/zerolog/log" ) // LoggerNone will be used to not log any data @@ -19,13 +18,13 @@ func CreateLoggerNone() (*LoggerNone, error) { // Settings - Function to prepare settings for the logger func (logNone *LoggerNone) Settings(mgr *settings.Settings) { - log.Printf("No none logging settings\n") + log.Info().Msg("No none logging settings") } // Log - Function that sends JSON result/status/query logs to stdout func (logNone *LoggerNone) Log(logType string, data []byte, environment, uuid string, debug bool) { if debug { - log.Printf("Sending %d bytes to none for %s - %s", len(data), environment, uuid) + log.Debug().Msgf("Sending %d bytes to none for %s - %s", len(data), environment, uuid) } switch logType { case types.StatusLog: @@ -37,15 +36,15 @@ func (logNone *LoggerNone) Log(logType string, data []byte, environment, uuid st // Status - Function that sends JSON status logs to stdout func (logNone *LoggerNone) Status(data []byte, environment, uuid string, debug bool) { - log.Printf("Skipping to log %d bytes of status from %s/%s", len(data), environment, uuid) + log.Info().Msgf("Skipping to log %d bytes of status from %s/%s", len(data), environment, uuid) } // Result - Function that sends JSON result logs to stdout func (logNone *LoggerNone) Result(data []byte, environment, uuid string, debug bool) { - log.Printf("Skipping to log %d bytes of result from %s/%s", len(data), environment, uuid) + log.Info().Msgf("Skipping to log %d bytes of result from %s/%s", len(data), environment, uuid) } // Query - Function that sends JSON query logs to stdout func (logNone *LoggerNone) Query(data []byte, environment, uuid, name string, status int, debug bool) { - log.Printf("Skipping to log %d bytes of query from %s/%s for query %s and status %d", len(data), environment, uuid, name, status) + log.Info().Msgf("Skipping to log %d bytes of query from %s/%s for query %s and status %d", len(data), environment, uuid, name, status) } diff --git a/logging/process.go b/logging/process.go index 4b6738ab..6ee54392 100644 --- a/logging/process.go +++ b/logging/process.go @@ -1,11 +1,11 @@ package logging import ( - "log" + "encoding/json" "github.com/jmpsec/osctrl/nodes" "github.com/jmpsec/osctrl/types" - "encoding/json" + "github.com/rs/zerolog/log" ) // ProcessLogs - Helper to process logs @@ -14,10 +14,10 @@ func (l *LoggerTLS) ProcessLogs(data json.RawMessage, logType, environment, ipad var logs []types.LogGenericData if err := json.Unmarshal(data, &logs); err != nil { // FIXME metrics for this - log.Printf("error parsing log %s %v", string(data), err) + log.Err(err).Msgf("error parsing log %s", string(data)) } if debug { - log.Printf("parsing logs for metadata in %s:%s", logType, environment) + log.Debug().Msgf("parsing logs for metadata in %s:%s", logType, environment) } // Iterate through received messages to extract metadata var uuids, hosts, names, users, osqueryusers, hashes, dhashes, osqueryversions []string @@ -32,7 +32,7 @@ func (l *LoggerTLS) ProcessLogs(data json.RawMessage, logType, environment, ipad osqueryversions = append(osqueryversions, l.Version) } if debug { - log.Printf("metadata and dispatch for %s", uniq(uuids)[0]) + log.Debug().Msgf("metadata and dispatch for %s", uniq(uuids)[0]) } // FIXME it only uses the first element from the []string that uniq returns metadata := nodes.NodeMetadata{ @@ -55,11 +55,11 @@ func (l *LoggerTLS) ProcessLogQueryResult(queriesWrite types.QueryWriteRequest, // Retrieve node node, err := l.Nodes.GetByKey(queriesWrite.NodeKey) if err != nil { - log.Printf("error retrieving node %s", err) + log.Err(err).Msg("error retrieving node") } // Integrity check if envid != node.EnvironmentID { - log.Printf("ProcessLogQueryResult: EnvID[%d] does not match Node.EnvironmentID[%d]", envid, node.EnvironmentID) + log.Error().Msgf("ProcessLogQueryResult: EnvID[%d] does not match Node.EnvironmentID[%d]", envid, node.EnvironmentID) } // Tap into results so we can update internal metrics for q, r := range queriesWrite.Queries { @@ -79,15 +79,15 @@ func (l *LoggerTLS) ProcessLogQueryResult(queriesWrite types.QueryWriteRequest, err = l.Queries.IncExecution(q, envid) } if err != nil { - log.Printf("error updating query %s", err) + log.Err(err).Msg("error updating query") } // Add a record for this query if err := l.Queries.TrackExecution(q, node.UUID, queriesWrite.Statuses[q]); err != nil { - log.Printf("error adding query execution %s", err) + log.Err(err).Msg("error adding query execution") } // Check if query is completed if err := l.Queries.VerifyComplete(q, envid); err != nil { - log.Printf("error verifying and completing query %s", err) + log.Err(err).Msg("error verifying and completing query") } } } diff --git a/logging/s3.go b/logging/s3.go index 636604da..a9b924aa 100644 --- a/logging/s3.go +++ b/logging/s3.go @@ -3,13 +3,13 @@ package logging import ( "bytes" "context" - "log" "net/http" "strconv" "time" "github.com/jmpsec/osctrl/settings" "github.com/jmpsec/osctrl/types" + "github.com/rs/zerolog/log" "github.com/spf13/viper" "github.com/aws/aws-sdk-go-v2/aws" @@ -65,7 +65,7 @@ func CreateLoggerS3File(s3File string) (*LoggerS3, error) { // LoadS3 - Function to load the S3 configuration from JSON file func LoadS3(file string) (types.S3Configuration, error) { var _s3Cfg types.S3Configuration - log.Printf("Loading %s", file) + log.Info().Msgf("Loading %s", file) // Load file and read config viper.SetConfigFile(file) if err := viper.ReadInConfig(); err != nil { @@ -81,14 +81,14 @@ func LoadS3(file string) (types.S3Configuration, error) { // Settings - Function to prepare settings for the logger func (logS3 *LoggerS3) Settings(mgr *settings.Settings) { - log.Printf("No s3 logging settings\n") + log.Info().Msg("No s3 logging settings") } // Send - Function that sends JSON logs to S3 func (logS3 *LoggerS3) Send(logType string, data []byte, environment, uuid string, debug bool) { ctx := context.Background() if debug { - log.Printf("DebugService: Sending %d bytes to S3 for %s - %s", len(data), environment, uuid) + log.Debug().Msgf("DebugService: Sending %d bytes to S3 for %s - %s", len(data), environment, uuid) } ptrContentLength := int64(len(data)) result, err := logS3.Uploader.Upload(ctx, &s3.PutObjectInput{ @@ -99,9 +99,9 @@ func (logS3 *LoggerS3) Send(logType string, data []byte, environment, uuid strin ContentType: aws.String(http.DetectContentType(data)), }) if err != nil { - log.Printf("Error sending data to s3 %s", err) + log.Err(err).Msg("Error sending data to s3") } if debug { - log.Printf("DebugService: S3 Upload %+v", result) + log.Debug().Msgf("DebugService: S3 Upload %+v", result) } } diff --git a/logging/splunk.go b/logging/splunk.go index c5a7fddd..5c6e8bb8 100644 --- a/logging/splunk.go +++ b/logging/splunk.go @@ -2,13 +2,13 @@ package logging import ( "encoding/json" - "log" "strings" "time" "github.com/jmpsec/osctrl/settings" "github.com/jmpsec/osctrl/types" "github.com/jmpsec/osctrl/utils" + "github.com/rs/zerolog/log" "github.com/spf13/viper" ) @@ -47,7 +47,7 @@ func CreateLoggerSplunk(splunkFile string) (*LoggerSplunk, error) { // LoadSplunk - Function to load the Splunk configuration from JSON file func LoadSplunk(file string) (SlunkConfiguration, error) { var _splunkCfg SlunkConfiguration - log.Printf("Loading %s", file) + log.Info().Msgf("Loading %s", file) // Load file and read config viper.SetConfigFile(file) if err := viper.ReadInConfig(); err != nil { @@ -80,13 +80,13 @@ type SplunkMessage struct { // Settings - Function to prepare settings for the logger func (logSP *LoggerSplunk) Settings(mgr *settings.Settings) { - log.Printf("Setting Splunk logging settings\n") + log.Info().Msg("Setting Splunk logging settings") } // Send - Function that sends JSON logs to Splunk HTTP Event Collector func (logSP *LoggerSplunk) Send(logType string, data []byte, environment, uuid string, debug bool) { if debug { - log.Printf("DebugService: Send %s via splunk", logType) + log.Debug().Msgf("DebugService: Send %s via splunk", logType) } // Check if this is result/status or query var sourceType string @@ -96,14 +96,14 @@ func (logSP *LoggerSplunk) Send(logType string, data []byte, environment, uuid s // For on-demand queries, just a JSON blob with results and statuses var result interface{} if err := json.Unmarshal(data, &result); err != nil { - log.Printf("error parsing data %s %v", string(data), err) + log.Err(err).Msgf("error parsing data %s", string(data)) } logs = append(logs, result) } else { sourceType = logType + ":" + environment // For scheduled queries, convert the array in an array of multiple events if err := json.Unmarshal(data, &logs); err != nil { - log.Printf("error parsing log %s %v", string(data), err) + log.Err(err).Msgf("error parsing log %s", string(data)) } } // Prepare data according to HTTP Event Collector format @@ -111,7 +111,7 @@ func (logSP *LoggerSplunk) Send(logType string, data []byte, environment, uuid s for _, l := range logs { jsonEvent, err := json.Marshal(l) if err != nil { - log.Printf("Error parsing data %s", err) + log.Err(err).Msg("Error parsing data") continue } eventData := SplunkMessage{ @@ -127,18 +127,18 @@ func (logSP *LoggerSplunk) Send(logType string, data []byte, environment, uuid s // Serialize data for Splunk jsonEvents, err := json.Marshal(events) if err != nil { - log.Printf("Error parsing data %s", err) + log.Err(err).Msgf("Error parsing data") } jsonParam := strings.NewReader(string(jsonEvents)) if debug { - log.Printf("DebugService: Sending %d bytes to Splunk for %s - %s", len(data), environment, uuid) + log.Debug().Msgf("DebugService: Sending %d bytes to Splunk for %s - %s", len(data), environment, uuid) } // Send log with a POST to the Splunk URL resp, body, err := utils.SendRequest(SplunkMethod, logSP.Configuration.URL, jsonParam, logSP.Headers) if err != nil { - log.Printf("Error sending request %s", err) + log.Err(err).Msgf("Error sending request") } if debug { - log.Printf("DebugService: HTTP %d %s", resp, body) + log.Debug().Msgf("DebugService: HTTP %d %s", resp, body) } } diff --git a/logging/stdout.go b/logging/stdout.go index 8c4f4496..06c15437 100644 --- a/logging/stdout.go +++ b/logging/stdout.go @@ -1,10 +1,9 @@ package logging import ( - "log" - "github.com/jmpsec/osctrl/settings" "github.com/jmpsec/osctrl/types" + "github.com/rs/zerolog/log" ) // LoggerStdout will be used to log data using stdout @@ -21,7 +20,7 @@ func CreateLoggerStdout() (*LoggerStdout, error) { // Settings - Function to prepare settings for the logger func (logStdout *LoggerStdout) Settings(mgr *settings.Settings) { - log.Printf("No stdout logging settings\n") + log.Info().Msg("No stdout logging settings") } // Log - Function that sends JSON result/status/query logs to stdout @@ -36,15 +35,15 @@ func (logStdout *LoggerStdout) Log(logType string, data []byte, environment, uui // Status - Function that sends JSON status logs to stdout func (logStdout *LoggerStdout) Status(data []byte, environment, uuid string, debug bool) { - log.Printf("Status: %s:%s - %d bytes [%s]", environment, uuid, len(data), string(data)) + log.Info().Msgf("Status: %s:%s - %d bytes [%s]", environment, uuid, len(data), string(data)) } // Result - Function that sends JSON result logs to stdout func (logStdout *LoggerStdout) Result(data []byte, environment, uuid string, debug bool) { - log.Printf("Result: %s:%s - %d bytes [%s]", environment, uuid, len(data), string(data)) + log.Info().Msgf("Result: %s:%s - %d bytes [%s]", environment, uuid, len(data), string(data)) } // Query - Function that sends JSON query logs to stdout func (logStdout *LoggerStdout) Query(data []byte, environment, uuid, name string, status int, debug bool) { - log.Printf("Query: %s:%d - %s:%s - %d bytes [%s]", name, status, environment, uuid, len(data), string(data)) + log.Info().Msgf("Query: %s:%d - %s:%s - %d bytes [%s]", name, status, environment, uuid, len(data), string(data)) } diff --git a/metrics/go.mod b/metrics/go.mod index 792c6786..5e5811c7 100644 --- a/metrics/go.mod +++ b/metrics/go.mod @@ -23,6 +23,8 @@ require ( github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jmpsec/osctrl/nodes v0.3.9 // indirect github.com/jmpsec/osctrl/utils v0.0.0-20240904183539-155969b2e259 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/pelletier/go-toml/v2 v2.2.3 // indirect github.com/sagikazarmark/locafero v0.6.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -48,6 +50,7 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rs/zerolog v1.33.0 github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect diff --git a/metrics/go.sum b/metrics/go.sum index cb414bde..f2614c58 100644 --- a/metrics/go.sum +++ b/metrics/go.sum @@ -1,5 +1,6 @@ github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -8,6 +9,7 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -32,17 +34,26 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/zerolog v1.33.0 h1:1cU2KZkvPxNyfgEmhHAz/1A9Bz+llsdYzklWFzgp0r8= +github.com/rs/zerolog v1.33.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/sagikazarmark/locafero v0.6.0 h1:ON7AQg37yzcRPU69mt7gwhFEBwxI6P9T4Qu3N51bwOk= github.com/sagikazarmark/locafero v0.6.0/go.mod h1:77OmuIc6VTraTXKXIs/uvUxKGUXjE1GbemJYHqdNjX0= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= @@ -90,6 +101,9 @@ golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= diff --git a/metrics/metrics.go b/metrics/metrics.go index 04f61281..0b33c028 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -3,11 +3,11 @@ package metrics import ( "bytes" "fmt" - "log" "net" "sync" "time" + "github.com/rs/zerolog/log" "github.com/spf13/viper" ) @@ -28,7 +28,7 @@ type Configuration struct { // LoadConfiguration - Function to load the metrics configuration from JSON file func LoadConfiguration() (Configuration, error) { var _metricsCfg Configuration - log.Printf("Loading %s", metricsConfigFile) + log.Info().Msgf("Loading %s", metricsConfigFile) // Load file and read config viper.SetConfigFile(metricsConfigFile) if err := viper.ReadInConfig(); err != nil { @@ -119,18 +119,18 @@ func (metrics *Metrics) Disconnect() error { // ConnectAndSend to connect and submit a metric via TCP or UDP func (metrics *Metrics) ConnectAndSend(name string, value int) { if err := metrics.Connect(); err != nil { - log.Printf("error connecting %v", err) + log.Err(err).Msg("error connecting") } err := metrics.Send(name, value) i := 0 for err != nil { - log.Printf("Something happened %v", err) + log.Err(err).Msg("Something happened in Send") _ = metrics.Connect() err = metrics.Send(name, value) if i < defaultRetries { i++ } else { - log.Printf("Too many retries, exiting") + log.Debug().Msg("Too many retries, exiting") break } }