Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add raw counters for MongoDB metrics #5264

Merged
merged 3 commits into from
Jan 9, 2019
Merged

Add raw counters for MongoDB metrics #5264

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ceb2d86
Add raw counters for MongoDB metrics
bozaro Oct 6, 2017
11d7b46
Remove some `_cnt` suffixes from raw MongoDB metrics
bozaro Jan 9, 2019
fa87489
Rename `_cnt` -> `_count` suffixes from raw MongoDB metrics
bozaro Jan 9, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions plugins/inputs/mongodb/mongodb_data.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,28 +31,43 @@ func NewMongodbData(statLine *StatLine, tags map[string]string) *MongodbData {
}

var DefaultStats = map[string]string{
"inserts": "InsertCnt",
"inserts_per_sec": "Insert",
"queries": "QueryCnt",
"queries_per_sec": "Query",
"updates": "UpdateCnt",
"updates_per_sec": "Update",
"deletes": "DeleteCnt",
"deletes_per_sec": "Delete",
"getmores": "GetMoreCnt",
"getmores_per_sec": "GetMore",
"commands": "CommandCnt",
"commands_per_sec": "Command",
"flushes": "FlushesCnt",
"flushes_per_sec": "Flushes",
"vsize_megabytes": "Virtual",
"resident_megabytes": "Resident",
"queued_reads": "QueuedReaders",
"queued_writes": "QueuedWriters",
"active_reads": "ActiveReaders",
"active_writes": "ActiveWriters",
"net_in_bytes_count": "NetInCnt",
"net_in_bytes": "NetIn",
"net_out_bytes_count": "NetOutCnt",
"net_out_bytes": "NetOut",
"open_connections": "NumConnections",
"ttl_deletes": "DeletedDocumentsCnt",
"ttl_deletes_per_sec": "DeletedDocuments",
"ttl_passes": "PassesCnt",
"ttl_passes_per_sec": "Passes",
"cursor_timed_out": "TimedOutC",
"cursor_timed_out_count": "TimedOutCCnt",
"cursor_no_timeout": "NoTimeoutC",
"cursor_no_timeout_count": "NoTimeoutCCnt",
"cursor_pinned": "PinnedC",
"cursor_pinned_count": "PinnedCCnt",
"cursor_total": "TotalC",
"cursor_total_count": "TotalCCnt",
"document_deleted": "DeletedD",
"document_inserted": "InsertedD",
"document_returned": "ReturnedD",
Expand All @@ -63,11 +78,17 @@ var DefaultStats = map[string]string{
}

var DefaultReplStats = map[string]string{
"repl_inserts": "InsertRCnt",
"repl_inserts_per_sec": "InsertR",
"repl_queries": "QueryRCnt",
"repl_queries_per_sec": "QueryR",
"repl_updates": "UpdateRCnt",
"repl_updates_per_sec": "UpdateR",
"repl_deletes": "DeleteRCnt",
"repl_deletes_per_sec": "DeleteR",
"repl_getmores": "GetMoreRCnt",
"repl_getmores_per_sec": "GetMoreR",
"repl_commands": "CommandRCnt",
"repl_commands_per_sec": "CommandR",
"member_status": "NodeType",
"state": "NodeState",
Expand Down Expand Up @@ -96,6 +117,7 @@ var ShardHostStats = map[string]string{
var MmapStats = map[string]string{
"mapped_megabytes": "Mapped",
"non-mapped_megabytes": "NonMapped",
"page_faults": "FaultsCnt",
"page_faults_per_sec": "Faults",
}

Expand Down
23 changes: 23 additions & 0 deletions plugins/inputs/mongodb/mongodb_data_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@ func TestAddNonReplStats(t *testing.T) {
Insert: 0,
Query: 0,
Update: 0,
UpdateCnt: 0,
Delete: 0,
GetMore: 0,
Command: 0,
Flushes: 0,
FlushesCnt: 0,
Virtual: 0,
Resident: 0,
QueuedReaders: 0,
Expand Down Expand Up @@ -191,41 +193,62 @@ func TestStateTag(t *testing.T) {
fields := map[string]interface{}{
"active_reads": int64(0),
"active_writes": int64(0),
"commands": int64(0),
"commands_per_sec": int64(0),
"deletes": int64(0),
"deletes_per_sec": int64(0),
"flushes": int64(0),
"flushes_per_sec": int64(0),
"getmores": int64(0),
"getmores_per_sec": int64(0),
"inserts": int64(0),
"inserts_per_sec": int64(0),
"member_status": "PRI",
"state": "PRIMARY",
"net_in_bytes_count": int64(0),
"net_in_bytes": int64(0),
"net_out_bytes_count": int64(0),
"net_out_bytes": int64(0),
"open_connections": int64(0),
"queries": int64(0),
"queries_per_sec": int64(0),
"queued_reads": int64(0),
"queued_writes": int64(0),
"repl_commands": int64(0),
"repl_commands_per_sec": int64(0),
"repl_deletes": int64(0),
"repl_deletes_per_sec": int64(0),
"repl_getmores": int64(0),
"repl_getmores_per_sec": int64(0),
"repl_inserts": int64(0),
"repl_inserts_per_sec": int64(0),
"repl_queries": int64(0),
"repl_queries_per_sec": int64(0),
"repl_updates": int64(0),
"repl_updates_per_sec": int64(0),
"repl_lag": int64(0),
"repl_oplog_window_sec": int64(0),
"resident_megabytes": int64(0),
"updates": int64(0),
"updates_per_sec": int64(0),
"vsize_megabytes": int64(0),
"ttl_deletes": int64(0),
"ttl_deletes_per_sec": int64(0),
"ttl_passes": int64(0),
"ttl_passes_per_sec": int64(0),
"jumbo_chunks": int64(0),
"total_in_use": int64(0),
"total_available": int64(0),
"total_created": int64(0),
"total_refreshing": int64(0),
"cursor_timed_out": int64(0),
"cursor_timed_out_count": int64(0),
"cursor_no_timeout": int64(0),
"cursor_no_timeout_count": int64(0),
"cursor_pinned": int64(0),
"cursor_pinned_count": int64(0),
"cursor_total": int64(0),
"cursor_total_count": int64(0),
"document_deleted": int64(0),
"document_inserted": int64(0),
"document_returned": int64(0),
Expand Down
100 changes: 57 additions & 43 deletions plugins/inputs/mongodb/mongostat.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -457,14 +457,22 @@ type StatLine struct {
LastPrinted time.Time

// Opcounter fields
Insert, Query, Update, Delete, GetMore, Command int64
Insert, InsertCnt int64
Query, QueryCnt int64
Update, UpdateCnt int64
Delete, DeleteCnt int64
GetMore, GetMoreCnt int64
Command, CommandCnt int64

// TTL fields
Passes, DeletedDocuments int64
Passes, PassesCnt int64
DeletedDocuments, DeletedDocumentsCnt int64

// Cursor fields
TimedOutC int64
NoTimeoutC, PinnedC, TotalC int64
TimedOutC, TimedOutCCnt int64
NoTimeoutC, NoTimeoutCCnt int64
PinnedC, PinnedCCnt int64
TotalC, TotalCCnt int64

// Document fields
DeletedD, InsertedD, ReturnedD, UpdatedD int64
Expand Down Expand Up @@ -494,20 +502,26 @@ type StatLine struct {
WorkerThreadEvictingPages int64

// Replicated Opcounter fields
InsertR, QueryR, UpdateR, DeleteR, GetMoreR, CommandR int64
ReplLag int64
OplogTimeDiff int64
Flushes int64
Mapped, Virtual, Resident, NonMapped int64
Faults int64
HighestLocked *LockStatus
QueuedReaders, QueuedWriters int64
ActiveReaders, ActiveWriters int64
NetIn, NetOut int64
NumConnections int64
ReplSetName string
NodeType string
NodeState string
InsertR, InsertRCnt int64
QueryR, QueryRCnt int64
UpdateR, UpdateRCnt int64
DeleteR, DeleteRCnt int64
GetMoreR, GetMoreRCnt int64
CommandR, CommandRCnt int64
ReplLag int64
OplogTimeDiff int64
Flushes, FlushesCnt int64
Mapped, Virtual, Resident, NonMapped int64
Faults, FaultsCnt int64
HighestLocked *LockStatus
QueuedReaders, QueuedWriters int64
ActiveReaders, ActiveWriters int64
NetIn, NetInCnt int64
NetOut, NetOutCnt int64
NumConnections int64
ReplSetName string
NodeType string
NodeState string

// Cluster fields
JumboChunksCount int64
Expand Down Expand Up @@ -576,12 +590,12 @@ func computeLockDiffs(prevLocks, curLocks map[string]LockUsage) []LockUsage {
return lockUsages
}

func diff(newVal, oldVal, sampleTime int64) int64 {
func diff(newVal, oldVal, sampleTime int64) (int64, int64) {
d := newVal - oldVal
if d < 0 {
d = newVal
}
return d / sampleTime
return d / sampleTime, newVal
}

// NewStatLine constructs a StatLine object from two MongoStatus objects.
Expand Down Expand Up @@ -612,25 +626,25 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
}

if newStat.Opcounters != nil && oldStat.Opcounters != nil {
returnVal.Insert = diff(newStat.Opcounters.Insert, oldStat.Opcounters.Insert, sampleSecs)
returnVal.Query = diff(newStat.Opcounters.Query, oldStat.Opcounters.Query, sampleSecs)
returnVal.Update = diff(newStat.Opcounters.Update, oldStat.Opcounters.Update, sampleSecs)
returnVal.Delete = diff(newStat.Opcounters.Delete, oldStat.Opcounters.Delete, sampleSecs)
returnVal.GetMore = diff(newStat.Opcounters.GetMore, oldStat.Opcounters.GetMore, sampleSecs)
returnVal.Command = diff(newStat.Opcounters.Command, oldStat.Opcounters.Command, sampleSecs)
returnVal.Insert, returnVal.InsertCnt = diff(newStat.Opcounters.Insert, oldStat.Opcounters.Insert, sampleSecs)
returnVal.Query, returnVal.QueryCnt = diff(newStat.Opcounters.Query, oldStat.Opcounters.Query, sampleSecs)
returnVal.Update, returnVal.UpdateCnt = diff(newStat.Opcounters.Update, oldStat.Opcounters.Update, sampleSecs)
returnVal.Delete, returnVal.DeleteCnt = diff(newStat.Opcounters.Delete, oldStat.Opcounters.Delete, sampleSecs)
returnVal.GetMore, returnVal.GetMoreCnt = diff(newStat.Opcounters.GetMore, oldStat.Opcounters.GetMore, sampleSecs)
returnVal.Command, returnVal.CommandCnt = diff(newStat.Opcounters.Command, oldStat.Opcounters.Command, sampleSecs)
}

if newStat.Metrics != nil && oldStat.Metrics != nil {
if newStat.Metrics.TTL != nil && oldStat.Metrics.TTL != nil {
returnVal.Passes = diff(newStat.Metrics.TTL.Passes, oldStat.Metrics.TTL.Passes, sampleSecs)
returnVal.DeletedDocuments = diff(newStat.Metrics.TTL.DeletedDocuments, oldStat.Metrics.TTL.DeletedDocuments, sampleSecs)
returnVal.Passes, returnVal.PassesCnt = diff(newStat.Metrics.TTL.Passes, oldStat.Metrics.TTL.Passes, sampleSecs)
returnVal.DeletedDocuments, returnVal.DeletedDocumentsCnt = diff(newStat.Metrics.TTL.DeletedDocuments, oldStat.Metrics.TTL.DeletedDocuments, sampleSecs)
}
if newStat.Metrics.Cursor != nil && oldStat.Metrics.Cursor != nil {
returnVal.TimedOutC = diff(newStat.Metrics.Cursor.TimedOut, oldStat.Metrics.Cursor.TimedOut, sampleSecs)
returnVal.TimedOutC, returnVal.TimedOutCCnt = diff(newStat.Metrics.Cursor.TimedOut, oldStat.Metrics.Cursor.TimedOut, sampleSecs)
if newStat.Metrics.Cursor.Open != nil && oldStat.Metrics.Cursor.Open != nil {
returnVal.NoTimeoutC = diff(newStat.Metrics.Cursor.Open.NoTimeout, oldStat.Metrics.Cursor.Open.NoTimeout, sampleSecs)
returnVal.PinnedC = diff(newStat.Metrics.Cursor.Open.Pinned, oldStat.Metrics.Cursor.Open.Pinned, sampleSecs)
returnVal.TotalC = diff(newStat.Metrics.Cursor.Open.Total, oldStat.Metrics.Cursor.Open.Total, sampleSecs)
returnVal.NoTimeoutC, returnVal.NoTimeoutCCnt = diff(newStat.Metrics.Cursor.Open.NoTimeout, oldStat.Metrics.Cursor.Open.NoTimeout, sampleSecs)
returnVal.PinnedC, returnVal.PinnedCCnt = diff(newStat.Metrics.Cursor.Open.Pinned, oldStat.Metrics.Cursor.Open.Pinned, sampleSecs)
returnVal.TotalC, returnVal.TotalCCnt = diff(newStat.Metrics.Cursor.Open.Total, oldStat.Metrics.Cursor.Open.Total, sampleSecs)
}
}
if newStat.Metrics.Document != nil {
Expand All @@ -642,18 +656,18 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
}

if newStat.OpcountersRepl != nil && oldStat.OpcountersRepl != nil {
returnVal.InsertR = diff(newStat.OpcountersRepl.Insert, oldStat.OpcountersRepl.Insert, sampleSecs)
returnVal.QueryR = diff(newStat.OpcountersRepl.Query, oldStat.OpcountersRepl.Query, sampleSecs)
returnVal.UpdateR = diff(newStat.OpcountersRepl.Update, oldStat.OpcountersRepl.Update, sampleSecs)
returnVal.DeleteR = diff(newStat.OpcountersRepl.Delete, oldStat.OpcountersRepl.Delete, sampleSecs)
returnVal.GetMoreR = diff(newStat.OpcountersRepl.GetMore, oldStat.OpcountersRepl.GetMore, sampleSecs)
returnVal.CommandR = diff(newStat.OpcountersRepl.Command, oldStat.OpcountersRepl.Command, sampleSecs)
returnVal.InsertR, returnVal.InsertRCnt = diff(newStat.OpcountersRepl.Insert, oldStat.OpcountersRepl.Insert, sampleSecs)
returnVal.QueryR, returnVal.QueryRCnt = diff(newStat.OpcountersRepl.Query, oldStat.OpcountersRepl.Query, sampleSecs)
returnVal.UpdateR, returnVal.UpdateRCnt = diff(newStat.OpcountersRepl.Update, oldStat.OpcountersRepl.Update, sampleSecs)
returnVal.DeleteR, returnVal.DeleteRCnt = diff(newStat.OpcountersRepl.Delete, oldStat.OpcountersRepl.Delete, sampleSecs)
returnVal.GetMoreR, returnVal.GetMoreRCnt = diff(newStat.OpcountersRepl.GetMore, oldStat.OpcountersRepl.GetMore, sampleSecs)
returnVal.CommandR, returnVal.CommandRCnt = diff(newStat.OpcountersRepl.Command, oldStat.OpcountersRepl.Command, sampleSecs)
}

returnVal.CacheDirtyPercent = -1
returnVal.CacheUsedPercent = -1
if newStat.WiredTiger != nil && oldStat.WiredTiger != nil {
returnVal.Flushes = newStat.WiredTiger.Transaction.TransCheckpoints - oldStat.WiredTiger.Transaction.TransCheckpoints
returnVal.Flushes, returnVal.FlushesCnt = diff(newStat.WiredTiger.Transaction.TransCheckpoints, oldStat.WiredTiger.Transaction.TransCheckpoints, sampleSecs)
returnVal.CacheDirtyPercent = float64(newStat.WiredTiger.Cache.TrackedDirtyBytes) / float64(newStat.WiredTiger.Cache.MaxBytesConfigured)
returnVal.CacheUsedPercent = float64(newStat.WiredTiger.Cache.CurrentCachedBytes) / float64(newStat.WiredTiger.Cache.MaxBytesConfigured)

Expand All @@ -670,7 +684,7 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
returnVal.ServerEvictingPages = newStat.WiredTiger.Cache.ServerEvictingPages
returnVal.WorkerThreadEvictingPages = newStat.WiredTiger.Cache.WorkerThreadEvictingPages
} else if newStat.BackgroundFlushing != nil && oldStat.BackgroundFlushing != nil {
returnVal.Flushes = newStat.BackgroundFlushing.Flushes - oldStat.BackgroundFlushing.Flushes
returnVal.Flushes, returnVal.FlushesCnt = diff(newStat.BackgroundFlushing.Flushes, oldStat.BackgroundFlushing.Flushes, sampleSecs)
}

returnVal.Time = newMongo.SampleTime
Expand Down Expand Up @@ -713,7 +727,7 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec

if oldStat.ExtraInfo != nil && newStat.ExtraInfo != nil &&
oldStat.ExtraInfo.PageFaults != nil && newStat.ExtraInfo.PageFaults != nil {
returnVal.Faults = diff(*(newStat.ExtraInfo.PageFaults), *(oldStat.ExtraInfo.PageFaults), sampleSecs)
returnVal.Faults, returnVal.FaultsCnt = diff(*(newStat.ExtraInfo.PageFaults), *(oldStat.ExtraInfo.PageFaults), sampleSecs)
}
if !returnVal.IsMongos && oldStat.Locks != nil {
globalCheck, hasGlobal := oldStat.Locks["Global"]
Expand Down Expand Up @@ -812,8 +826,8 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
}

if oldStat.Network != nil && newStat.Network != nil {
returnVal.NetIn = diff(newStat.Network.BytesIn, oldStat.Network.BytesIn, sampleSecs)
returnVal.NetOut = diff(newStat.Network.BytesOut, oldStat.Network.BytesOut, sampleSecs)
returnVal.NetIn, returnVal.NetInCnt = diff(newStat.Network.BytesIn, oldStat.Network.BytesIn, sampleSecs)
returnVal.NetOut, returnVal.NetOutCnt = diff(newStat.Network.BytesOut, oldStat.Network.BytesOut, sampleSecs)
}

if newStat.Connections != nil {
Expand Down