Skip to content

Commit

Permalink
Add raw counters to mongodb input (influxdata#5264)
Browse files Browse the repository at this point in the history
  • Loading branch information
@bozaro @danielnelson
bozaro authored and danielnelson committed Jan 9, 2019
1 parent 10b3e45 commit 4b3580c
Show file tree
Hide file tree
Showing 3 changed files with 102 additions and 43 deletions.
22 changes: 22 additions & 0 deletions plugins/inputs/mongodb/mongodb_data.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,28 +31,43 @@ func NewMongodbData(statLine *StatLine, tags map[string]string) *MongodbData {
}

var DefaultStats = map[string]string{
"inserts": "InsertCnt",
"inserts_per_sec": "Insert",
"queries": "QueryCnt",
"queries_per_sec": "Query",
"updates": "UpdateCnt",
"updates_per_sec": "Update",
"deletes": "DeleteCnt",
"deletes_per_sec": "Delete",
"getmores": "GetMoreCnt",
"getmores_per_sec": "GetMore",
"commands": "CommandCnt",
"commands_per_sec": "Command",
"flushes": "FlushesCnt",
"flushes_per_sec": "Flushes",
"vsize_megabytes": "Virtual",
"resident_megabytes": "Resident",
"queued_reads": "QueuedReaders",
"queued_writes": "QueuedWriters",
"active_reads": "ActiveReaders",
"active_writes": "ActiveWriters",
"net_in_bytes_count": "NetInCnt",
"net_in_bytes": "NetIn",
"net_out_bytes_count": "NetOutCnt",
"net_out_bytes": "NetOut",
"open_connections": "NumConnections",
"ttl_deletes": "DeletedDocumentsCnt",
"ttl_deletes_per_sec": "DeletedDocuments",
"ttl_passes": "PassesCnt",
"ttl_passes_per_sec": "Passes",
"cursor_timed_out": "TimedOutC",
"cursor_timed_out_count": "TimedOutCCnt",
"cursor_no_timeout": "NoTimeoutC",
"cursor_no_timeout_count": "NoTimeoutCCnt",
"cursor_pinned": "PinnedC",
"cursor_pinned_count": "PinnedCCnt",
"cursor_total": "TotalC",
"cursor_total_count": "TotalCCnt",
"document_deleted": "DeletedD",
"document_inserted": "InsertedD",
"document_returned": "ReturnedD",
Expand All @@ -63,11 +78,17 @@ var DefaultStats = map[string]string{
}

var DefaultReplStats = map[string]string{
"repl_inserts": "InsertRCnt",
"repl_inserts_per_sec": "InsertR",
"repl_queries": "QueryRCnt",
"repl_queries_per_sec": "QueryR",
"repl_updates": "UpdateRCnt",
"repl_updates_per_sec": "UpdateR",
"repl_deletes": "DeleteRCnt",
"repl_deletes_per_sec": "DeleteR",
"repl_getmores": "GetMoreRCnt",
"repl_getmores_per_sec": "GetMoreR",
"repl_commands": "CommandRCnt",
"repl_commands_per_sec": "CommandR",
"member_status": "NodeType",
"state": "NodeState",
Expand Down Expand Up @@ -96,6 +117,7 @@ var ShardHostStats = map[string]string{
var MmapStats = map[string]string{
"mapped_megabytes": "Mapped",
"non-mapped_megabytes": "NonMapped",
"page_faults": "FaultsCnt",
"page_faults_per_sec": "Faults",
}

Expand Down
23 changes: 23 additions & 0 deletions plugins/inputs/mongodb/mongodb_data_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@ func TestAddNonReplStats(t *testing.T) {
Insert: 0,
Query: 0,
Update: 0,
UpdateCnt: 0,
Delete: 0,
GetMore: 0,
Command: 0,
Flushes: 0,
FlushesCnt: 0,
Virtual: 0,
Resident: 0,
QueuedReaders: 0,
Expand Down Expand Up @@ -191,41 +193,62 @@ func TestStateTag(t *testing.T) {
fields := map[string]interface{}{
"active_reads": int64(0),
"active_writes": int64(0),
"commands": int64(0),
"commands_per_sec": int64(0),
"deletes": int64(0),
"deletes_per_sec": int64(0),
"flushes": int64(0),
"flushes_per_sec": int64(0),
"getmores": int64(0),
"getmores_per_sec": int64(0),
"inserts": int64(0),
"inserts_per_sec": int64(0),
"member_status": "PRI",
"state": "PRIMARY",
"net_in_bytes_count": int64(0),
"net_in_bytes": int64(0),
"net_out_bytes_count": int64(0),
"net_out_bytes": int64(0),
"open_connections": int64(0),
"queries": int64(0),
"queries_per_sec": int64(0),
"queued_reads": int64(0),
"queued_writes": int64(0),
"repl_commands": int64(0),
"repl_commands_per_sec": int64(0),
"repl_deletes": int64(0),
"repl_deletes_per_sec": int64(0),
"repl_getmores": int64(0),
"repl_getmores_per_sec": int64(0),
"repl_inserts": int64(0),
"repl_inserts_per_sec": int64(0),
"repl_queries": int64(0),
"repl_queries_per_sec": int64(0),
"repl_updates": int64(0),
"repl_updates_per_sec": int64(0),
"repl_lag": int64(0),
"repl_oplog_window_sec": int64(0),
"resident_megabytes": int64(0),
"updates": int64(0),
"updates_per_sec": int64(0),
"vsize_megabytes": int64(0),
"ttl_deletes": int64(0),
"ttl_deletes_per_sec": int64(0),
"ttl_passes": int64(0),
"ttl_passes_per_sec": int64(0),
"jumbo_chunks": int64(0),
"total_in_use": int64(0),
"total_available": int64(0),
"total_created": int64(0),
"total_refreshing": int64(0),
"cursor_timed_out": int64(0),
"cursor_timed_out_count": int64(0),
"cursor_no_timeout": int64(0),
"cursor_no_timeout_count": int64(0),
"cursor_pinned": int64(0),
"cursor_pinned_count": int64(0),
"cursor_total": int64(0),
"cursor_total_count": int64(0),
"document_deleted": int64(0),
"document_inserted": int64(0),
"document_returned": int64(0),
Expand Down
100 changes: 57 additions & 43 deletions plugins/inputs/mongodb/mongostat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -457,14 +457,22 @@ type StatLine struct {
LastPrinted time.Time

// Opcounter fields
Insert, Query, Update, Delete, GetMore, Command int64
Insert, InsertCnt int64
Query, QueryCnt int64
Update, UpdateCnt int64
Delete, DeleteCnt int64
GetMore, GetMoreCnt int64
Command, CommandCnt int64

// TTL fields
Passes, DeletedDocuments int64
Passes, PassesCnt int64
DeletedDocuments, DeletedDocumentsCnt int64

// Cursor fields
TimedOutC int64
NoTimeoutC, PinnedC, TotalC int64
TimedOutC, TimedOutCCnt int64
NoTimeoutC, NoTimeoutCCnt int64
PinnedC, PinnedCCnt int64
TotalC, TotalCCnt int64

// Document fields
DeletedD, InsertedD, ReturnedD, UpdatedD int64
Expand Down Expand Up @@ -494,20 +502,26 @@ type StatLine struct {
WorkerThreadEvictingPages int64

// Replicated Opcounter fields
InsertR, QueryR, UpdateR, DeleteR, GetMoreR, CommandR int64
ReplLag int64
OplogTimeDiff int64
Flushes int64
Mapped, Virtual, Resident, NonMapped int64
Faults int64
HighestLocked *LockStatus
QueuedReaders, QueuedWriters int64
ActiveReaders, ActiveWriters int64
NetIn, NetOut int64
NumConnections int64
ReplSetName string
NodeType string
NodeState string
InsertR, InsertRCnt int64
QueryR, QueryRCnt int64
UpdateR, UpdateRCnt int64
DeleteR, DeleteRCnt int64
GetMoreR, GetMoreRCnt int64
CommandR, CommandRCnt int64
ReplLag int64
OplogTimeDiff int64
Flushes, FlushesCnt int64
Mapped, Virtual, Resident, NonMapped int64
Faults, FaultsCnt int64
HighestLocked *LockStatus
QueuedReaders, QueuedWriters int64
ActiveReaders, ActiveWriters int64
NetIn, NetInCnt int64
NetOut, NetOutCnt int64
NumConnections int64
ReplSetName string
NodeType string
NodeState string

// Cluster fields
JumboChunksCount int64
Expand Down Expand Up @@ -576,12 +590,12 @@ func computeLockDiffs(prevLocks, curLocks map[string]LockUsage) []LockUsage {
return lockUsages
}

func diff(newVal, oldVal, sampleTime int64) int64 {
func diff(newVal, oldVal, sampleTime int64) (int64, int64) {
d := newVal - oldVal
if d < 0 {
d = newVal
}
return d / sampleTime
return d / sampleTime, newVal
}

// NewStatLine constructs a StatLine object from two MongoStatus objects.
Expand Down Expand Up @@ -612,25 +626,25 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
}

if newStat.Opcounters != nil && oldStat.Opcounters != nil {
returnVal.Insert = diff(newStat.Opcounters.Insert, oldStat.Opcounters.Insert, sampleSecs)
returnVal.Query = diff(newStat.Opcounters.Query, oldStat.Opcounters.Query, sampleSecs)
returnVal.Update = diff(newStat.Opcounters.Update, oldStat.Opcounters.Update, sampleSecs)
returnVal.Delete = diff(newStat.Opcounters.Delete, oldStat.Opcounters.Delete, sampleSecs)
returnVal.GetMore = diff(newStat.Opcounters.GetMore, oldStat.Opcounters.GetMore, sampleSecs)
returnVal.Command = diff(newStat.Opcounters.Command, oldStat.Opcounters.Command, sampleSecs)
returnVal.Insert, returnVal.InsertCnt = diff(newStat.Opcounters.Insert, oldStat.Opcounters.Insert, sampleSecs)
returnVal.Query, returnVal.QueryCnt = diff(newStat.Opcounters.Query, oldStat.Opcounters.Query, sampleSecs)
returnVal.Update, returnVal.UpdateCnt = diff(newStat.Opcounters.Update, oldStat.Opcounters.Update, sampleSecs)
returnVal.Delete, returnVal.DeleteCnt = diff(newStat.Opcounters.Delete, oldStat.Opcounters.Delete, sampleSecs)
returnVal.GetMore, returnVal.GetMoreCnt = diff(newStat.Opcounters.GetMore, oldStat.Opcounters.GetMore, sampleSecs)
returnVal.Command, returnVal.CommandCnt = diff(newStat.Opcounters.Command, oldStat.Opcounters.Command, sampleSecs)
}

if newStat.Metrics != nil && oldStat.Metrics != nil {
if newStat.Metrics.TTL != nil && oldStat.Metrics.TTL != nil {
returnVal.Passes = diff(newStat.Metrics.TTL.Passes, oldStat.Metrics.TTL.Passes, sampleSecs)
returnVal.DeletedDocuments = diff(newStat.Metrics.TTL.DeletedDocuments, oldStat.Metrics.TTL.DeletedDocuments, sampleSecs)
returnVal.Passes, returnVal.PassesCnt = diff(newStat.Metrics.TTL.Passes, oldStat.Metrics.TTL.Passes, sampleSecs)
returnVal.DeletedDocuments, returnVal.DeletedDocumentsCnt = diff(newStat.Metrics.TTL.DeletedDocuments, oldStat.Metrics.TTL.DeletedDocuments, sampleSecs)
}
if newStat.Metrics.Cursor != nil && oldStat.Metrics.Cursor != nil {
returnVal.TimedOutC = diff(newStat.Metrics.Cursor.TimedOut, oldStat.Metrics.Cursor.TimedOut, sampleSecs)
returnVal.TimedOutC, returnVal.TimedOutCCnt = diff(newStat.Metrics.Cursor.TimedOut, oldStat.Metrics.Cursor.TimedOut, sampleSecs)
if newStat.Metrics.Cursor.Open != nil && oldStat.Metrics.Cursor.Open != nil {
returnVal.NoTimeoutC = diff(newStat.Metrics.Cursor.Open.NoTimeout, oldStat.Metrics.Cursor.Open.NoTimeout, sampleSecs)
returnVal.PinnedC = diff(newStat.Metrics.Cursor.Open.Pinned, oldStat.Metrics.Cursor.Open.Pinned, sampleSecs)
returnVal.TotalC = diff(newStat.Metrics.Cursor.Open.Total, oldStat.Metrics.Cursor.Open.Total, sampleSecs)
returnVal.NoTimeoutC, returnVal.NoTimeoutCCnt = diff(newStat.Metrics.Cursor.Open.NoTimeout, oldStat.Metrics.Cursor.Open.NoTimeout, sampleSecs)
returnVal.PinnedC, returnVal.PinnedCCnt = diff(newStat.Metrics.Cursor.Open.Pinned, oldStat.Metrics.Cursor.Open.Pinned, sampleSecs)
returnVal.TotalC, returnVal.TotalCCnt = diff(newStat.Metrics.Cursor.Open.Total, oldStat.Metrics.Cursor.Open.Total, sampleSecs)
}
}
if newStat.Metrics.Document != nil {
Expand All @@ -642,18 +656,18 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
}

if newStat.OpcountersRepl != nil && oldStat.OpcountersRepl != nil {
returnVal.InsertR = diff(newStat.OpcountersRepl.Insert, oldStat.OpcountersRepl.Insert, sampleSecs)
returnVal.QueryR = diff(newStat.OpcountersRepl.Query, oldStat.OpcountersRepl.Query, sampleSecs)
returnVal.UpdateR = diff(newStat.OpcountersRepl.Update, oldStat.OpcountersRepl.Update, sampleSecs)
returnVal.DeleteR = diff(newStat.OpcountersRepl.Delete, oldStat.OpcountersRepl.Delete, sampleSecs)
returnVal.GetMoreR = diff(newStat.OpcountersRepl.GetMore, oldStat.OpcountersRepl.GetMore, sampleSecs)
returnVal.CommandR = diff(newStat.OpcountersRepl.Command, oldStat.OpcountersRepl.Command, sampleSecs)
returnVal.InsertR, returnVal.InsertRCnt = diff(newStat.OpcountersRepl.Insert, oldStat.OpcountersRepl.Insert, sampleSecs)
returnVal.QueryR, returnVal.QueryRCnt = diff(newStat.OpcountersRepl.Query, oldStat.OpcountersRepl.Query, sampleSecs)
returnVal.UpdateR, returnVal.UpdateRCnt = diff(newStat.OpcountersRepl.Update, oldStat.OpcountersRepl.Update, sampleSecs)
returnVal.DeleteR, returnVal.DeleteRCnt = diff(newStat.OpcountersRepl.Delete, oldStat.OpcountersRepl.Delete, sampleSecs)
returnVal.GetMoreR, returnVal.GetMoreRCnt = diff(newStat.OpcountersRepl.GetMore, oldStat.OpcountersRepl.GetMore, sampleSecs)
returnVal.CommandR, returnVal.CommandRCnt = diff(newStat.OpcountersRepl.Command, oldStat.OpcountersRepl.Command, sampleSecs)
}

returnVal.CacheDirtyPercent = -1
returnVal.CacheUsedPercent = -1
if newStat.WiredTiger != nil && oldStat.WiredTiger != nil {
returnVal.Flushes = newStat.WiredTiger.Transaction.TransCheckpoints - oldStat.WiredTiger.Transaction.TransCheckpoints
returnVal.Flushes, returnVal.FlushesCnt = diff(newStat.WiredTiger.Transaction.TransCheckpoints, oldStat.WiredTiger.Transaction.TransCheckpoints, sampleSecs)
returnVal.CacheDirtyPercent = float64(newStat.WiredTiger.Cache.TrackedDirtyBytes) / float64(newStat.WiredTiger.Cache.MaxBytesConfigured)
returnVal.CacheUsedPercent = float64(newStat.WiredTiger.Cache.CurrentCachedBytes) / float64(newStat.WiredTiger.Cache.MaxBytesConfigured)

Expand All @@ -670,7 +684,7 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
returnVal.ServerEvictingPages = newStat.WiredTiger.Cache.ServerEvictingPages
returnVal.WorkerThreadEvictingPages = newStat.WiredTiger.Cache.WorkerThreadEvictingPages
} else if newStat.BackgroundFlushing != nil && oldStat.BackgroundFlushing != nil {
returnVal.Flushes = newStat.BackgroundFlushing.Flushes - oldStat.BackgroundFlushing.Flushes
returnVal.Flushes, returnVal.FlushesCnt = diff(newStat.BackgroundFlushing.Flushes, oldStat.BackgroundFlushing.Flushes, sampleSecs)
}

returnVal.Time = newMongo.SampleTime
Expand Down Expand Up @@ -713,7 +727,7 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec

if oldStat.ExtraInfo != nil && newStat.ExtraInfo != nil &&
oldStat.ExtraInfo.PageFaults != nil && newStat.ExtraInfo.PageFaults != nil {
returnVal.Faults = diff(*(newStat.ExtraInfo.PageFaults), *(oldStat.ExtraInfo.PageFaults), sampleSecs)
returnVal.Faults, returnVal.FaultsCnt = diff(*(newStat.ExtraInfo.PageFaults), *(oldStat.ExtraInfo.PageFaults), sampleSecs)
}
if !returnVal.IsMongos && oldStat.Locks != nil {
globalCheck, hasGlobal := oldStat.Locks["Global"]
Expand Down Expand Up @@ -812,8 +826,8 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec
}

if oldStat.Network != nil && newStat.Network != nil {
returnVal.NetIn = diff(newStat.Network.BytesIn, oldStat.Network.BytesIn, sampleSecs)
returnVal.NetOut = diff(newStat.Network.BytesOut, oldStat.Network.BytesOut, sampleSecs)
returnVal.NetIn, returnVal.NetInCnt = diff(newStat.Network.BytesIn, oldStat.Network.BytesIn, sampleSecs)
returnVal.NetOut, returnVal.NetOutCnt = diff(newStat.Network.BytesOut, oldStat.Network.BytesOut, sampleSecs)
}

if newStat.Connections != nil {
Expand Down

0 comments on commit 4b3580c

Please sign in to comment.