From aed4831afd215c8b230cda3836e71712fdf48389 Mon Sep 17 00:00:00 2001 From: Alex K <8418476+fearful-symmetry@users.noreply.github.com> Date: Tue, 6 Oct 2020 12:02:29 -0700 Subject: [PATCH] Remove nil-zero metrics and linux-exclusive metrics from Metricbeat (#21457) * refactor metricbeat to remove nil-zero metrics and linux-exclusive metrics * update xpack docs * fix non-linux diskstat builds * fix linux test builds * fix python tests * move windows files for disk performance * properly fix test_drop_fields * try to fix different system test * mage fmt * fix windows filesystem tests * fix platform test * add changelog --- CHANGELOG.next.asciidoc | 1 + .../system/diskio/disk_performance_386.go | 0 .../system/diskio/disk_performance_amd64.go | 0 .../metric}/system/diskio/diskstat.go | 11 +- .../metric}/system/diskio/diskstat_linux.go | 81 ++-- .../system/diskio/diskstat_linux_test.go | 60 +-- .../metric}/system/diskio/diskstat_other.go | 14 +- .../metric}/system/diskio/diskstat_windows.go | 14 +- .../system/diskio/diskstat_windows_helper.go | 0 .../system/diskio/diskstat_windows_test.go | 0 metricbeat/docs/fields.asciidoc | 354 +++++++++++++++++- metricbeat/docs/modules/linux.asciidoc | 10 + metricbeat/docs/modules/linux/iostat.asciidoc | 23 ++ metricbeat/docs/modules/linux/memory.asciidoc | 24 ++ metricbeat/docs/modules_list.asciidoc | 4 +- metricbeat/include/list_common.go | 2 + metricbeat/metricbeat.reference.yml | 2 + metricbeat/module/linux/_meta/config.yml | 2 + metricbeat/module/linux/_meta/fields.yml | 1 + metricbeat/module/linux/fields.go | 2 +- .../module/linux/iostat/_meta/data.json | 49 +++ .../module/linux/iostat/_meta/docs.asciidoc | 3 + .../module/linux/iostat/_meta/fields.yml | 61 +++ metricbeat/module/linux/iostat/data.go | 58 +++ metricbeat/module/linux/iostat/iostat.go | 104 +++++ metricbeat/module/linux/iostat/iostat_test.go | 55 +++ .../module/linux/memory/_meta/data.json | 59 +++ .../module/linux/memory/_meta/docs.asciidoc | 3 + .../module/linux/memory/_meta/fields.yml | 78 ++++ metricbeat/module/linux/memory/data.go | 99 +++++ metricbeat/module/linux/memory/memory.go | 62 +++ metricbeat/module/linux/memory/memory_test.go | 55 +++ metricbeat/module/system/cpu/_meta/data.json | 44 +-- metricbeat/module/system/cpu/cpu.go | 48 +-- metricbeat/module/system/cpu/data.go | 112 ++++++ .../module/system/diskio/_meta/data.json | 16 +- metricbeat/module/system/diskio/diskio.go | 57 ++- .../module/system/diskio/diskio_test.go | 40 ++ metricbeat/module/system/fields.go | 2 +- .../module/system/filesystem/_meta/data.json | 27 +- metricbeat/module/system/filesystem/helper.go | 9 +- .../module/system/fsstat/_meta/fields.yml | 2 +- metricbeat/module/system/fsstat/fsstat.go | 26 +- .../module/system/memory/_meta/data.json | 42 +-- metricbeat/module/system/memory/memory.go | 81 +--- .../module/system/process/_meta/data.json | 132 +++---- .../module/system/process/_meta/fields.yml | 16 +- metricbeat/module/system/process/process.go | 36 +- .../system/process_summary/_meta/data.json | 19 +- .../system/process_summary/process_summary.go | 32 +- .../process_summary/process_summary_test.go | 31 +- metricbeat/module/system/system.go | 38 +- metricbeat/module/system/test_system.py | 59 ++- metricbeat/modules.d/linux.yml.disabled | 2 + metricbeat/tests/system/test_processors.py | 15 +- x-pack/metricbeat/metricbeat.reference.yml | 2 + 56 files changed, 1700 insertions(+), 479 deletions(-) rename {metricbeat/module => libbeat/metric}/system/diskio/disk_performance_386.go (100%) rename {metricbeat/module => libbeat/metric}/system/diskio/disk_performance_amd64.go (100%) rename {metricbeat/module => libbeat/metric}/system/diskio/diskstat.go (88%) rename {metricbeat/module => libbeat/metric}/system/diskio/diskstat_linux.go (55%) rename {metricbeat/module => libbeat/metric}/system/diskio/diskstat_linux_test.go (55%) rename {metricbeat/module => libbeat/metric}/system/diskio/diskstat_other.go (79%) rename {metricbeat/module => libbeat/metric}/system/diskio/diskstat_windows.go (78%) rename {metricbeat/module => libbeat/metric}/system/diskio/diskstat_windows_helper.go (100%) rename {metricbeat/module => libbeat/metric}/system/diskio/diskstat_windows_test.go (100%) create mode 100644 metricbeat/docs/modules/linux/iostat.asciidoc create mode 100644 metricbeat/docs/modules/linux/memory.asciidoc create mode 100644 metricbeat/module/linux/iostat/_meta/data.json create mode 100644 metricbeat/module/linux/iostat/_meta/docs.asciidoc create mode 100644 metricbeat/module/linux/iostat/_meta/fields.yml create mode 100644 metricbeat/module/linux/iostat/data.go create mode 100644 metricbeat/module/linux/iostat/iostat.go create mode 100644 metricbeat/module/linux/iostat/iostat_test.go create mode 100644 metricbeat/module/linux/memory/_meta/data.json create mode 100644 metricbeat/module/linux/memory/_meta/docs.asciidoc create mode 100644 metricbeat/module/linux/memory/_meta/fields.yml create mode 100644 metricbeat/module/linux/memory/data.go create mode 100644 metricbeat/module/linux/memory/memory.go create mode 100644 metricbeat/module/linux/memory/memory_test.go create mode 100644 metricbeat/module/system/cpu/data.go diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3099a033153..6b4a999e0ec 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -93,6 +93,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Move service config under metrics and simplify metric types. {pull}18691[18691] - Fix ECS compliance of user.id field in system/users metricset {pull}19019[19019] - Rename googlecloud stackdriver metricset to metrics. {pull}19718[19718] +- Remove "invalid zero" metrics on Windows and Darwin, don't report linux-only memory and diskio metrics when running under agent. {pull}21457[21457] *Packetbeat* diff --git a/metricbeat/module/system/diskio/disk_performance_386.go b/libbeat/metric/system/diskio/disk_performance_386.go similarity index 100% rename from metricbeat/module/system/diskio/disk_performance_386.go rename to libbeat/metric/system/diskio/disk_performance_386.go diff --git a/metricbeat/module/system/diskio/disk_performance_amd64.go b/libbeat/metric/system/diskio/disk_performance_amd64.go similarity index 100% rename from metricbeat/module/system/diskio/disk_performance_amd64.go rename to libbeat/metric/system/diskio/disk_performance_amd64.go diff --git a/metricbeat/module/system/diskio/diskstat.go b/libbeat/metric/system/diskio/diskstat.go similarity index 88% rename from metricbeat/module/system/diskio/diskstat.go rename to libbeat/metric/system/diskio/diskstat.go index 949d69778fa..3cea37c552a 100644 --- a/metricbeat/module/system/diskio/diskstat.go +++ b/libbeat/metric/system/diskio/diskstat.go @@ -25,11 +25,11 @@ import ( sigar "github.com/elastic/gosigar" ) -// mapping fields which output by `iostat -x` on linux +// IOMetric contains mapping fields which are outputed by `iostat -x` on linux // // Device: rrqm/s wrqm/s r/s w/s rsec/s wsec/s avgrq-sz avgqu-sz await r_await w_await svctm %util // sda 0.06 0.78 0.09 0.27 9.42 8.06 48.64 0.00 1.34 0.99 1.45 0.77 0.03 -type DiskIOMetric struct { +type IOMetric struct { ReadRequestMergeCountPerSec float64 `json:"rrqmCps"` WriteRequestMergeCountPerSec float64 `json:"wrqmCps"` ReadRequestCountPerSec float64 `json:"rrqCps"` @@ -46,8 +46,9 @@ type DiskIOMetric struct { BusyPct float64 `json:"busy"` } -type DiskIOStat struct { +// IOStat carries disk statistics for all devices +type IOStat struct { lastDiskIOCounters map[string]disk.IOCountersStat - lastCpu sigar.Cpu - curCpu sigar.Cpu + lastCPU sigar.Cpu + curCPU sigar.Cpu } diff --git a/metricbeat/module/system/diskio/diskstat_linux.go b/libbeat/metric/system/diskio/diskstat_linux.go similarity index 55% rename from metricbeat/module/system/diskio/diskstat_linux.go rename to libbeat/metric/system/diskio/diskstat_linux.go index 4ecfdf700ac..826aed78c27 100644 --- a/metricbeat/module/system/diskio/diskstat_linux.go +++ b/libbeat/metric/system/diskio/diskstat_linux.go @@ -26,7 +26,8 @@ import ( "github.com/elastic/beats/v7/libbeat/metric/system/cpu" ) -func Get_CLK_TCK() uint32 { +// GetCLKTCK emulates the _SC_CLK_TCK syscall +func GetCLKTCK() uint32 { // return uint32(C.sysconf(C._SC_CLK_TCK)) // NOTE: _SC_CLK_TCK should be fetched from sysconf using cgo return uint32(100) @@ -38,77 +39,78 @@ func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { } // NewDiskIOStat :init DiskIOStat object. -func NewDiskIOStat() *DiskIOStat { - return &DiskIOStat{ +func NewDiskIOStat() *IOStat { + return &IOStat{ lastDiskIOCounters: map[string]disk.IOCountersStat{}, } } // OpenSampling creates current cpu sampling // need call as soon as get IOCounters. -func (stat *DiskIOStat) OpenSampling() error { - return stat.curCpu.Get() +func (stat *IOStat) OpenSampling() error { + return stat.curCPU.Get() } -// CalIOStatistics calculates IO statistics. -func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { +// CalcIOStatistics calculates IO statistics. +func (stat *IOStat) CalcIOStatistics(counter disk.IOCountersStat) (IOMetric, error) { var last disk.IOCountersStat var ok bool // if last counter not found, create one and return all 0 if last, ok = stat.lastDiskIOCounters[counter.Name]; !ok { stat.lastDiskIOCounters[counter.Name] = counter - return nil + return IOMetric{}, nil } // calculate the delta ms between the CloseSampling and OpenSampling - deltams := 1000.0 * float64(stat.curCpu.Total()-stat.lastCpu.Total()) / float64(cpu.NumCores) / float64(Get_CLK_TCK()) + deltams := 1000.0 * float64(stat.curCPU.Total()-stat.lastCPU.Total()) / float64(cpu.NumCores) / float64(GetCLKTCK()) if deltams <= 0 { - return errors.New("The delta cpu time between close sampling and open sampling is less or equal to 0") + return IOMetric{}, errors.New("The delta cpu time between close sampling and open sampling is less or equal to 0") } - rd_ios := counter.ReadCount - last.ReadCount - rd_merges := counter.MergedReadCount - last.MergedReadCount - rd_bytes := counter.ReadBytes - last.ReadBytes - rd_ticks := counter.ReadTime - last.ReadTime - wr_ios := counter.WriteCount - last.WriteCount - wr_merges := counter.MergedWriteCount - last.MergedWriteCount - wr_bytes := counter.WriteBytes - last.WriteBytes - wr_ticks := counter.WriteTime - last.WriteTime + rdIOs := counter.ReadCount - last.ReadCount + rdMerges := counter.MergedReadCount - last.MergedReadCount + rdBytes := counter.ReadBytes - last.ReadBytes + rdTicks := counter.ReadTime - last.ReadTime + wrIOs := counter.WriteCount - last.WriteCount + wrMerges := counter.MergedWriteCount - last.MergedWriteCount + wrBytes := counter.WriteBytes - last.WriteBytes + wrTicks := counter.WriteTime - last.WriteTime ticks := counter.IoTime - last.IoTime aveq := counter.WeightedIO - last.WeightedIO - n_ios := rd_ios + wr_ios - n_ticks := rd_ticks + wr_ticks - n_bytes := rd_bytes + wr_bytes + nIOs := rdIOs + wrIOs + nTicks := rdTicks + wrTicks + nBytes := rdBytes + wrBytes size := float64(0) wait := float64(0) svct := float64(0) - if n_ios > 0 { - size = float64(n_bytes) / float64(n_ios) - wait = float64(n_ticks) / float64(n_ios) - svct = float64(ticks) / float64(n_ios) + if nIOs > 0 { + size = float64(nBytes) / float64(nIOs) + wait = float64(nTicks) / float64(nIOs) + svct = float64(ticks) / float64(nIOs) } queue := float64(aveq) / deltams - per_sec := func(x uint64) float64 { + perSec := func(x uint64) float64 { return 1000.0 * float64(x) / deltams } - result.ReadRequestMergeCountPerSec = per_sec(rd_merges) - result.WriteRequestMergeCountPerSec = per_sec(wr_merges) - result.ReadRequestCountPerSec = per_sec(rd_ios) - result.WriteRequestCountPerSec = per_sec(wr_ios) - result.ReadBytesPerSec = per_sec(rd_bytes) - result.WriteBytesPerSec = per_sec(wr_bytes) + result := IOMetric{} + result.ReadRequestMergeCountPerSec = perSec(rdMerges) + result.WriteRequestMergeCountPerSec = perSec(wrMerges) + result.ReadRequestCountPerSec = perSec(rdIOs) + result.WriteRequestCountPerSec = perSec(wrIOs) + result.ReadBytesPerSec = perSec(rdBytes) + result.WriteBytesPerSec = perSec(wrBytes) result.AvgRequestSize = size result.AvgQueueSize = queue result.AvgAwaitTime = wait - if rd_ios > 0 { - result.AvgReadAwaitTime = float64(rd_ticks) / float64(rd_ios) + if rdIOs > 0 { + result.AvgReadAwaitTime = float64(rdTicks) / float64(rdIOs) } - if wr_ios > 0 { - result.AvgWriteAwaitTime = float64(wr_ticks) / float64(wr_ios) + if wrIOs > 0 { + result.AvgWriteAwaitTime = float64(wrTicks) / float64(wrIOs) } result.AvgServiceTime = svct result.BusyPct = 100.0 * float64(ticks) / deltams @@ -117,10 +119,11 @@ func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCou } stat.lastDiskIOCounters[counter.Name] = counter - return nil + return result, nil } -func (stat *DiskIOStat) CloseSampling() { - stat.lastCpu = stat.curCpu +// CloseSampling closes the disk sampler +func (stat *IOStat) CloseSampling() { + stat.lastCPU = stat.curCPU } diff --git a/metricbeat/module/system/diskio/diskstat_linux_test.go b/libbeat/metric/system/diskio/diskstat_linux_test.go similarity index 55% rename from metricbeat/module/system/diskio/diskstat_linux_test.go rename to libbeat/metric/system/diskio/diskstat_linux_test.go index 56a8d9dc7cc..74c0bef231e 100644 --- a/metricbeat/module/system/diskio/diskstat_linux_test.go +++ b/libbeat/metric/system/diskio/diskstat_linux_test.go @@ -27,53 +27,11 @@ import ( "github.com/stretchr/testify/assert" sigar "github.com/elastic/gosigar" - - mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" - "github.com/elastic/beats/v7/metricbeat/module/system" ) -func Test_Get_CLK_TCK(t *testing.T) { +func Test_GetCLKTCK(t *testing.T) { //usually the tick is 100 - assert.Equal(t, uint32(100), Get_CLK_TCK()) -} - -func TestDataNameFilter(t *testing.T) { - oldFS := system.HostFS - newFS := "_meta/testdata" - system.HostFS = &newFS - defer func() { - system.HostFS = oldFS - }() - - conf := map[string]interface{}{ - "module": "system", - "metricsets": []string{"diskio"}, - "diskio.include_devices": []string{"sda", "sda1", "sda2"}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, conf) - data, errs := mbtest.ReportingFetchV2Error(f) - assert.Empty(t, errs) - assert.Equal(t, 3, len(data)) -} - -func TestDataEmptyFilter(t *testing.T) { - oldFS := system.HostFS - newFS := "_meta/testdata" - system.HostFS = &newFS - defer func() { - system.HostFS = oldFS - }() - - conf := map[string]interface{}{ - "module": "system", - "metricsets": []string{"diskio"}, - } - - f := mbtest.NewReportingMetricSetV2Error(t, conf) - data, errs := mbtest.ReportingFetchV2Error(f) - assert.Empty(t, errs) - assert.Equal(t, 10, len(data)) + assert.Equal(t, uint32(100), GetCLKTCK()) } func TestDiskIOStat_CalIOStatistics(t *testing.T) { @@ -85,9 +43,9 @@ func TestDiskIOStat_CalIOStatistics(t *testing.T) { Name: "iostat", } - stat := &DiskIOStat{ + stat := &IOStat{ lastDiskIOCounters: map[string]disk.IOCountersStat{ - "iostat": disk.IOCountersStat{ + "iostat": { ReadCount: 3, WriteCount: 5, ReadTime: 7, @@ -95,17 +53,17 @@ func TestDiskIOStat_CalIOStatistics(t *testing.T) { Name: "iostat", }, }, - lastCpu: sigar.Cpu{Idle: 100}, - curCpu: sigar.Cpu{Idle: 1}, + lastCPU: sigar.Cpu{Idle: 100}, + curCPU: sigar.Cpu{Idle: 1}, } - expected := DiskIOMetric{ + expected := IOMetric{ AvgAwaitTime: 24.0 / 22.0, AvgReadAwaitTime: 1.2, AvgWriteAwaitTime: 1, } - var got DiskIOMetric - err := stat.CalIOStatistics(&got, counter) + + got, err := stat.CalcIOStatistics(counter) if err != nil { t.Fatal(err) } diff --git a/metricbeat/module/system/diskio/diskstat_other.go b/libbeat/metric/system/diskio/diskstat_other.go similarity index 79% rename from metricbeat/module/system/diskio/diskstat_other.go rename to libbeat/metric/system/diskio/diskstat_other.go index d2669117d9e..ad5208b1b4f 100644 --- a/metricbeat/module/system/diskio/diskstat_other.go +++ b/libbeat/metric/system/diskio/diskstat_other.go @@ -25,24 +25,24 @@ import ( ) // NewDiskIOStat :init DiskIOStat object. -func NewDiskIOStat() *DiskIOStat { - return &DiskIOStat{ +func NewDiskIOStat() *IOStat { + return &IOStat{ lastDiskIOCounters: map[string]disk.IOCountersStat{}, } } // OpenSampling stub for linux implementation. -func (stat *DiskIOStat) OpenSampling() error { +func (stat *IOStat) OpenSampling() error { return nil } -// CalIOStatistics stub for linux implementation. -func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { - return errors.New("not implemented out of linux") +// CalcIOStatistics stub for linux implementation. +func (stat *IOStat) CalcIOStatistics(rcounter disk.IOCountersStat) (IOMetric, error) { + return IOMetric{}, errors.New("not implemented out of linux") } // CloseSampling stub for linux implementation. -func (stat *DiskIOStat) CloseSampling() {} +func (stat *IOStat) CloseSampling() {} // IOCounters should map functionality to disk package for linux os. func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { diff --git a/metricbeat/module/system/diskio/diskstat_windows.go b/libbeat/metric/system/diskio/diskstat_windows.go similarity index 78% rename from metricbeat/module/system/diskio/diskstat_windows.go rename to libbeat/metric/system/diskio/diskstat_windows.go index 1e2b363074b..665caab9130 100644 --- a/metricbeat/module/system/diskio/diskstat_windows.go +++ b/libbeat/metric/system/diskio/diskstat_windows.go @@ -25,24 +25,24 @@ import ( ) // NewDiskIOStat :init DiskIOStat object. -func NewDiskIOStat() *DiskIOStat { - return &DiskIOStat{ +func NewDiskIOStat() *IOStat { + return &IOStat{ lastDiskIOCounters: map[string]disk.IOCountersStat{}, } } // OpenSampling stub for linux implementation. -func (stat *DiskIOStat) OpenSampling() error { +func (stat *IOStat) OpenSampling() error { return nil } -// CalIOStatistics stub for linux implementation. -func (stat *DiskIOStat) CalIOStatistics(result *DiskIOMetric, counter disk.IOCountersStat) error { - return errors.New("iostat is not implement for Windows") +// CalcIOStatistics stub for linux implementation. +func (stat *IOStat) CalcIOStatistics(counter disk.IOCountersStat) (IOMetric, error) { + return IOMetric{}, errors.New("iostat is not implement for Windows") } // CloseSampling stub for linux implementation. -func (stat *DiskIOStat) CloseSampling() {} +func (stat *IOStat) CloseSampling() {} // IOCounters should map functionality to disk package for linux os. func IOCounters(names ...string) (map[string]disk.IOCountersStat, error) { diff --git a/metricbeat/module/system/diskio/diskstat_windows_helper.go b/libbeat/metric/system/diskio/diskstat_windows_helper.go similarity index 100% rename from metricbeat/module/system/diskio/diskstat_windows_helper.go rename to libbeat/metric/system/diskio/diskstat_windows_helper.go diff --git a/metricbeat/module/system/diskio/diskstat_windows_test.go b/libbeat/metric/system/diskio/diskstat_windows_test.go similarity index 100% rename from metricbeat/module/system/diskio/diskstat_windows_test.go rename to libbeat/metric/system/diskio/diskstat_windows_test.go diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index dc7c0c45d3e..7f1d0f69ab9 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -29167,6 +29167,7 @@ linux module [float] === linux +linux system metrics @@ -29264,6 +29265,147 @@ type: long -- +[float] +=== iostat + +iostat + + + +*`linux.iostat.read.request.merges_per_sec`*:: ++ +-- +The number of read requests merged per second that were queued to the device. + + +type: float + +-- + +*`linux.iostat.write.request.merges_per_sec`*:: ++ +-- +The number of write requests merged per second that were queued to the device. + + +type: float + +-- + +*`linux.iostat.read.request.per_sec`*:: ++ +-- +The number of read requests that were issued to the device per second + + +type: float + +-- + +*`linux.iostat.write.request.per_sec`*:: ++ +-- +The number of write requests that were issued to the device per second + + +type: float + +-- + +*`linux.iostat.read.per_sec.bytes`*:: ++ +-- +The number of Bytes read from the device per second. + + +type: float + +format: bytes + +-- + +*`linux.iostat.read.await`*:: ++ +-- +The average time spent for read requests issued to the device to be served. + + +type: float + +-- + +*`linux.iostat.write.per_sec.bytes`*:: ++ +-- +The number of Bytes write from the device per second. + + +type: float + +format: bytes + +-- + +*`linux.iostat.write.await`*:: ++ +-- +The average time spent for write requests issued to the device to be served. + + +type: float + +-- + +*`linux.iostat.request.avg_size`*:: ++ +-- +The average size (in bytes) of the requests that were issued to the device. + + +type: float + +-- + +*`linux.iostat.queue.avg_size`*:: ++ +-- +The average queue length of the requests that were issued to the device. + + +type: float + +-- + +*`linux.iostat.await`*:: ++ +-- +The average time spent for requests issued to the device to be served. + + +type: float + +-- + +*`linux.iostat.service_time`*:: ++ +-- +The average service time (in milliseconds) for I/O requests that were issued to the device. + + +type: float + +-- + +*`linux.iostat.busy`*:: ++ +-- +Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%. + + +type: float + +-- + [float] === ksm @@ -29338,6 +29480,186 @@ type: long -- +[float] +=== memory + +Linux memory data + + + +[float] +=== page_stats + +memory page statistics + + +*`linux.memory.page_stats.pgscan_kswapd.pages`*:: ++ +-- +pages scanned by kswapd + +type: long + +format: number + +-- + +*`linux.memory.page_stats.pgscan_direct.pages`*:: ++ +-- +pages scanned directly + +type: long + +format: number + +-- + +*`linux.memory.page_stats.pgfree.pages`*:: ++ +-- +pages freed by the system + +type: long + +format: number + +-- + +*`linux.memory.page_stats.pgsteal_kswapd.pages`*:: ++ +-- +number of pages reclaimed by kswapd + +type: long + +format: number + +-- + +*`linux.memory.page_stats.pgsteal_direct.pages`*:: ++ +-- +number of pages reclaimed directly + +type: long + +format: number + +-- + +*`linux.memory.page_stats.direct_efficiency.pct`*:: ++ +-- +direct reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory. + +type: scaled_float + +format: percent + +-- + +*`linux.memory.page_stats.kswapd_efficiency.pct`*:: ++ +-- +kswapd reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory. + +type: scaled_float + +format: percent + +-- + +[float] +=== hugepages + +This group contains statistics related to huge pages usage on the system. + + +*`linux.memory.hugepages.total`*:: ++ +-- +Number of huge pages in the pool. + + +type: long + +format: number + +-- + +*`linux.memory.hugepages.used.bytes`*:: ++ +-- +Memory used in allocated huge pages. + + +type: long + +format: bytes + +-- + +*`linux.memory.hugepages.used.pct`*:: ++ +-- +Percentage of huge pages used. + + +type: long + +format: percent + +-- + +*`linux.memory.hugepages.free`*:: ++ +-- +Number of available huge pages in the pool. + + +type: long + +format: number + +-- + +*`linux.memory.hugepages.reserved`*:: ++ +-- +Number of reserved but not allocated huge pages in the pool. + + +type: long + +format: number + +-- + +*`linux.memory.hugepages.surplus`*:: ++ +-- +Number of overcommited huge pages. + + +type: long + +format: number + +-- + +*`linux.memory.hugepages.default_size`*:: ++ +-- +Default size for huge pages. + + +type: long + +format: bytes + +-- + [float] === pageinfo @@ -40709,7 +41031,7 @@ type: long *`system.fsstat.total_files`*:: + -- -Total number of files. +Total number of files. Not on Windows. type: long @@ -41571,7 +41893,7 @@ format: bytes *`system.process.memory.rss.bytes`*:: + -- -The Resident Set Size. The amount of memory the process occupied in main memory (RAM). On Windows this represents the current working set size, in bytes. +The Resident Set Size. The amount of memory the process occupied in main memory (RAM). On Windows this represents the current working set size, in bytes. Not available on Windows. type: long @@ -41583,7 +41905,31 @@ format: bytes *`system.process.memory.rss.pct`*:: + -- -The percentage of memory the process occupied in main memory (RAM). +The percentage of memory the process occupied in main memory (RAM). Not available on Windows. + + +type: scaled_float + +format: percent + +-- + +*`system.process.memory.wss.bytes`*:: ++ +-- +The Working Set Size. The amount of memory the process occupied in main memory (RAM). Windows only. + + +type: long + +format: bytes + +-- + +*`system.process.memory.wss.pct`*:: ++ +-- +The percentage of memory the process occupied in main memory (RAM). Windows only. type: scaled_float @@ -41595,7 +41941,7 @@ format: percent *`system.process.memory.share`*:: + -- -The shared memory the process uses. +The shared memory the process uses. Not available on Windows. type: long diff --git a/metricbeat/docs/modules/linux.asciidoc b/metricbeat/docs/modules/linux.asciidoc index d63528730cb..ab911885afd 100644 --- a/metricbeat/docs/modules/linux.asciidoc +++ b/metricbeat/docs/modules/linux.asciidoc @@ -22,8 +22,10 @@ metricbeat.modules: period: 10s metricsets: - "pageinfo" + - "memory" # - ksm # - conntrack + # - iostat enabled: true #hostfs: /hostfs @@ -36,13 +38,21 @@ The following metricsets are available: * <> +* <> + * <> +* <> + * <> include::linux/conntrack.asciidoc[] +include::linux/iostat.asciidoc[] + include::linux/ksm.asciidoc[] +include::linux/memory.asciidoc[] + include::linux/pageinfo.asciidoc[] diff --git a/metricbeat/docs/modules/linux/iostat.asciidoc b/metricbeat/docs/modules/linux/iostat.asciidoc new file mode 100644 index 00000000000..7a41297b819 --- /dev/null +++ b/metricbeat/docs/modules/linux/iostat.asciidoc @@ -0,0 +1,23 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-linux-iostat]] +=== linux iostat metricset + +beta[] + +include::../../../module/linux/iostat/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/linux/iostat/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/linux/memory.asciidoc b/metricbeat/docs/modules/linux/memory.asciidoc new file mode 100644 index 00000000000..9ea3d482e57 --- /dev/null +++ b/metricbeat/docs/modules/linux/memory.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-linux-memory]] +=== linux memory metricset + +beta[] + +include::../../../module/linux/memory/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../module/linux/memory/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index 5ff2305665f..b37f7f831a3 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -176,8 +176,10 @@ This file is generated! See scripts/mage/docs_collector.go .2+| .2+| |<> beta[] |<> beta[] |<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | -.3+| .3+| |<> beta[] +.5+| .5+| |<> beta[] +|<> beta[] |<> beta[] +|<> beta[] |<> beta[] |<> |image:./images/icon-no.png[No prebuilt dashboards] | .2+| .2+| |<> diff --git a/metricbeat/include/list_common.go b/metricbeat/include/list_common.go index 39fadeea0e0..b77bb57f9a6 100644 --- a/metricbeat/include/list_common.go +++ b/metricbeat/include/list_common.go @@ -95,7 +95,9 @@ import ( _ "github.com/elastic/beats/v7/metricbeat/module/kvm/status" _ "github.com/elastic/beats/v7/metricbeat/module/linux" _ "github.com/elastic/beats/v7/metricbeat/module/linux/conntrack" + _ "github.com/elastic/beats/v7/metricbeat/module/linux/iostat" _ "github.com/elastic/beats/v7/metricbeat/module/linux/ksm" + _ "github.com/elastic/beats/v7/metricbeat/module/linux/memory" _ "github.com/elastic/beats/v7/metricbeat/module/linux/pageinfo" _ "github.com/elastic/beats/v7/metricbeat/module/logstash" _ "github.com/elastic/beats/v7/metricbeat/module/logstash/node" diff --git a/metricbeat/metricbeat.reference.yml b/metricbeat/metricbeat.reference.yml index 9b6f37eb447..9df18e4d7f9 100644 --- a/metricbeat/metricbeat.reference.yml +++ b/metricbeat/metricbeat.reference.yml @@ -572,8 +572,10 @@ metricbeat.modules: period: 10s metricsets: - "pageinfo" + - "memory" # - ksm # - conntrack + # - iostat enabled: true #hostfs: /hostfs diff --git a/metricbeat/module/linux/_meta/config.yml b/metricbeat/module/linux/_meta/config.yml index e00fe571846..490d3245c19 100644 --- a/metricbeat/module/linux/_meta/config.yml +++ b/metricbeat/module/linux/_meta/config.yml @@ -2,8 +2,10 @@ period: 10s metricsets: - "pageinfo" + - "memory" # - ksm # - conntrack + # - iostat enabled: true #hostfs: /hostfs diff --git a/metricbeat/module/linux/_meta/fields.yml b/metricbeat/module/linux/_meta/fields.yml index c3fe9657e89..44236f625e9 100644 --- a/metricbeat/module/linux/_meta/fields.yml +++ b/metricbeat/module/linux/_meta/fields.yml @@ -7,4 +7,5 @@ - name: linux type: group description: > + linux system metrics fields: diff --git a/metricbeat/module/linux/fields.go b/metricbeat/module/linux/fields.go index e4935eb2d7f..90e7e3ed940 100644 --- a/metricbeat/module/linux/fields.go +++ b/metricbeat/module/linux/fields.go @@ -32,5 +32,5 @@ func init() { // AssetLinux returns asset data. // This is the base64 encoded gzipped contents of module/linux. func AssetLinux() string { - return "eJy8l0tv4zYQx+/+FIMcF91Hsm8fCgTdRVEUKYIGe+mhwogcWawpUuWQybqfviBl2YoiOUpTiZcgIfWfn/6ch/IStrRbg1YmfF8BeOU1reEs/X62AnCkCZnWkJPHFYAkFk7VXlmzhh9XANA8C5WVQdMKoFCkJa/T1kswWNFRPi6/q2kNG2dDvf/LgOZ9ma6UsMZ4h2J72BmSjKuP3q7BcM0aEu+DdGE4VBW63b29MZxHQse1lwNbgCmyAwywR6/YK8E/pDMkAYWzzPDT9TcQ1hH3tIagu+DS2T7bkVxbsxnYfAQ+rhrFljwn+ZokyEDg7dFWKFBpFRyNghE6vctmwjtykPFO0RHUW6hwS+CsraCwDgzdgTUPfO2ANgozULZsyoAvqQPtMdfjzhU2GDkDDgchiLkIWu+ACZ0oSY6+fkujNsYOXPP/l2JMZAC1I5S75BEJ31wk9u65X54dSMPkfBaTkuaw7rdQ5eRiObd3eleSI9CK/T54+6NhgBOot6jVMyEfyj9w1JfoQaAx1kNOkFwc8ObQAFM+ZI7Yo/MzWJhyHrS121BH+5QoocR0zznBPu6x0zTHHbH6p5OcLe2Wqzmmxn3Zk/PCo+/XzH+eFr/eXHUmwxMHQI0b4oxLdLNk/k0SbqLERhaYXk1gUYPxng2jPDHs9Ztoj8EEM5s134z6O9AJjAoPGLdWo1cDTf/5GNfpakSJZhNd8dZCgez3hdW8/bhLcRpkLNDMMQCPbdOrihgqcptU1006lXhLkMdmFQHMKUxO/SAzVlImSlSz4DZOpsaZ0BxhGpEVfs8icZvZ0zBlqOf1VIZaK4GxacYO0svDbhkoU9g5+uWA9qmmmQcpd1nvgXGgCYZ8QY9QOFvB69pZ8TpFiAEasVgPgQ/tK9+BdZLcEzvsl6vL0XscYp7AndivLhNXeomBE2NYXbSzN2eD+4+k2UTCROGIQJTBbDnm3MWfb15cX/78Nbv55Y+vp9HOF0c7n4p2sTjaxVS0t4ujvZ2K9m5xtHdT0d4vjvZ+KtqHxdE+TEX7uDjax6lonxZH+zQV7fPiaJ8nt9zlx8H52DxooeK3EL96MTjxbf4Xif4/mxNIfsc7QK2twHgK0sDvfAXEqRoD3PvSWP0bAAD//4Pi/m8=" + return "eJzEmd9v2zgSx9/zVwwCHNAWVzfp7/rhgNylOBR3uQu27csudoUxObK5pkiVP+y6f/2CpBXLtmQrcaT6pWgkz3z4neHMkH4Oc1qNQQrlv58BOOEkjeE8/v/8DMCQJLQ0hgk5PAPgZJkRpRNajeEfZwCQvguF5l7SGUAuSHI7jo+eg8KCNubDx61KGsPUaF+u/9Jgc2PXrqyjAgpyRjC7flj3UffDtFLOIJvfPWnyFz6766o+LSzh02R8F6QOY31RoFltPWvDOeI6fNbmQOeg8uwOBqxDJ6wTzP49vkMckBltLfzr9iswbcju2GqCroNzo3fZNuRSq2nDwyPw4VMim5Oz0XxJHLgncHojK+QopPCGWsEIjVxlPeFtOEg5I2gD6jQUOCcwWheQawOKlqDVnq410GShB8qKTShwM6pBO5zIduVy7RXvAcd6xsja3Eu5Akto2Ix46/IrGjFVuiHMj5dilkgBSkPIV1EjYi4FEnfivLs9a5DKknFZSErqQ7r/+WJCJmznKqbLGRkCKaxbO6/+SQxwAHWBUpwIuW9+T1E3QwcMldIOJgRRxQZt7gpgzIfMkHVoXA8SxpwHqfXcl0E+wWYwwxjnCcHa76bSpNcNWfGjlpx3IupQSftoHHuWD3WNkLMjQ988WTcqyEzJZiWZzBJr7CS51Lir7VHpvswI1F3+BZewdmkh+uRQkgFLTCuewr4MufnNk0/7KBQfTgvBaNS4jKURjgZeR/T52AvZiseggdjQCmv3aGvr6hCAYZU/jTwqvgYeTVZur5E8GvY/g/Gkem500cw42p2etCnQjWGfbGsBuESxi3ciOC7I4JTAiYLAlqRcnEa2s6ZR8VQQLZkF8UP7dUDVU8o8puxpCcPpvpP0DxS+2qG4mGahMfWDHizDE6GSfE9DGAJlxx3bTB5raM/c0QdIUlM3exToIbflaYkRnglGWTDbU1IkDwk8JEchpBRp+9mncRGfXvz/NL0n3jafgB9Ef0uGkXIBXufxfBvZuTdCTdcD4BZyexN6MkHFl4K7GXgnpPiBwW1c9OatpyO4Tq9bdN6kVzRj3sRpPUzEwsICpQ9egEltY2gvLy7+ttFjb9Sc26KPOXPb7MGrCYeuub4/4GLiP59vapcQ97xrKDEMhnaGppdD1udoOHkJZ2ZvabexNLGIRn8nw4jQ+Nb2k7djMF71Js1XJb55OoBR4B3GQkt0ouF+4XSM2xgaNkM1Dao4rSFH66oCGVffrlLupcwsQ9XHXcvmhB6qTDpQxCNkSqcZLggm4VwcANQhTBuPnpnSnDI2Q9ELblIyFumIZgjjbUyB37NAXGV2N0zuy3415b6UgmE4n4cKspOHFVJBhd66pHm0avnfdG0d7QPHra8dqp1JyocX0LXHMnbgB5fNaci5bG6XWPJR1O6+waoG6jSYH4tmSvl1osNkBcn1MUAuDDE3PGDyK9tv98ppboiGAwveom5huEi/aRzSzhHKAaO7OZwlWkNMoii6RjrSDhfqdtqjYU8vZJTngglSbDUqWfvNpGUoiWdNo2qdukxT6THs5LuihQ1DZQCnNIIrkHpJpvY3EIrHQmlryRPGTeuMn05lapt3dlN9aS/yKZw/R4Lk+6dIUC1/5qfUlKPt1bs0lIvvYzj/Larw+/mh6v4lHAiiFWBaudDqa1U+9Clc/xIRQNYJ7G080aja4vauP440BKcdyn633bGGXlvQ+tepUmvZnojeEm+8aerM3fblDtg3qQsHhkCLUuo0i2xWcYT80K45wt1xt7SMeVtn4K0sOjgpG7r3+P64+YELFDLOz/fNFEPpouTn8lcUMPEOlHaNSdNtQdabUvqe2+Sx9egFGaaLQnRNe045eumarvs6o5+wZa+T+3SXmWvTyFyf0oXKdR8nhwbbhw4ME8/5Ktv5QjtQBzWu0WG6On9RGs1eRA/BQTIWumEsbCkjJyvQhu/ly7GOcn1z1RrkJuYO3JH95iqdfa63j1zHsOpo5xe7PXgbryUHOxJGCkMEbObV3Iat8vKPi2e3V//+mH3+9OvHw2iXg6NddkV7OTjay65orwZHe9UV7fXgaK+7or0ZHO1NV7S3g6O97Yr2bnC0d13R3g+O9r4r2ofB0T50LrnDt4PLtn5QQSnNyY6eNXZ8PfmT9g4PHUh+wWU1cwqtIDb82hQQumpwsDVpnP0VAAD//2DzKvg=" } diff --git a/metricbeat/module/linux/iostat/_meta/data.json b/metricbeat/module/linux/iostat/_meta/data.json new file mode 100644 index 00000000000..99f23d8cc95 --- /dev/null +++ b/metricbeat/module/linux/iostat/_meta/data.json @@ -0,0 +1,49 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "linux.iostat", + "duration": 115000, + "module": "linux" + }, + "linux": { + "iostat": { + "await": 0, + "busy": 0, + "name": "sr0", + "queue": { + "avg_size": 0 + }, + "read": { + "await": 0, + "per_sec": { + "bytes": 0 + }, + "request": { + "merges_per_sec": 0, + "per_sec": 0 + } + }, + "request": { + "avg_size": 0 + }, + "service_time": 0, + "write": { + "await": 0, + "per_sec": { + "bytes": 0 + }, + "request": { + "merges_per_sec": 0, + "per_sec": 0 + } + } + } + }, + "metricset": { + "name": "iostat", + "period": 10000 + }, + "service": { + "type": "linux" + } +} \ No newline at end of file diff --git a/metricbeat/module/linux/iostat/_meta/docs.asciidoc b/metricbeat/module/linux/iostat/_meta/docs.asciidoc new file mode 100644 index 00000000000..ec4c4de1618 --- /dev/null +++ b/metricbeat/module/linux/iostat/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +The iostat module reports per-disk IO statistics that emulate `iostat -x` on linux. + +NOTE: as of now, this data is part of system/diskio on Metricbeat, but can only be found in the Linux integration in Fleet. In the future, this data will be removed from system/memory. \ No newline at end of file diff --git a/metricbeat/module/linux/iostat/_meta/fields.yml b/metricbeat/module/linux/iostat/_meta/fields.yml new file mode 100644 index 00000000000..a7ded63f8fc --- /dev/null +++ b/metricbeat/module/linux/iostat/_meta/fields.yml @@ -0,0 +1,61 @@ +- name: iostat + type: group + release: beta + description: > + iostat + fields: + - name: read.request.merges_per_sec + type: float + description: > + The number of read requests merged per second that were queued to the device. + - name: write.request.merges_per_sec + type: float + description: > + The number of write requests merged per second that were queued to the device. + - name: read.request.per_sec + type: float + description: > + The number of read requests that were issued to the device per second + - name: write.request.per_sec + type: float + description: > + The number of write requests that were issued to the device per second + - name: read.per_sec.bytes + type: float + description: > + The number of Bytes read from the device per second. + format: bytes + - name: read.await + type: float + description: > + The average time spent for read requests issued to the device to be served. + - name: write.per_sec.bytes + type: float + description: > + The number of Bytes write from the device per second. + format: bytes + - name: write.await + type: float + description: > + The average time spent for write requests issued to the device to be served. + - name: request.avg_size + type: float + description: > + The average size (in bytes) of the requests that were issued to the device. + - name: queue.avg_size + type: float + description: > + The average queue length of the requests that were issued to the device. + - name: await + type: float + description: > + The average time spent for requests issued to the device to be served. + - name: service_time + type: float + description: > + The average service time (in milliseconds) for I/O requests that were issued to the device. + - name: busy + type: float + description: > + Percentage of CPU time during which I/O requests were issued to the device (bandwidth utilization for the device). Device saturation occurs when this value is close to 100%. + diff --git a/metricbeat/module/linux/iostat/data.go b/metricbeat/module/linux/iostat/data.go new file mode 100644 index 00000000000..4e546deaa34 --- /dev/null +++ b/metricbeat/module/linux/iostat/data.go @@ -0,0 +1,58 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iostat + +import ( + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/metric/system/diskio" +) + +// AddLinuxIOStat adds the linux iostat data to the provided map +func AddLinuxIOStat(extraMetrics diskio.IOMetric) common.MapStr { + return common.MapStr{ + "read": common.MapStr{ + "request": common.MapStr{ + "merges_per_sec": extraMetrics.ReadRequestMergeCountPerSec, + "per_sec": extraMetrics.ReadRequestCountPerSec, + }, + "per_sec": common.MapStr{ + "bytes": extraMetrics.ReadBytesPerSec, + }, + "await": extraMetrics.AvgReadAwaitTime, + }, + "write": common.MapStr{ + "request": common.MapStr{ + "merges_per_sec": extraMetrics.WriteRequestMergeCountPerSec, + "per_sec": extraMetrics.WriteRequestCountPerSec, + }, + "per_sec": common.MapStr{ + "bytes": extraMetrics.WriteBytesPerSec, + }, + "await": extraMetrics.AvgWriteAwaitTime, + }, + "queue": common.MapStr{ + "avg_size": extraMetrics.AvgQueueSize, + }, + "request": common.MapStr{ + "avg_size": extraMetrics.AvgRequestSize, + }, + "await": extraMetrics.AvgAwaitTime, + "service_time": extraMetrics.AvgServiceTime, + "busy": extraMetrics.BusyPct, + } +} diff --git a/metricbeat/module/linux/iostat/iostat.go b/metricbeat/module/linux/iostat/iostat.go new file mode 100644 index 00000000000..049b4daadf7 --- /dev/null +++ b/metricbeat/module/linux/iostat/iostat.go @@ -0,0 +1,104 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package iostat + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/libbeat/metric/system/diskio" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("linux", "iostat", New) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet + stats *diskio.IOStat + includeDevices []string +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The linux iostat metricset is beta.") + + config := struct { + IncludeDevices []string `config:"iostat.include_devices"` + }{IncludeDevices: []string{}} + if err := base.Module().UnpackConfig(&config); err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + includeDevices: config.IncludeDevices, + stats: diskio.NewDiskIOStat(), + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + + IOstats, err := diskio.IOCounters(m.includeDevices...) + if err != nil { + return errors.Wrap(err, "disk io counters") + } + + // Sample the current cpu counter + m.stats.OpenSampling() + + // Store the last cpu counter when finished + defer m.stats.CloseSampling() + + for _, counters := range IOstats { + event := common.MapStr{ + "name": counters.Name, + } + if counters.SerialNumber != "" { + event["serial_number"] = counters.SerialNumber + } + result, err := m.stats.CalcIOStatistics(counters) + if err != nil { + return errors.Wrap(err, "error calculating iostat") + } + IOstats := AddLinuxIOStat(result) + event.DeepUpdate(IOstats) + + isOpen := report.Event(mb.Event{ + MetricSetFields: event, + }) + if !isOpen { + return nil + } + } + return nil +} diff --git a/metricbeat/module/linux/iostat/iostat_test.go b/metricbeat/module/linux/iostat/iostat_test.go new file mode 100644 index 00000000000..6a607fc66d4 --- /dev/null +++ b/metricbeat/module/linux/iostat/iostat_test.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build linux + +package iostat + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + events, errs := mbtest.ReportingFetchV2Error(f) + + assert.Empty(t, errs) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), + events[0].BeatEvent("linux", "iostat").Fields.StringToPrint()) +} + +func TestData(t *testing.T) { + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + err := mbtest.WriteEventsReporterV2Error(f, t, ".") + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "linux", + "metricsets": []string{"iostat"}, + } +} diff --git a/metricbeat/module/linux/memory/_meta/data.json b/metricbeat/module/linux/memory/_meta/data.json new file mode 100644 index 00000000000..79a3d431869 --- /dev/null +++ b/metricbeat/module/linux/memory/_meta/data.json @@ -0,0 +1,59 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "linux.memory", + "duration": 115000, + "module": "linux" + }, + "linux": { + "memory": { + "hugepages": { + "default_size": 2097152, + "free": 0, + "reserved": 0, + "surplus": 0, + "swap": { + "out": { + "fallback": 0, + "pages": 0 + } + }, + "total": 0, + "used": { + "bytes": 0, + "pct": 0 + } + }, + "page_stats": { + "direct_efficiency": { + "pct": 0.9228 + }, + "kswapd_efficiency": { + "pct": 0.7523 + }, + "pgfree": { + "pages": 16061818710 + }, + "pgscan_direct": { + "pages": 1198580 + }, + "pgscan_kswapd": { + "pages": 50222460 + }, + "pgsteal_direct": { + "pages": 1106083 + }, + "pgsteal_kswapd": { + "pages": 37782783 + } + } + } + }, + "metricset": { + "name": "memory", + "period": 10000 + }, + "service": { + "type": "linux" + } +} \ No newline at end of file diff --git a/metricbeat/module/linux/memory/_meta/docs.asciidoc b/metricbeat/module/linux/memory/_meta/docs.asciidoc new file mode 100644 index 00000000000..8b0bbaed750 --- /dev/null +++ b/metricbeat/module/linux/memory/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +The memory metricset extends system/memory and adds linux-specific memory metrics, including Huge Pages and overall paging statistics. + +NOTE: as of now, this data is part of system/memory on Metricbeat, but can only be found in the Linux integration in Fleet. In the future, this data will be removed from system/memory. \ No newline at end of file diff --git a/metricbeat/module/linux/memory/_meta/fields.yml b/metricbeat/module/linux/memory/_meta/fields.yml new file mode 100644 index 00000000000..2490e69aa21 --- /dev/null +++ b/metricbeat/module/linux/memory/_meta/fields.yml @@ -0,0 +1,78 @@ +- name: memory + type: group + release: beta + description: > + Linux memory data + fields: + - name: page_stats + type: group + description: memory page statistics + fields: + - name: pgscan_kswapd.pages + type: long + format: number + description: pages scanned by kswapd + - name: pgscan_direct.pages + type: long + format: number + description: pages scanned directly + - name: pgfree.pages + type: long + format: number + description: pages freed by the system + - name: pgsteal_kswapd.pages + type: long + format: number + description: number of pages reclaimed by kswapd + - name: pgsteal_direct.pages + type: long + format: number + description: number of pages reclaimed directly + - name: direct_efficiency.pct + type: scaled_float + format: percent + description: direct reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory. + - name: kswapd_efficiency.pct + type: scaled_float + format: percent + description: kswapd reclaim efficiency percentage. A lower percentage indicates the system is struggling to reclaim memory. + - name: hugepages + type: group + prefix: "[float]" + description: This group contains statistics related to huge pages usage on the system. + fields: + - name: total + type: long + format: number + description: > + Number of huge pages in the pool. + - name: used.bytes + type: long + format: bytes + description: > + Memory used in allocated huge pages. + - name: used.pct + type: long + format: percent + description: > + Percentage of huge pages used. + - name: free + type: long + format: number + description: > + Number of available huge pages in the pool. + - name: reserved + type: long + format: number + description: > + Number of reserved but not allocated huge pages in the pool. + - name: surplus + type: long + format: number + description: > + Number of overcommited huge pages. + - name: default_size + type: long + format: bytes + description: > + Default size for huge pages. diff --git a/metricbeat/module/linux/memory/data.go b/metricbeat/module/linux/memory/data.go new file mode 100644 index 00000000000..d3f1a5ef2f8 --- /dev/null +++ b/metricbeat/module/linux/memory/data.go @@ -0,0 +1,99 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build darwin freebsd linux openbsd windows + +package memory + +import ( + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/libbeat/common" + mem "github.com/elastic/beats/v7/libbeat/metric/system/memory" +) + +// FetchLinuxMemStats gets page_stat and huge pages data for linux +func FetchLinuxMemStats(baseMap common.MapStr) error { + + vmstat, err := mem.GetVMStat() + if err != nil { + return errors.Wrap(err, "VMStat") + } + + if vmstat != nil { + pageStats := common.MapStr{ + "pgscan_kswapd": common.MapStr{ + "pages": vmstat.PgscanKswapd, + }, + "pgscan_direct": common.MapStr{ + "pages": vmstat.PgscanDirect, + }, + "pgfree": common.MapStr{ + "pages": vmstat.Pgfree, + }, + "pgsteal_kswapd": common.MapStr{ + "pages": vmstat.PgstealKswapd, + }, + "pgsteal_direct": common.MapStr{ + "pages": vmstat.PgstealDirect, + }, + } + // This is similar to the vmeff stat gathered by sar + // these ratios calculate thhe efficiency of page reclaim + if vmstat.PgscanDirect != 0 { + pageStats["direct_efficiency"] = common.MapStr{ + "pct": common.Round(float64(vmstat.PgstealDirect)/float64(vmstat.PgscanDirect), common.DefaultDecimalPlacesCount), + } + } + + if vmstat.PgscanKswapd != 0 { + pageStats["kswapd_efficiency"] = common.MapStr{ + "pct": common.Round(float64(vmstat.PgstealKswapd)/float64(vmstat.PgscanKswapd), common.DefaultDecimalPlacesCount), + } + } + baseMap["page_stats"] = pageStats + } + + hugePagesStat, err := mem.GetHugeTLBPages() + if err != nil { + return errors.Wrap(err, "hugepages") + } + if hugePagesStat != nil { + mem.AddHugeTLBPagesPercentage(hugePagesStat) + thp := common.MapStr{ + "total": hugePagesStat.Total, + "used": common.MapStr{ + "bytes": hugePagesStat.TotalAllocatedSize, + "pct": hugePagesStat.UsedPercent, + }, + "free": hugePagesStat.Free, + "reserved": hugePagesStat.Reserved, + "surplus": hugePagesStat.Surplus, + "default_size": hugePagesStat.DefaultSize, + } + if vmstat != nil { + thp["swap"] = common.MapStr{ + "out": common.MapStr{ + "pages": vmstat.ThpSwpout, + "fallback": vmstat.ThpSwpoutFallback, + }, + } + } + baseMap["hugepages"] = thp + } + return nil +} diff --git a/metricbeat/module/linux/memory/memory.go b/metricbeat/module/linux/memory/memory.go new file mode 100644 index 00000000000..163e3bea771 --- /dev/null +++ b/metricbeat/module/linux/memory/memory.go @@ -0,0 +1,62 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package memory + +import ( + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/common/cfgwarn" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +// init registers the MetricSet with the central registry as soon as the program +// starts. The New function will be called later to instantiate an instance of +// the MetricSet for each host defined in the module's configuration. After the +// MetricSet has been created then Fetch will begin to be called periodically. +func init() { + mb.Registry.MustAddMetricSet("linux", "memory", New, mb.DefaultMetricSet()) +} + +// MetricSet holds any configuration or state information. It must implement +// the mb.MetricSet interface. And this is best achieved by embedding +// mb.BaseMetricSet because it implements all of the required mb.MetricSet +// interface methods except for Fetch. +type MetricSet struct { + mb.BaseMetricSet +} + +// New creates a new instance of the MetricSet. New is responsible for unpacking +// any MetricSet specific configuration options if there are any. +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + cfgwarn.Beta("The linux memory metricset is beta.") + + return &MetricSet{ + BaseMetricSet: base, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(report mb.ReporterV2) error { + rootEvent := common.MapStr{} + FetchLinuxMemStats(rootEvent) + report.Event(mb.Event{ + MetricSetFields: rootEvent, + }) + return nil +} diff --git a/metricbeat/module/linux/memory/memory_test.go b/metricbeat/module/linux/memory/memory_test.go new file mode 100644 index 00000000000..2980c94841e --- /dev/null +++ b/metricbeat/module/linux/memory/memory_test.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build darwin freebsd linux openbsd windows + +package memory + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" +) + +func TestFetch(t *testing.T) { + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + events, errs := mbtest.ReportingFetchV2Error(f) + + assert.Empty(t, errs) + if !assert.NotEmpty(t, events) { + t.FailNow() + } + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), + events[0].BeatEvent("linux", "memory").Fields.StringToPrint()) +} + +func TestData(t *testing.T) { + f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) + err := mbtest.WriteEventsReporterV2Error(f, t, ".") + if err != nil { + t.Fatal("write", err) + } +} + +func getConfig() map[string]interface{} { + return map[string]interface{}{ + "module": "linux", + "metricsets": []string{"memory"}, + } +} diff --git a/metricbeat/module/system/cpu/_meta/data.json b/metricbeat/module/system/cpu/_meta/data.json index 4a5fd7c8ff7..cb9548ec6e4 100644 --- a/metricbeat/module/system/cpu/_meta/data.json +++ b/metricbeat/module/system/cpu/_meta/data.json @@ -7,7 +7,7 @@ }, "host": { "cpu": { - "pct": 0.0816 + "pct": 0.1629 } }, "metricset": { @@ -19,68 +19,68 @@ }, "system": { "cpu": { - "cores": 12, + "cores": 4, "idle": { "norm": { - "pct": 0.9184 + "pct": 0.8371 }, - "pct": 11.0208, - "ticks": 1964402 + "pct": 3.3484, + "ticks": 806327077 }, "iowait": { "norm": { "pct": 0 }, "pct": 0, - "ticks": 5083 + "ticks": 312229 }, "irq": { "norm": { - "pct": 0 + "pct": 0.0125 }, - "pct": 0, - "ticks": 0 + "pct": 0.0501, + "ticks": 6548016 }, "nice": { "norm": { "pct": 0 }, "pct": 0, - "ticks": 9752 + "ticks": 517231 }, "softirq": { "norm": { - "pct": 0.0058 + "pct": 0.0025 }, - "pct": 0.0699, - "ticks": 10386 + "pct": 0.01, + "ticks": 1561223 }, "steal": { "norm": { "pct": 0 }, "pct": 0, - "ticks": 0 + "ticks": 156646 }, "system": { "norm": { - "pct": 0.005 + "pct": 0.0551 }, - "pct": 0.06, - "ticks": 22274 + "pct": 0.2206, + "ticks": 31037256 }, "total": { "norm": { - "pct": 0.0816 + "pct": 0.1629 }, - "pct": 0.9792 + "pct": 0.6516 }, "user": { "norm": { - "pct": 0.0708 + "pct": 0.0927 }, - "pct": 0.8493, - "ticks": 123767 + "pct": 0.3709, + "ticks": 139619320 } } } diff --git a/metricbeat/module/system/cpu/cpu.go b/metricbeat/module/system/cpu/cpu.go index 7333df6dec7..7650b65cd80 100644 --- a/metricbeat/module/system/cpu/cpu.go +++ b/metricbeat/module/system/cpu/cpu.go @@ -20,11 +20,8 @@ package cpu import ( - "strings" - "github.com/pkg/errors" - "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/metric/system/cpu" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" @@ -69,50 +66,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { return errors.Wrap(err, "failed to fetch CPU times") } - event := common.MapStr{"cores": cpu.NumCores} - hostFields := common.MapStr{} - for _, metric := range m.config.Metrics { - switch strings.ToLower(metric) { - case percentages: - pct := sample.Percentages() - event.Put("user.pct", pct.User) - event.Put("system.pct", pct.System) - event.Put("idle.pct", pct.Idle) - event.Put("iowait.pct", pct.IOWait) - event.Put("irq.pct", pct.IRQ) - event.Put("nice.pct", pct.Nice) - event.Put("softirq.pct", pct.SoftIRQ) - event.Put("steal.pct", pct.Steal) - event.Put("total.pct", pct.Total) - case normalizedPercentages: - normalizedPct := sample.NormalizedPercentages() - event.Put("user.norm.pct", normalizedPct.User) - event.Put("system.norm.pct", normalizedPct.System) - event.Put("idle.norm.pct", normalizedPct.Idle) - event.Put("iowait.norm.pct", normalizedPct.IOWait) - event.Put("irq.norm.pct", normalizedPct.IRQ) - event.Put("nice.norm.pct", normalizedPct.Nice) - event.Put("softirq.norm.pct", normalizedPct.SoftIRQ) - event.Put("steal.norm.pct", normalizedPct.Steal) - event.Put("total.norm.pct", normalizedPct.Total) - hostFields.Put("host.cpu.pct", normalizedPct.Total) - case ticks: - ticks := sample.Ticks() - event.Put("user.ticks", ticks.User) - event.Put("system.ticks", ticks.System) - event.Put("idle.ticks", ticks.Idle) - event.Put("iowait.ticks", ticks.IOWait) - event.Put("irq.ticks", ticks.IRQ) - event.Put("nice.ticks", ticks.Nice) - event.Put("softirq.ticks", ticks.SoftIRQ) - event.Put("steal.ticks", ticks.Steal) - } - } - - r.Event(mb.Event{ - RootFields: hostFields, - MetricSetFields: event, - }) + r.Event(collectCPUMetrics(m.config.Metrics, sample)) return nil } diff --git a/metricbeat/module/system/cpu/data.go b/metricbeat/module/system/cpu/data.go new file mode 100644 index 00000000000..497a91a6173 --- /dev/null +++ b/metricbeat/module/system/cpu/data.go @@ -0,0 +1,112 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build darwin freebsd linux openbsd windows + +package cpu + +import ( + "runtime" + "strings" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/metric/system/cpu" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +// CPU metrics are highly OS-specific, so we need to build the event per-OS +func getPlatformCPUMetrics(sample *cpu.Metrics, selectors []string, event common.MapStr) { + for _, metric := range selectors { + switch strings.ToLower(metric) { + case percentages: + pct := sample.Percentages() + event.Put("user.pct", pct.User) + event.Put("system.pct", pct.System) + event.Put("idle.pct", pct.Idle) + event.Put("total.pct", pct.Total) + + if runtime.GOOS != "windows" { + event.Put("nice.pct", pct.Nice) + } + if runtime.GOOS == "linux" || runtime.GOOS == "openbsd" { + event.Put("irq.pct", pct.IRQ) + } + if runtime.GOOS == "linux" || runtime.GOOS == "aix" { + event.Put("iowait.pct", pct.IOWait) + } + if runtime.GOOS == "linux" { + event.Put("softirq.pct", pct.SoftIRQ) + event.Put("steal.pct", pct.Steal) + } + case normalizedPercentages: + normalizedPct := sample.NormalizedPercentages() + event.Put("user.norm.pct", normalizedPct.User) + event.Put("system.norm.pct", normalizedPct.System) + event.Put("idle.norm.pct", normalizedPct.Idle) + event.Put("total.norm.pct", normalizedPct.Total) + + if runtime.GOOS != "windows" { + event.Put("nice.norm.pct", normalizedPct.Nice) + } + if runtime.GOOS == "linux" || runtime.GOOS == "openbsd" { + event.Put("irq.norm.pct", normalizedPct.IRQ) + } + if runtime.GOOS == "linux" || runtime.GOOS == "aix" { + event.Put("iowait.norm.pct", normalizedPct.IOWait) + } + if runtime.GOOS == "linux" { + event.Put("softirq.norm.pct", normalizedPct.SoftIRQ) + event.Put("steal.norm.pct", normalizedPct.Steal) + } + case ticks: + ticks := sample.Ticks() + event.Put("user.ticks", ticks.User) + event.Put("system.ticks", ticks.System) + event.Put("idle.ticks", ticks.Idle) + + if runtime.GOOS != "windows" { + event.Put("nice.ticks", ticks.Nice) + } + if runtime.GOOS == "linux" || runtime.GOOS == "openbsd" { + event.Put("irq.ticks", ticks.IRQ) + } + if runtime.GOOS == "linux" || runtime.GOOS == "aix" { + event.Put("iowait.ticks", ticks.IOWait) + } + if runtime.GOOS == "linux" { + event.Put("softirq.ticks", ticks.SoftIRQ) + event.Put("steal.ticks", ticks.Steal) + } + } + } +} + +// gather CPU metrics +func collectCPUMetrics(selectors []string, sample *cpu.Metrics) mb.Event { + event := common.MapStr{"cores": cpu.NumCores} + getPlatformCPUMetrics(sample, selectors, event) + + //generate the host fields here, since we don't want users disabling it. + normalizedPct := sample.NormalizedPercentages() + hostFields := common.MapStr{} + hostFields.Put("host.cpu.pct", normalizedPct.Total) + + return mb.Event{ + RootFields: hostFields, + MetricSetFields: event, + } +} diff --git a/metricbeat/module/system/diskio/_meta/data.json b/metricbeat/module/system/diskio/_meta/data.json index b9c8533b0c8..a2301ef8888 100644 --- a/metricbeat/module/system/diskio/_meta/data.json +++ b/metricbeat/module/system/diskio/_meta/data.json @@ -15,7 +15,7 @@ "system": { "diskio": { "io": { - "time": 364 + "time": 601740 }, "iostat": { "await": 0, @@ -48,16 +48,16 @@ } } }, - "name": "loop1", + "name": "sdb1", "read": { - "bytes": 5267456, - "count": 4124, - "time": 557 + "bytes": 25128030208, + "count": 3146154, + "time": 833872 }, "write": { - "bytes": 0, - "count": 0, - "time": 0 + "bytes": 34401640448, + "count": 861040, + "time": 11224168 } } } diff --git a/metricbeat/module/system/diskio/diskio.go b/metricbeat/module/system/diskio/diskio.go index 9da3a3c2344..1359180cff6 100644 --- a/metricbeat/module/system/diskio/diskio.go +++ b/metricbeat/module/system/diskio/diskio.go @@ -20,9 +20,14 @@ package diskio import ( + "fmt" + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/metric/system/diskio" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/metricbeat/module/linux/iostat" + "github.com/elastic/beats/v7/metricbeat/module/system" "github.com/pkg/errors" ) @@ -36,9 +41,10 @@ func init() { // MetricSet for fetching system disk IO metrics. type MetricSet struct { mb.BaseMetricSet - statistics *DiskIOStat + statistics *diskio.IOStat includeDevices []string prevCounters diskCounter + IsAgent bool } // diskCounter stores previous disk counter values for calculating gauges in next collection @@ -57,17 +63,23 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + systemModule, ok := base.Module().(*system.Module) + if !ok { + return nil, fmt.Errorf("unexpected module type") + } + return &MetricSet{ BaseMetricSet: base, - statistics: NewDiskIOStat(), + statistics: diskio.NewDiskIOStat(), includeDevices: config.IncludeDevices, prevCounters: diskCounter{}, + IsAgent: systemModule.IsAgent, }, nil } // Fetch fetches disk IO metrics from the OS. func (m *MetricSet) Fetch(r mb.ReporterV2) error { - stats, err := IOCounters(m.includeDevices...) + stats, err := diskio.IOCounters(m.includeDevices...) if err != nil { return errors.Wrap(err, "disk io counters") } @@ -101,40 +113,13 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { diskReadBytes += counters.ReadBytes diskWriteBytes += counters.WriteBytes - var extraMetrics DiskIOMetric - err := m.statistics.CalIOStatistics(&extraMetrics, counters) - if err == nil { - event["iostat"] = common.MapStr{ - "read": common.MapStr{ - "request": common.MapStr{ - "merges_per_sec": extraMetrics.ReadRequestMergeCountPerSec, - "per_sec": extraMetrics.ReadRequestCountPerSec, - }, - "per_sec": common.MapStr{ - "bytes": extraMetrics.ReadBytesPerSec, - }, - "await": extraMetrics.AvgReadAwaitTime, - }, - "write": common.MapStr{ - "request": common.MapStr{ - "merges_per_sec": extraMetrics.WriteRequestMergeCountPerSec, - "per_sec": extraMetrics.WriteRequestCountPerSec, - }, - "per_sec": common.MapStr{ - "bytes": extraMetrics.WriteBytesPerSec, - }, - "await": extraMetrics.AvgWriteAwaitTime, - }, - "queue": common.MapStr{ - "avg_size": extraMetrics.AvgQueueSize, - }, - "request": common.MapStr{ - "avg_size": extraMetrics.AvgRequestSize, - }, - "await": extraMetrics.AvgAwaitTime, - "service_time": extraMetrics.AvgServiceTime, - "busy": extraMetrics.BusyPct, + //Add linux-only data if agent is off as not to make breaking changes. + if !m.IsAgent { + result, err := m.statistics.CalcIOStatistics(counters) + if err != nil { + return errors.Wrap(err, "error calculating iostat") } + event["iostat"] = iostat.AddLinuxIOStat(result) } if counters.SerialNumber != "" { diff --git a/metricbeat/module/system/diskio/diskio_test.go b/metricbeat/module/system/diskio/diskio_test.go index aabc44c374b..b5494ac4bfe 100644 --- a/metricbeat/module/system/diskio/diskio_test.go +++ b/metricbeat/module/system/diskio/diskio_test.go @@ -27,8 +27,48 @@ import ( "github.com/stretchr/testify/assert" mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/metricbeat/module/system" ) +func TestDataNameFilter(t *testing.T) { + oldFS := system.HostFS + newFS := "_meta/testdata" + system.HostFS = &newFS + defer func() { + system.HostFS = oldFS + }() + + conf := map[string]interface{}{ + "module": "system", + "metricsets": []string{"diskio"}, + "diskio.include_devices": []string{"sda", "sda1", "sda2"}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, conf) + data, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, errs) + assert.Equal(t, 3, len(data)) +} + +func TestDataEmptyFilter(t *testing.T) { + oldFS := system.HostFS + newFS := "_meta/testdata" + system.HostFS = &newFS + defer func() { + system.HostFS = oldFS + }() + + conf := map[string]interface{}{ + "module": "system", + "metricsets": []string{"diskio"}, + } + + f := mbtest.NewReportingMetricSetV2Error(t, conf) + data, errs := mbtest.ReportingFetchV2Error(f) + assert.Empty(t, errs) + assert.Equal(t, 10, len(data)) +} + func TestFetch(t *testing.T) { f := mbtest.NewReportingMetricSetV2Error(t, getConfig()) events, errs := mbtest.ReportingFetchV2Error(f) diff --git a/metricbeat/module/system/fields.go b/metricbeat/module/system/fields.go index 28135f5fa95..d78002d2847 100644 --- a/metricbeat/module/system/fields.go +++ b/metricbeat/module/system/fields.go @@ -32,5 +32,5 @@ func init() { // AssetSystem returns asset data. // This is the base64 encoded gzipped contents of module/system. func AssetSystem() string { - return "eJzsff+PGzey5+/5KwgfFhm/m5E93mxe3vxwgON5uRvAWQ9sB+8Bh4NMdZck7rDJDsmWrPz1BxbZ39nqbqmlkYMMFtlkRiI/VSwW6xuLN+QJdndE77SB5DtCDDMc7siLT/iLF98REoOOFEsNk+KO/K/vCCHE/ZFoQ02mSQJGsUhfE86egLx7/I1QEZMEEql2JNN0BdfErKkhVAGJJOcQGYjJUsmEmDUQmYKihomVRzH7jhC9lsrMIymWbHVHjMrgO0IUcKAa7siKfkfIkgGP9R0CuiGCJlAhw/6YXWo/q2SW+t8ESLE/X9zXvpBICkOZ0ITLiHI/Wk7fzH++Om917kgqKH4Zmn0PggqKGztOBYrlp0dAllIRSjQTKw44H5FLQkmSccPwexUO5j91puU/TSKqhLC49uucFC7FqvGHPdTYHwv9nUUlsmQBqkRV++T/II+gIhCGrkAHAWUa1CyNTBCWjiiHeL7kkjY/sJQqoeaOpG78ceA/ryH/Il0hoy05hiVAdArCECYQGNEpjaCDthoFhkVPehrWWnA0kZkwRwLz8nKJzH0CJYCPoWJCBvdyeAQ6wSK4PA5LQbjc3qSKScXMjqRKRqA16CHUnI3Th6JkMb9AniOqAcDPJ8gDAMktZeYCeSmIBUaupCAx008vh9FxTh0xDp/6/fKYrEFtWGRNM2vSramIuf2PNVXx1lpzTBhQKktN735Uv5+P9ZOh1nJpvqV1sXgPo/C51+YA5AYov7yVYYIwsZE8E4aqnVMBix36ORumTEY5fmO7Zhzwt+tdalmipWpNtqW6xi9p1qDyI1CqWesLbzeUcbrgQKTgO3t4/ibY10GMPKdevFwGFb5cmh3lykVp1vImLVXWY9bHeWfWzZtyoZxvli8Ujk5SBdpbX7gCUpuZ+7AUN8LuH87+gKabSCo7Q5Mt45ys6Qasg0q/siRLyIbyDDfNl9vXr/9G/s1N9wXHbg1WzlMbl3IFNN4RQ5+sfDDtR2XCSEKjCMXO6ZZNe9AAFgvlT+2akg+iHSLQ161hdzIjERVu0aosL4I3KwXUgLK/EI5v5BepCHylScrhmrAl+XtrWCdS9uvUkB9f/81Cu7Zy5YTLhz1mUZrNcm5+cdKzAHL7U+fi/Llc2D+Xk/jtul9/Fm/nG7Ja/7LLAxT+Zd1OY90aaS6UkdYWBE0c2XiiPsQcUHAePvyX1UJdRsk/S8tokH1iLamLZMHYMPXFEjL2oL9MQo467S+TpOFH/oXiP+Dcv0xKJj/8vykyD7UALpPIb9UMuDRuDrECrvNAiIY4Z3IZs0HnOkB7w2L43IrufSuZ6UvO6X4bWdALTCZedBLuuVMhh5+Iz4380EPur9xDlSdWTpn8rsmKMekHO0Ql/2D/kzx8KMrIBtbg5T/jcxT2n8H1fILdVqpm4sDHj++Ijunt+OVG8uyUfcIGilE+d4fnCHgDIXyv/Qx5uRv5vGaaJHRHhDRkAVY4Nix2xzjlvGR6a0wfo+8hSAGNZ5jwmHDzoKVUsTDsJFZk7ApZkdFZZCV8mXG+68G3VczAyQHiLAciRA4udmZ4Ri03BUNfOgA8DoMw6rDJB0HeM5F9dSku1pyKNOxADZGRyo+EyZ6UMy9pglCts8RyBj9FNPsD7dB/3L4ZtILPzyCLw4CYhkf5YAPZ1Bq1n20oVvbcOaHYJ4xbnyCSItb+ePNqBXfsoIV9Nohuz/Yai6cGGMYYS3sOPrz60A/Qem8zXG0Fv2egzSwBtQI9T0HNNURB7CEPswd8M1WP29xPqQnOiVly4ihxGdstKCC/Z5BBTIzEzRDDhvX6Np4sJyLnpQvnPDVhtfU660KV6JnWLfQVOg9YoPOuzLSU4Ip4AvacNhOQ8XN53ha2bwtz03YPH2m9BFHrX0xLCN2Aoiuo+jRLqRpSFlwRI60Fah0WiMfs/zOuihOxUy6LI+l869LYNBMtTL7j6WY1tzbKaUhB6+eKCcfel3aZLOqBGmAYJajDT0wHzkE4iJVZn4SIc27zaQXJhS9g3mllHS9EbgZHiBWmqrn1Eol6ePVh2vVYZHo3HTWP4ch9nClrJG7XLFrXSeg+FK8WVMRbFps1yQzj7A9qp0UmlJ96OSP37uOamky5j8goyqzj4mrmypJHTSIuNS59vYoxZwkIo2S6OyaYVIat/HXI9pjjA0Q0H3S+YGbS0F+B1g5sl6wNt4Tx/KmgEq/HeW25SQ3bQC49qZS8cNl/eP0fP7ZWeck41G6+koOihuUwrdrl8k9TlDAXRJ8ppoABQszqVPhtpHX5M5EqtmEcrJ+Buan8xJsFobtNOh8Z4BwVxKyW1N6RL69i2Lyyf739EkRk5z0BFDtGEwp8NT+EQWDAfZ5K1hHpOxgLDmw1LY7d4k0YDUrrCcMGdnwiZAzaSovdo/ibduS8AknBs0r7fqm26OZTc63CLwVwCNOQ72fimlvjCu/2cyzTcN7AsZ1wJLznP90aoPfVKRSiqO0Bc9Q55kXKjVQ5yiqHWJ4Jo6uVghUtUmGUc6dyGpdbyq8efXvn0GTIP+vqx6MhS5k1PePa9jliW38OqL0OeXNTBZy4fVIfXtk24aBxfUqqSSwj3YoGBLhO9mvgvazoQ9/CGfIeiGciasDGHmgCtJvl2QDiTu0BGFLH50Po1N4VAk15ppGnL9suD5c0PkZ9WBfPjpH7sEdu+Be3L8YqYfsnJlbzJY2MVHfWtRuniN9X4BfuJafakISJzEB4D7/4xyUh/YfH2qFwXtxeFNrbANwwbixBfC6ZCMgCiVlRkzCstLBNznMtRVhgpqDo2aSrQ6qOpAk/FKbotJeGW9rZdQU7yrxzQ7RCFL7f2AThibO5HXiuOdz9/aPO5278Zo/YQbDO6NU6/6ys7EOLyq954Qu5iisXHI0laCy8YiLiWVx8OJLCVXksdrk5GdFoDZpQ0ba/FtlyCUqTKw2Fr+pZQyOTUT5rmCEX744NWlhH22H2ehvJWxyt7AgIMdaNWs71WfF7reXgjiBnsEg9QRV+VmTwwRAFXhlqF9lnVohAREAWYLbgb757kcaqhmqsxq9QsCmC/Wl+ksSQgoh1rnk/fHJxskQqIDEYyri+JimqQRKtIXoqfOSKDH/pEAny/D6UZ3d4yz8YzINQHmUcHfkFtctS4UW9TMxph7y/wK+QlAkODAG8SpWMXiWQMLGU121e2B+pqhPi16rg0D0plUqhRNiyProFbjVUsaBt38v+fBDkw6f/JgwJpURnSVMB5jLEBI0wdZCL0AdB/ouJWG71tf8+/N7e2H4VZSEW/utDxaJDvZEhKo70qjky0Ets51Zau7SvQHhLm5qtW+elCpbs6x158X+RrP/XNK/qoRQreThKabZYS4VpwyLtUj5lvtDiqPVPzaU5FCztj3w8s99eEjNUlJ5LraPhMw7vc2nEMik7Cq7MzCxtXRYfgLmGKcqNMBwKIaRW5WamHwETpwPARP/8CmhM11hvdjQM5H0xIKkPOAABHhGjY377IOCIZF3NqX9rWjsbtgmLFD5dwRydvsOs1fzIxmqVQiOP1LDpSkdUzJ8s7AMFK+dm8HZKC7WX+4gK4TwZN3UfwJgpiA7UAMcAdPPyZllOFR86A2cDZmcrgimt0okW7wxQfsbVLaMrDq2CiFOWDF1pRHu+pe5G27vs7gNzWC5ZxEBEuzPqIzd3jpaUGCr6aEbeEi63oKo6iomYRXhpuxQea1lro7LVCi9CGlmM21RiTRa45XweFri5z86CoB5fZysICevZDXALxEvyc9vew/Zf+GAt08UVgnzhRSolv3Rb/NdKsIgJQjmXES5RSc4UnmkPAUfZNvXS0ZpcdVbokol8i2lEp4w0HSxEClxF8vMSkqMgi8y4kEtAnkZSpjOV8uzEh2sfYXIDKpJJwkZvjRiWNOMmVLQxmIYj9ve9m94Vti6lGgfeHlyzqr/ZBB46MFrIKksf8mEr9HZoedJwREK8IH3MbMEagqiiJSjnCxo9TTL1u9yxrrAGa/KTTOMVdp1yZv9liZ1ktzStwiuu/4PZSlVFND7L58eopPn8b6qNDGrv4eR/x+YTy0Yly/l6GIBZj0z8YkK1CX5IQwOZmbPWIDZvZWsQXU0KK+Ge50SoIALWfx/GhcWiJ5j0LkLVMcKxBzLsdEhUgWQgY5iYgVJSnYYtbmjfbsUhYmI1YK3OhUmDiPsRMTGLlbTK+iSImIhkgiXwfu3KS1J+2gEcOyVAmZmV3A+wmphnmlC+pbv2Yfna+lr3VG2twS9i8vOne7KAiGYafPbKmm4KUqlMGb7pbl3TOI/mOksSOqD6pDgsFmDosPPqV38iucs7zv9dcbmgvFDtmJpjZjfw/GHp7N+CyyUX/4KWR9OzYA+PLmYOKtwFzkRTzvb5Xc90WTzldL/d908358zAxHO+Zwb2T8yiZNJVfPdrgNLCAHWtp46yuvwYFavL/8aaXDSmhl5XHyS8rr702HgmkUxrdVHOaFNjpNSsC7pnga8mbOVuUBZPSLZnxAaMIwy9IVU3nmc4dOPO0guVCcHE6kW4rjXteHyxn/z2N4dQnx4x4YEzrg6fsf3VITNGScyZmHiNlxnnxHreVMQ3dngXqjLSrroyLpDgcF/7EjQ8FgI1PVStsgSLhTSkVFF/tgWr8dlKSAVzupAbuCNvXv/wU1jjaVAHbCXXLfywfRRtD11WezoysfIpi3p56NDZQWyGq1n3y/mREgBiw5QUduXIhipGF9zH9oJS4B7QsSo01KmKVpoDkl8UwM+f7q9d2ZJTsh8+kf8Oq4z6W0Vkupj5u8ffbnQKEVuyqBosT8s+h2PD4Z3dZsmorHd3LjnQ+tFUNfK+NrRNsK5nMBqtJ0JbvEFkwbpsg2YiAic9Xl908boJ9PJS+Y3um95eL9YCKS2K3bM0xtPywVQcBc0SxqnyhVHBaf9mZykYWZ0gZjrldFd6CkamucrO22+2Oy2GmdvROfqb4jBsauGH+shV96zy8lbrvkFZ72+5yAxRVHQFPrEw8nW7O0WTxXtaPZMz64VwC+gmYCcTp8TraoP3Lu8eflrtEerqUqKL20bvGHQW0zZ/wStnInbEtVNDx4XU1uUPMrxOx+UDx55Hfedd33n1TMmRUgLytsTex6qye011tcDXVTc3Ks/fYWqIvFtTtQJyZQIXKYqRqTNXcm+OCroCZWchLsGEpc4YcPcuTI7kZfECn4+6uktMTPdLqtL62TLMlskfQbPYbq1PYMgn9gfMGtoiwHcZRVnKXFo6ofYf7jNXH9/++rJ3RaJMKTuhN3qJBpcDu+640d/k1uWdQaNZ1L3b1lQ913bDueMQMZnubJsR9ngOuCHzC+NQfEYqbwvmERV3PFtJQXa7SCPTFadhGWhQ7u482FPa+xNTK0eZgjjq+Gt0CanzQOP4g888zhJmZlouR9d6DBUQuTRulrwiqAd6YT0Fh6x5hZWxIyrIAki0tmZV3LToqCFU7PD87WPFmrac2qlYYYc+FSsqY1tWYKv8BRBF8/dPlJSmwxEObbyDt2Qe0bcbCPHosjWlmwnbomIDODxXqX5yF3QSwPbvrRH9t4reIwrKXEbLmLLnrhtIr1mKJVCtAYUUN5YdfmRkoIbaBMi/WnAB1cJYv70VdyM9EbQBDCZemh7u0cCwkiSxAYujRhOqtYwYhsO2zKzdcWrZHPZhHtD7w+Z74ntDaD7qw70LyvjW0/noOBrSnV8GC45KF3uStqRW/2HWp2OSHT2/HuTlqNknzv9aZwvnT32vXSsb1zlrFMtwtnMwrR27IuNKeLo5FqVZyQuiozXEGQeNPhXFLvLOTqX6qaj88vsoOOZb951cP0thlOTca7atLGK3xVRKX5N3v3xCBfLxc3hQ+3dtqIgdmPwNA74jS8pUOZTXM6mSVl8wKSgPlFUjd7BRgLf+c/cxv3WaL2NxRXILbLU2M/LxcwVGcFwFlHtftAFKg9GVd7WDnnbQHiXlO0b1BUAm+3vaeadNSlZsA8Lankzuq54cVq0VVGhkwH4lTQl8uM/jTk3p2QugQ10cBCG8CezP4yFqo3O0kDrZS2S01DO/YMFCSTK6QG0PqTgProV/Wi1hkZJ5a38sMZRbomCVcarsqdg5lGPJ9zrXE0aiLCvQMlMRaKLXMuMx2iVQVJKO4MnvmTT09Cz53HD1OxnjNjLl4YvBCClXk7S6R1Um8v0pBfi9Sa6oJjEsmTP7urlcFY6uDgoh7qGrdmrevRVYi7cC5aMbGCDx4SewCq/YSIinqvA6B611H82NxhpbZ5W8QD5Z7LVjNyfTzDPFmd95seYbYoWerdZVa3Qve5W54P1a7Mtu/nbsVwzDjN2oysxUJtDVugRmYBhfihVog9YHE5nMtN9znQMz0XBR6pt4TTfQxbWBbHINdxyMU7OprHv3qgbLZTeUa1Q6tQ1jN0VdxXQrN7u1kRXAabr/ckabdLNW0hgO8dmZYGVFd63qwrUZ8djIFRLJ9HXnuPm1iK1LYFvdnhffmTXsPIO+rmmG/RjxAbPlXr1UUXdWqmsr5OIBTBE8C4eq/ybHxcmP0CJinrd9dx3Zr5ggggpZa2Xvd1qxHj0GRmidhvlMNNoTBR7kN3kvKO+uHKjdyn/+sqaLn2e2pn0m+jxWY/WN4oqg19pnWRVQ9Z975H3UHnelSNPQWtBSBV8Ax1KQRMZdty3C+HzO+nwIr1xq+uUYqCmocISF7C+Qyn9qhVLHS1ZBZUt7FmRLQYBGa/xoQ8L2HN9M94vY3iQ0Gac9/eVUHxZ2NbB/KdATKdDxijKBZIYZtM7cMhmyQ/syiyMIr7Z69Mm9xa4z/FW+ujSa4IR+vRyi11BEBau9/6am3OW7LpHqMvTiDpl67zpLbV4XbP327uOTRmt46Us02vFqTPRULPdMDz0fLPeWlPHs9PGUeq7Xey6NmhOX9rtqrOlLsm0VEJc/CrCJ0nCC9fbSdMMa8p5+eXFITVFgN0FsiORaoPs91DnelHurYNal6pVWpY49jNvMqjFljyFxLLMuXhXlkSQvcE2m4WLvj5NMrYD09pJUUHOz4YJ2jnjVWnVUViOV0tOl2iu+GPZkZsvT5dstTRb0mi+do47nzMUrE7ls8GefguiOEh6iOJ4u03RprtuUtosde26i9CJVRU1H2EPm87vHsu9xq7J1DKGXqhqqOqFJcUBH9ATHDtOeyKZvQU94ZjX51FIYfVw62NIouHWZSqO5kN3JqvH2hQtYus7gcypkuCXLYAZMKCtvhRS7RGa6tEBdB1spiO9kzoFqc6MgAmH47gZ329X7j791M4gzbWpXbpN0qcmVXieQvAyV2Q9nnvXSz8y8XxiHmwWNnsri9JI57z/+VpB7AFXI6zPT82gPCJx46jVaM1BURWsWUT53rJpflmqsho0LTyyH7a2novFCRU843deduZ2EXXp7mdwqPbLBfOscss7Pw/iWv7Hw7WjS4lWIqrqo7bxuB7e5Iw/i1DOozW5OhRVqkEcHSEeCTfsui+JP/u1wR+2Ng0j8/+Gjnt2qeFqdg83ase/j2YtkrI6gxe2KundqFFutQEFsP7EvAIbQR8rDv6SafwN0I9AewsmLX+2nXrj/1GRtRUiUd1d8MMC9u8J3eIfFyH3ur3u2Bpti4OWamFVvdwyUKD3vDLucoPAMe2Laf2L1may81sTcpby8Q9MBdHT1+jw1ITKrOGnHkrLvTu8gUs5xLPrUm90higqdUsy7FE3IX14TIbvjvtMarkrruZ35Yrj2z0YXTbkktGBkkF/jSme2NL0YWj8VaY8DVy8TsGGRwee7LoWoXyvh2IgKIY27q+AfZhhEaU7lgj+xkA4fUS7zM5dRtW/vX1UyU1fJHFAk46oJL0Vim2/NO8WDumYJSrlon3s00j/Tv0Chiu3m65yc7EnWjGITk+epuiwZ8PDqQ97ZVAos87fcdhVylvzDCcc6bCiv1vvaY2zoITmLdu0GqvkN7UADVWY43JFHb19+GthhdQ9T/BC1Ht+Vi9Ggi/YjwYeGT/7gb19nuMY6lrARLtMtvJUbJ46wiZBUXj+o92sZhIXFrYKk44HYQUeh0BwgPQVL8oHHoTFTNlGugHHjjsLyh0wWbPoVcsOOQhIDnZ4lMT6oF0ZBHsz3mmxA7UgmOHsC7k0dZtytdOuWUoVvfTBBtEz8XTrKiWYm8yqVGZLQnXdiw6Rl4knIbdO5PJ66krDKtZG1e4AOWwrzWHzvbTajGGys3lfWJfOI2ipa0Zp1NFrtNr5/8vcS+nhFk6KfnzvqurYkNa3beUfMmzfpvnFLMQABhw2ED4+DO4vatXDj1gGEObAT0dzClmE5PQjFO1+IaAcnbvBrwhyWj28f7glViu7cvco4EzEVJtx7Pmb6KU+fTbSNqu8TuZitm2TP/Kc84HGGyiLhXQamjXWb92FCH3p6luCwcT9LlpTxyY6yyvxu3P75cX/pMc3Rj+/aSxKKTXsU3SIKp2/DrdvRvZhWciphFRy8KjRryWMMw5Pb129+uLHuTw5hHzy7P09gkHh83sD2EF0oWeGNMDtvD9pCP4Fq6K7JHl2ough5nxc3mz7DoYUVHpVjqk1o5ZCQNJ4f1Wn+M17/pjGpHUz75jx6uvwsHDFltjieSp0tbsYROcdGt8E5A31OWxNipsTQJM0nxG653hbDPmwz3ykph4LBcXf2UBHn/tW1s1Ht/4wmWdru0lZ0K/8K0TyS8VF8+vTwv9/9n/f3xI5TtibzCL/XrvFi+1GIismY3/QPouhtmeZ3XNFtrNWtK7xy/c3GojTzpX/B25XDO9iVXbXr9w07Z/YJkP0FlsPnrxVFtjLozcmxDG6GGZejZq1UnWFh3fCFab2Y04ljUNg3f/gml7PO2wWDYr/nrrVwEciOzGIFVfi1rtPhyh9Z6kHW/aTZYGjBafueI+x6k+6wWX3+qTJn276QFtgxru/nd49+FF0aOU69HxdXdA9adDlm3Q9j+I0z6/p+14MYQRBLmrBWr7ihCOznjpmcy4jyGQs35Wz9ungk5/Y/3sxez97MbolU5M3r17d3r+9//unu7c//eX/30z/+/uPd3e040/a9xUEeHgmNY+V7jbKimR8V5OFx84Od7OFx82PxoSG0pVI1N0SniBf0vXlzCHw7VQ8mBYk0cAEM/4hAJua4p+4sLPcEDOf5Wuowqp73Qv/9x5s3t7c3t7f/fvP3H2diO/N/mUWy9dp4D+bHzx+JgkiqOHjoq3xNZuQBX9OTC0OxS9uGUaJgA0q3j+eHR8KlfOpMmDXYAIbH85Rnei5HPblUvp96KPn4Js9yCZFPlKY3LoQWS7SEr+Dz+/uXuYnveWEXzVWYSgEkke1rSpwugNfe8LrGAexo//MWXc8XSylnC6pmK8mpWM2kWs1eWP6+qP6ilfQungOyY8RgQCVM5G++2OFJJBPwXYepIJAsII4hJpFMd0VgkJpWmyH8wtqY9O7VqzRbcBbpbLlkXxHHYFme40uYhzoobeH8Tzuc/9AiJ9O1lyrWBCXQixvxFzV6EHc/f9Z3xo1/OG0vAP+wzIEgAoGIw1BM/dbZL5V3zkht6L044Ouhz/hZ3zjDcppj+IHtg0aLRPhb4yfuDCv1TL3MOJ+PEIW6Ddydnv+EfydD3z8dkZ2XS9emP7efWZmT9wGCoyzodkvSg/u5v0U5FsJZ1M1F6I1J7HXLfafQPoc4XP1hgSEPu9FVe/trA4EigQmxFFOg8dP5XuyU6+IejD10bXo6OnUzZIKnQ36tXwyvupJ5wOe6bLddhmaKZqS+DhfLU11ALXXPwP0BM/JOKgU6xcZrRub9pjRgXvuV1Ziv9E6/EmBesXTzwysTpfMEkhn50NH2v7vML9z89+hO7P2rSwYGgKRK13R/nXf3Sg9Ei4jdXveL5KeF2Ip8vrTd/N1LQZcOmZqAXJ/0832YXjkBPgttn55pwgNtLQKm161k1wkAlnmwyrSjuBlxqWG+pZ2tQ06CtoHQ6oh5iWQeTAjVcRuWXAbsAsgQ1Hon5jr8oNVZQec4hmJWEDUfrX0WzBbHEMxLJnBNmqGgs4MugIxB3Yz/PBvqN0NQc6rNnEahDMxZQec4hmC2uuYsJ0i/ymNiFUJcOGnxpObrb/d/EvPVEvKM5msWX6L5un91yUDz9dzGXxfqPf9S7I608YjF6CjBFzfEl3o3A3+dQaxyUXGf8rGEI1NtvjH7LAlXMwRSA/n2yb/a+DMTaWbm+YcSxjkLlw8MKOj88CmnFV92KIdql0tlGpTu5f0BxVLv5WoF8U3x9jlozaRoBpD38bgjnHZwmWt5CcuDCc6qofV41BHzvhXV1AiXK2Y1V3OKPfe9jqT5/udM+0pG98raAA4EkrBHorBfz2euSkPHAoRqRY5Zg0L4hpam1NMTQSQLKTm04gO9SOzXCBMxi5xmonlmaC9HjilxC69I3vm1Ufi2B0Mkp5aKymo4BR0HZinL3mncOqwOrqleA77rSR6H6QS3RvORKdfeI/RtLS3oc9Jly9QGoPJf/n8AAAD//y766bE=" + return "eJzsff+PGzey5+/5KwgfFhm/m5E93mzevvnhAMfzcjeAEw9sB/uAw0GmuksS37DJDsmWrPz1BxbZ39nqbqmlkYMMFtlkRiI/9YXFqmKxeEOeYHdH9E4bSL4jxDDD4Y68+IS/ePEdITHoSLHUMCnuyP/6jhBC3B+JNtRkmiRgFIv0NeHsCci7x98IFTFJIJFqRzJNV3BNzJoaQhWQSHIOkYGYLJVMiFkDkSkoaphYeRSz7wjRa6nMPJJiyVZ3xKgMviNEAQeq4Y6s6HeELBnwWN8hoBsiaAIVMuyP2aX2s0pmqf9NgBT788V97QuJpDCUCU24jCj3o+X0zfznq/NW546kguKXodn3IKiguLHjVKBYfnoEZCkVoUQzseKA8xG5JJQkGTcMv1fhYP5TZ1r+0ySiSgiLa7/OSeFSrBp/2EON/bHQ31lUIksWoEpUtU/+D/IIKgJh6Ap0EFCmQc3SyARh6YhyiOdLLmnzA0upEmruSOrGHwf+8xryL9IVMtqSY1gCRKcgDGECgRGd0gg6aKtRYFj0pKdhrQVHE5kJcyQwry+XyNwnUAL4GComZHAvh0egEyyCy+OwFITL7U2qmFTM7EiqZARagx5Czdk4fShKFvML5DmiGgD8fIo8AJDcUmYukJeCWGDkSgoSM/30chgd57QR4/Cp3y+PyRrUhkXWNbMu3ZqKmNv/WFMVb603x4QBpbLU9K5H9fv5WD8Zai2X5luSi8V7GIXPLZsDkBug/PIkwwRhYiN5JgxVO2cCFjuMczZMmYxy/MZ2zTjgb9e71LJES9WabEt1jV/SrEHlW6BUs9YX3m4o43TBgUjBd3bz/E2wr4MYeU67eLkMKmK5NDsqlIvSrBVNWqpsxKyPi85smDeloFxslgsKRyepAu29L5SA1GbmPizFjbDrh7M/oBkmksrK0GTLOCdrugEboNKvLMkSsqE8w0Xz5fb167+Rf3PTfcGxW4OV89TGpVwBjXfE0CerH0z7UZkwktAoQrVztmXTHjSAxUL5U4em5INopwj0dWvYncxIRIUTWpXlRfJmpYAaUPYXwvGN/CwVga80STlcE7Ykf28N61TKfp0a8uPrv1lo11avnHL5tMcsSrNZzs0vTnsWQG7/2SmcP1cI++cKEr/d8OvPEu18Q17rX355gMK/vNtpvFsjzYUy0vqCoIkjG3fUh5gDKs7Dh39ZK9TllPxaekaD/BPrSV0kC8amqS+WkLEb/WUSctRuf5kkDd/yLxT/Afv+ZVIy+eb/TZF5qAdwmUR+q27ApXFziBdwnSdCNMQ5k8ucDQbXAdobHsPnVnbvWzmZvuQz3W/jFPQCDxMv+hDuuY9CDt8Rnxv5oZvcX2cPVZ5YPWXyuyYrxhw/2CEq5w/2P8nDh6KMbGANXv4z/ozC/jMozyfYbaVqHhz4/PEd0TG9HS9uJM9O2adsoBjlc7d5joA3EML32s+Ql7uRz2umSUJ3REhDFmCVY8Nit41Tzkumt8b0OfoeghTQeIYHHhMuHvSUKh6GncSqjJWQVRmdRVbDlxnnux58W8UMnBwgznIgQuTgYmeGn6jlrmDoSweAx2EQRh02+SDIeyayr+6IizWnIg0/UENkpPIj4WFPypnXNEGo1lliOYOfIpr9gX7oP27fDJLg8zPI4jAgpuFRPthANrVG7WcbqpXdd06o9gnjNiaIpIi13968WcEVO0iwzwbRrdleZ/HUAMMYY2n3wYdXH/oB2uhthtJW8HsG2swSUCvQ8xTUXEMUxB6KMHvAN4/qcZn7KTXBOfGUnDhK3IntFhSQ3zPIICZG4mKIYcN6YxtPllOR89KFc56asJq8ziqoEj3TuoW+QucBAjqvZKalBCXiCdiz20xAxk/lflv4vi3MTd89vKX1EkRtfDEtIXQDiq6gGtMspWpoWVAiRloP1AYsEI9Z/2eUilOxU4rFkXQ+uTQWzUSCyVc83azm1kc5DSno/Vwx4dj70orJoh5oAYZRgjb8xHTgHISDWJn1SYg45zKfVpFc+gLmnV7W8UrkZnCEWGWqulsvkaiHVx+mlcci07vpqHkMZ+7jTFkncbtm0bpOQvemeLWgIt6y2KxJZhhnf1A7LTKh/NTLGbl3H9fUZMp9REZRZgMXVzNXljxqEnGpUfT1KsacJSCMkunumGRSmbby1yHbY45PENF80PmCmUlTfwVaO7AVWRtuCeP5j4JKvB7nteUmNWwDufakUvIiZP/h9X/82JLyknGo3XwlB2UNy2Fatcvln6YoYS6IPlNOAROEeKpT4beRNuTPRKrYhnGwcQaeTeU73iwI3S3S+cgE56gkZrWk9o58eRXD5pX96+2XICI77wmg2DGaUOCr+SEMAhPu81SyjkzfwVhwYGtpcewWb8JoUFtPmDaw4xMhY9BWW+waxd+0M+cVSAqeVdv3a7VFN5+aaxV+KYBDmIZ8PxPXnIwrvNvPsUzDeRPHdsKR8J5/d2uA3lenUKiithvMUfuYVyk3UmUrq2xi+UkYXa0UrGhxFEY5dyancbml/OrRt3cOPQz5tW5+PBqylFkzMq4tnyOW9eeA2dMz8qvEgoR/MRHLbYf+uakDQd2+VRCWdJsRoFFeJRdILCPdyg4EpED2W+S9rOlD38IZiiaIZypaxMaaaAK0i+fZAOLK7QEYMs/nQ+jM4BUCTXmmkacv2yEQlzQ+xpzYkM+Okce0RxqAF7cvxhpl+ycmVvMljYxUdzbUG2eY31fgF+Emp9qQhInMQHgNv/jHJSH9h8faYXBe3F4U2tsA3DBuLEl8Lp0I6AKJWVGjMKzUsE3Oc4kirDBTUPRs2tWhVUfShB8KU3TaS8Qt6+y6hB3l7rkhWikL339sgnTF2cIQ3Ncc7v5+UucLP36zW+wgWGeMcl28Vlb6oUflZV7ERq4CyyVLYwkaC7GYiHgWFx+OpHBVH4td7k5GNFqDJlS0/a9FtlyC0uRKQxG7etbQyGSUzxpuyMWHZ4ME62g7zF9vI3mLo5UdAiHGOlLLuT4vfq+3HFwR5AweqSeows+KDj4YosAbQ+0y/cwqEYgIyALMFvxNeK/SWOVQzd14CQWbJNif5idJDCmIWOeW98MnlzdLpAISg6GM62uSohkk0RqipyJmrujwlw6VIM8fQ3l2h5f8g8FzEcqjjGNgv6BWLBVe1MvGnHXI+w38Akl54IEpgVepktGrBBImlvK6zQv7I1V1QvxaFRyGJ6VRKYwIW9ZHt8CthSoE2o697M8HQT58+i/CkFBKdJY0DWCuQ0zQCI8SchX6UMTt1/778Ht7YXspykIt/NeHqkWHeSNDTBzpNXNkYJTYPmtprdK+guEtbVq2bpuXKliyr3fkxf9Fsv5f072qp1as5uEopdtiPRWmDYu0OwIqzw8tjlo/1VybQ8nT/szHM8ftJTFDVem5zDo6PuPwPpdFLA9pR8GVmZmlrcvjAzDXMEW5E4ZDIYTUmtzM9CNg4nQAmOifXwGN6Rrrz46GgbwvBiT1AQcgwC1idM5vHwQckayrZ+zfmtXOhi3C4kifrmCOQd9h3mq+ZWP1SmGRR1rYdKUjKuZPFvaBipVzM3hbpYXa631EhXCRjJu6D2DMFEQHWoBjALp5ebNMp4oPg4GzAbOzFcmUVilFi3cGKD+jdMvsikOrIOKUJUMljWjPJ+putL1idx+Yw3LJIgYi2p3RHrm5c7SkxFCxRzPylnC5BVW1UUzELMJL3KXyWM9aG5WtVngx0shi3KYRa7LAifN5WODmPjsLgnZ8na0gpKxnd8AtEK/Jz+17D1t/4Y21PD6uEOQLMVIp+aX74r9UkkVMEMq5jFBEJTlTRKY9BBzl29RLSWt61VmxSyaKLaZRnTLTdLASKXAVys9LSI6CLDLjUi4BfRpJmc5UyrMTb659hMkNqEgmCRu9NGJY0oybUNHGYBqOWN/3bnpX6LqUahx4u3HNqvFmE3how2ghq4g+FMNW6O2w8qQRiIR4QfqY2YI1BFHFSlDOFzR6mmTqd3lgXWEN1ugnmcYr7TrlzP7LEjvLbmlahVe0AwCzlaqKaPwpnx+jcsznf1NtbFB7Hyf/OzajWDYqWc7X0wDMeuTBLx6oNsEPaXAgM3PWmsTmLW0NoqtpYSXd85wIFUTA+u/HuLRY9AST3k2oBkY49kCGnQ6JKpAMZAwTM1BKqtOwxQ3t2684REysBsjqXJg0iLgfEROzWElrrE+CiIlIJlgS72VXXpry0w7g2CkBysys5H6A1YN5pgnlW7prb5avbax1T9XWOvwiJj99uicLiGimwZ9eWddNQSqVKdM33a1sGvvRXGdJQgdUnxSbxQIMHbZf/eJ3JHeZx8W/Ky4XlBemHY/mmNkN3H9YOvu3oLjk4r+hFdH0COzh0eXMQYW7wploytk+v+uZLounnO63+/7p5pwZmHjO98zA/olZlEwqxXe/BCgtHFDXiuoor8uPUfG6/G+sy0Vjauh19YHC6+rLj41nE8m0XhfljDYtRkrNuqB7FvhqwlbuRmXxpGR7RmzIOMLRG1J143mGQzfuML1QmRBMrF6E61rTjscY+8lvf3MI9ekREx444+rwGdtfHTJjlMSciYllvMw4JzbypiK+scO7VJWRVurKuESCw33tS9BwWwjU9FC1yhIsFtKQUkX93hasxmcrIRXM6UJu4I68ef3DP8MWT4M6YCm57uGHraNoe6hY7e7IxMofWdTLQ4fODmIz3My6X86P1AAQG6aksJIjG6oYXXCf2wtqgXtQx5rQUOcqWmkWSH5WAD99ur92ZUvOyH74RP4rbDLqbxeR6XLm7x5/u9EpRGzJomqyPC37Ho5Nh3d2nyWjTr27z5IDrSBN1SLva0vbBOt6CKPTeiK0xZtEFqw7bdBMROC0x9uLLl43gV7eUX6jG6f31wtZIKVFsXuWxrhbPphKoKBZwjhVvjAqOO3f7CwFI6sTxEynnO7KSMHINDfZeTvOdufFMHM7Okl/UxyGTS39UB+5Gp5VXuJq3Tco6/0tF5khioquxCcWRr5ud6tosnhP62dyZrsQbgndBOx04pR4XW3wXvHu4ae1HqEuLyW6uO30jkFnMW3zF71yJmKHXDs1dFxQbV3+IMPrdNx54Nj9qG+/69uvnulwpNSAvE2xj7Gq7F5TXS3wddXNjcrzd3g0RN6tqVoBuTKBixTFyNS5K3k0RwVdgbKzEHfAhKXOmHD3IUyO5GXxIp/PurpLTEz3a6rS+tlOmC2TP4JmsV1an8CQT+wPmDWsRYDvMoqylLlj6YTaf7jPXH18+8vLXolEmVJ2Qu/0Eg3uDOy6UtL+a7UmvPPudZOPl7c7jWfeeMq3z6hBjuZ/eUlOp0K5/kjB91QObS9A5tNIfRi9ek3VcxljnDsOkZbp4Su2OLINR8oH3Kz6mXEoPiOVjyHyTJxz66yFQbG4DDXTFaTLQKN7d1fGenc+Dp16U5UpiKPcpka3mToPNI4/2FfiLGFmpuVydI3QUNWRS+NmySvJeqAXXndwyFo2oTJ2RAVZAInW1h2Pm5EANYSKHfptfaxY01YyZCpW2KFPxYrK2JYV+OTCAoii+Ts6SkrTkUAJLbyDl2R+EmQXEOLRZYtTNxO218VGguiPUf3kLnYlgM8ItEb03yp62Cgoz8BaTrj119xAes1SLJ1rDSikuLHs8CMjAzXUJkD+1SwamoWx+Z5Wvpb0ZF4HMJh4bXq4R8fUapLERj6OGk2o1jJimEbdMrN2bphlczj2fcCsATZxFN8bQvNRH+5dMs+3MM9Hx9GQ7vwSYXBUuthz2E9qdUNmfTom2dHza2Vej5r9Bv2vdbZwcfj32rVEch3YRrEMZzsH09o5TzKu9KubY1GalbwgOlpDnHHQGItTfI3AxTdUPxUVg34dBcd8676T22cpjJKce8u2lUXOv5hK6Wvy7udPaEA+fg4Pav+uDRWxA5O/hcF3ZEmZKofydiZV0toLJgXlgXJ85A42mPBRY552yG8r52IsrtZuga3WZkY+fq7ACI6rgHKfw2iA0mB05X32YIYm6LmS8j2sugCQyf5+f96xlZIV24CwPiqT+6puh1X5BQ0aGbBeSVMDH+7zfGVTe/YC6DAXB0Ho8ukJeTzEbHSOFjIne4mMlnrmBRYssCWjCxv3kIrzoCz8E30Ji5TMn4jA0lS5JQpWGafK7oqdQzmWfK9zO2Ek6rICLTMVgSZ6LTMeo18CRQXyCJ78nklDT8+Sz40UUSdj3EKmPHyhHCHlZpJW16jKRL4+pQC/NskV1SSGJXNuXzeXq8rR1XkjxD0M4k7Nu7cCazhXoHxWDBNrPm0J1uAVCwnxVA1e56C1Lra501hj66xynpRPFnvr2M3JNPNMce53XuT7hlilZ6t11Rvdy15lLni9Fuuym78d6xXTd2MXqjIzlQkMtS6BGXj8I8UKtEHvg4lMZtqvuc6BmWiEKPVFvKYb6OLaQDa5Rk0OxqnZVN6X8KYGy6w3lGs0OrUFYxdF3cR0Gze7tJEVwGm6/1JPm3SzVtIYDvHZmWB1RXdJdeHa03hs5AqJZPq6c9z8Os3WFT5Y254XbZo17DyDvq5phn088SG85V67VDF3VqtrEnL5AKYI7oVDzX+T4+LkW2hx0pI/H+A6+18xQQQVsvYkgl9phTx6HIyQnIbFTDTaky8eFDf5KCjv0h2o+ct//vKmi59n9qZ9BcN5vMbqW9cVRa+1XbMmoBo/9+j7qDXuStimobWgpQq+AI4lRImMu27phPH5WofzIbxyJQ0vx0BNQYUzLGR/YV3+UyuwO16zCipb1rMgWwoCNFrjRxsatmf7ZrpfxfYWL5Bx1tNfavZpYVc7/ZcBPZEBHW8oE0hmeILWeaJMhqzQvjPHEYRXW4T6w73FrjP9Vb7eNZrghH69HKLXUGQFqz0jp6bcnXddItVl6sVtMvWeh5bavJ7cxu3d2yeN1vDSl/a089V40FPx3DM9dH+w3FtSxrPT51PqZ70+cmnUKrljv6uGTF+SbavwvPxRgM23hhOst5dmG9aQ94LMi4pqhgK7UGIjLdc636+hzvGmXFsFsy7VrrTKc+xm3GZWjSl7HIljmXXxpijPJHmFazINhb0/TzK1AdLbSzJBzcWGAu0c8aoldTRWI43S06X6K76I+mRuy9Pl+y1NFvS6L52jjufMxRsTuWzwZ5+B6M4SHmI4ni7TdWnKbUrfxY49N1F6kaaiZiPsJvP53WPZL7v15tkYQi/VNFRtQpPigI3oSY4dZj2RTd+CnfDMavKpZTD6uHSwp1Fw6zKNRlOQ3YdV4/0Ll7B0HeXnVMhwK5/BDJhQV94KKXaJzHTpgbrOx1IQ3wGfA9XmRkEEwvDdDa62q/cff+tmEGfa1K5qJ+lSkyu9TiB5eT3WGNWYZ6P0MzPvZ8bhZkGjp7JsvWTO+4+/FeQeQBXy+sz0PNoNAieeWkZrBoqqaM0iyueOVfPLMo3VtHERieWwvfdUNOyo2Aln+7pPbidhl95eJrfKiGww3zqHrPPzML7lb3N8O5a0eE2kai5qK687wG2uyIM49Qxms5tTYYMa5NEB2pFgs8fLoviTf4PeUXvjIBL/f/g4bLcpntbmYJN/7Bd69iIZayNocbuiHp0axVYrUBDbT+xLgCH0kfrw31LNvwG6EWgP4eTFL/ZTL9x/arK2KiTKuys+GeDe6+E7vMNi5L7w1z13hM1U8HJNzKq3OwZqlJ53pl1OUHiGvVTtP7H6TFZe+WLuUl7e2esAOrp6xJ6aEJlVgrRjSdl3F3wQKefYFv3Rm10higqdUjx3KZrXv7wmQnbnfad1XJXWczvzxXDt10b3VbkktGBkkF/jSme2NL0YWj8Vxx4HSi8TsGGRwWffLoWoXyrp2IgKIY27q+Af9BhEaU7lgj+xkA0fUS7zE5dRtd/zX1UyU1fJHFAk46oJL0VjXRa52fYZbc0SlHLZPvfYqEywLGyBShXbxdc5OdlzWDOKTUyep+qyZMDDqw95R1wpsMzfcttVyFnyDycc67ChvFrva4+xEYzkLNq1G+/mN7QDjXeZ4XBHHr1/+WlgZ949TPFD1HrDVy5Ggy7a1gQfqD75Q9F9HQUbcixhI1ymW3grN04cYRMhqbyaUe/zMwgLi1sFSccDsYOOQqE5QHoKluQDj0Njpmy+XQHjxh2F5Q+ZLNj0EnLDjkISA52eJTE+xBhGQR7M95psQO1IJjh7Au5dHWbcrXQbllKFb8QwQbRM/F06yolmJvMmlRmS0J0PYsOkZeJJyG0zuDyeupKwyrWRtXu4EFtR81h87302oxhsrN1XNiTziNomWtGadzTa7Da+f/J3Nvp4RZOiD6Tb6rqWJDWt23lHzJs3d79xohiAgMMGwpvHwR1prSzcuHUAYQ7sRDS3sGVYTw9C8c4XItrBiRv8mjCH5ePbh3tClaI7d68yzkRMhQm/WRAz/ZQfn020jKrvWrmcrZtkz/yn3OBxhoqQ8C4D08aGzfswYQw9PUtw2LifJUvK+GRbWWV+N27//Li+9Jim+sd3eyYJxaY9im4RhbO34Zb/GF5MqzmVtAoOXlWateQxpuHJ7es3P9zY8CeHsA+eXZ8ncEg8Pu9ge4gulazwRpidtwdtYZ9ANWzXZI91VEOEvM+Lm02fYdPCCo/KNtUmtLJJSBrPj3qh4DNe/6YxqW1M++Y8erp8LxwxZbY4nkqdLW7GETnHBsnBOQP9cVsT4kmJoUmaT4hdlr0vhn3YZr5TUg4Fk+Nu76EizuOra+ej2v8ZTbK03aWt6HL/FaJ5JOOj+PTp4X+/+z/v74kdp2xN5hF+r12DxvZjIhWXMb/pH0TR2zLNr7ii21irW1dYcv3NxqI086V/wduVwzvYld3Y6/cNO2f2ByD7CyyHz18rimydoDcnxzK4GZ64HDVrpeoMC+uGC6b10lInjkFp3/zBpFzPOm8XDMr9nrvWwmUgO04WK6jCr7ydDlf+OFcPsu6n8AZDC07b94xl11uGh83qz58qc7b9C2mBHRP6fn736EfRpZPjzPtxeUX3EEpXYNb9oIpfOLOu73c9pBIEsaQJa/WKG4rAfu6YybmMKJ+xcFPO1q+Lx5Vu/+PN7PXszeyWSEXevH59e/f6/qd/3r396T/v7/75j7//eHd3O861fW9xkIdHQuNY+V6jrGjmRwV5eNz8YCd7eNz8WHxoCG2pVM0F0aniBX1v3hwC307Vg0lBIg1cAMM/IpCJOe6pOwvLPQHDeb6WOoyq553Zf//x5s3t7c3t7b/f/P3HmdjO/F9mkWy9Ut+D+fHzR6IgkioObvoql8mMPOArjHJhKHZp2zBKFGxA6fb2/PBIuJRPnQdmDTaA4fE85Zmey1FPdZXv7h5KPr7ltFxC5A9K0xuXQoslesJX8Pn9/cvcxfe8sEJzFaZSAElk+5oSpwvgtbffrnEAO9r/vMXQ88VSytmCqtlKcipWM6lWsxeWvy+qv2gdehfPSNkxYjCgEibyt4Ls8CSSCfiuw1QQSBYQxxCTSKa7IjFITavNEH5hbUx69+pVmi04i3S2XLKviGOwLs/xBdVDA5S2cv6nHc5/aJGT6dpLFTJBDfTqRvxFjR7E3c/m9e1x4x/c2wvAP0h0IIhAIuIwFFO/kfdz5X08Uht6Lw74eujzjzY2zrCc5hh+YPug0SoR/tb4iTvTSj1TLzPO5yNUoe4Ddx/Pf8K/k6Hv5o44nZdL16Y/959ZeSbvEwRHedDtlqQH93N/i3oshPOom0LozUnsDct9p9C+gDhc/WGBIQ+70VV7+2sDgSKBCbEUU6Dz0/nO8JRycQ8NHyqbno5O3QyZ4FGRX+oXw6uhZJ7wuS7bbZepmaIZqa/DxfJUl1BL3fOBf8CMvJNKgU6x8ZqReb8pDXiu/cpazFd6p18JMK9YuvnhlYnSeQLJjHzoaPvfXeYXbv57dCf2fumSgQkgqdI13V/n3S3pgWgRsVvrXkh+Woityuei7ebvXgq6bMjUBOT2pJ/vw+zKCfBZaPvsTBMeaOsRML1uHXadAGB5DlaZdhQ3Iy41zLe0s3XISdA2EFobMS+RzIMHQnXchiWXAbsAMgS13om5Dj99dVbQOY6hmBVEzceOnwWzxTEE85IJlEkzFXR20AWQMaib+Z9nQ/1mCGpOtZnTKHQCc1bQOY4hmK2tOcsO0m/ymFiFEBdBWjyp+/rb/Z/EfbWEPKP7msWX6L7uly4Z6L6e2/nrQr3nX4rVkTYesRidJfjihvhS72bgrzOIVa4q7lM+l3DkUZtvzD5LwtUMgaOBfPnkX238mYk0M/P8QwnjnIXLBwYUdH74lNOKLzuUQ7XLpTINSvfy/oBiqfdytYL4pngzH7RmUjQTyPt43JFOO7jMtbyE5cEEZ9XQejzqiHnfiurRCJcrZi1Xc4o9972OpPn+p0z7Skb3ytoADgQOYY9EYb+ez1zVhg4BhGpFjpFBoXxDS1PqxxNBJAspObTyA71I7NcIEzGLnGWi+cnQXo4cU+IWlkje+bVR+LYHQySn1oqKNJyBjgOzlGXvNG5tVgfXVK8B3/Ukj8NsgpPRfOSRa+8W+rZ2LOjPpMuWqQ1A5b/8/wAAAP//AKSNsw==" } diff --git a/metricbeat/module/system/filesystem/_meta/data.json b/metricbeat/module/system/filesystem/_meta/data.json index 2cb9f0897ac..4f13fff0465 100644 --- a/metricbeat/module/system/filesystem/_meta/data.json +++ b/metricbeat/module/system/filesystem/_meta/data.json @@ -1,33 +1,30 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "dataset": "system.filesystem", "duration": 115000, "module": "system" }, "metricset": { - "name": "filesystem" + "name": "filesystem", + "period": 10000 }, "service": { "type": "system" }, "system": { "filesystem": { - "available": 53067177984, - "device_name": "/dev/dm-5", - "files": 0, - "free": 54342492160, - "free_files": 0, - "mount_point": "/var/lib/lxd/storage-pools/default", - "total": 86301999104, - "type": "btrfs", + "available": 148708327424, + "device_name": "/dev/mapper/fedora-root", + "files": 105089024, + "free": 148708327424, + "free_files": 103974920, + "mount_point": "/", + "total": 215211835392, + "type": "xfs", "used": { - "bytes": 31959506944, - "pct": 0.3759 + "bytes": 66503507968, + "pct": 0.309 } } } diff --git a/metricbeat/module/system/filesystem/helper.go b/metricbeat/module/system/filesystem/helper.go index d64ef95f562..4d53ddf2a79 100644 --- a/metricbeat/module/system/filesystem/helper.go +++ b/metricbeat/module/system/filesystem/helper.go @@ -145,20 +145,23 @@ func AddFileSystemUsedPercentage(f *FSStat) { // GetFilesystemEvent turns a stat struct into a MapStr func GetFilesystemEvent(fsStat *FSStat) common.MapStr { - return common.MapStr{ + evt := common.MapStr{ "type": fsStat.SysTypeName, "device_name": fsStat.DevName, "mount_point": fsStat.Mount, "total": fsStat.Total, - "free": fsStat.Free, "available": fsStat.Avail, "files": fsStat.Files, - "free_files": fsStat.FreeFiles, "used": common.MapStr{ "pct": fsStat.UsedPercent, "bytes": fsStat.Used, }, } + if runtime.GOOS != "windows" { + evt.Put("free", fsStat.Free) + evt.Put("free_files", fsStat.FreeFiles) + } + return evt } // Predicate is a function predicate for use with filesystems. It returns true diff --git a/metricbeat/module/system/fsstat/_meta/fields.yml b/metricbeat/module/system/fsstat/_meta/fields.yml index d9d2b0e7ce1..d9bd3acd7f3 100644 --- a/metricbeat/module/system/fsstat/_meta/fields.yml +++ b/metricbeat/module/system/fsstat/_meta/fields.yml @@ -10,7 +10,7 @@ description: Number of file systems found. - name: total_files type: long - description: Total number of files. + description: Total number of files. Not on Windows. - name: total_size format: bytes type: group diff --git a/metricbeat/module/system/fsstat/fsstat.go b/metricbeat/module/system/fsstat/fsstat.go index a8a1eaa0c94..0f43cf4af55 100644 --- a/metricbeat/module/system/fsstat/fsstat.go +++ b/metricbeat/module/system/fsstat/fsstat.go @@ -20,6 +20,7 @@ package fsstat import ( + "runtime" "strings" "github.com/elastic/beats/v7/libbeat/common" @@ -91,16 +92,23 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { totalSizeUsed += stat.Used } - r.Event(mb.Event{ - MetricSetFields: common.MapStr{ - "total_size": common.MapStr{ - "free": totalSizeFree, - "used": totalSizeUsed, - "total": totalSize, - }, - "count": len(fss), - "total_files": totalFiles, + event := common.MapStr{ + "total_size": common.MapStr{ + "free": totalSizeFree, + "used": totalSizeUsed, + "total": totalSize, }, + "count": len(fss), + "total_files": totalFiles, + } + + //We don't get the `Files` field on Windows + if runtime.GOOS == "windows" { + event["total_files"] = totalFiles + } + + r.Event(mb.Event{ + MetricSetFields: event, }) return nil diff --git a/metricbeat/module/system/memory/_meta/data.json b/metricbeat/module/system/memory/_meta/data.json index 01231bede4c..06abee40a35 100644 --- a/metricbeat/module/system/memory/_meta/data.json +++ b/metricbeat/module/system/memory/_meta/data.json @@ -15,13 +15,13 @@ "system": { "memory": { "actual": { - "free": 14247317504, + "free": 8461623296, "used": { - "bytes": 1407057920, - "pct": 0.0899 + "bytes": 7159164928, + "pct": 0.4583 } }, - "free": 4859097088, + "free": 1299234816, "hugepages": { "default_size": 2097152, "free": 0, @@ -41,49 +41,49 @@ }, "page_stats": { "direct_efficiency": { - "pct": 0.9976 + "pct": 0.9242 }, "kswapd_efficiency": { - "pct": 0.6213 + "pct": 0.7518 }, "pgfree": { - "pages": 4382105954 + "pages": 15924304810 }, "pgscan_direct": { - "pages": 485820 + "pages": 1185751 }, "pgscan_kswapd": { - "pages": 77390925 + "pages": 50008148 }, "pgsteal_direct": { - "pages": 484631 + "pages": 1095884 }, "pgsteal_kswapd": { - "pages": 48081976 + "pages": 37594071 } }, "swap": { - "free": 7846490112, + "free": 7823421440, "in": { - "pages": 1111 + "pages": 2702 }, "out": { - "pages": 20255 + "pages": 23582 }, "readahead": { - "cached": 28, - "pages": 65 + "cached": 554, + "pages": 986 }, "total": 7897870336, "used": { - "bytes": 51380224, - "pct": 0.0065 + "bytes": 74448896, + "pct": 0.0094 } }, - "total": 15654375424, + "total": 15620788224, "used": { - "bytes": 10795278336, - "pct": 0.6896 + "bytes": 14321553408, + "pct": 0.9168 } } } diff --git a/metricbeat/module/system/memory/memory.go b/metricbeat/module/system/memory/memory.go index 90283f801be..27e76b85489 100644 --- a/metricbeat/module/system/memory/memory.go +++ b/metricbeat/module/system/memory/memory.go @@ -20,12 +20,16 @@ package memory import ( + "fmt" + + "github.com/pkg/errors" + "github.com/elastic/beats/v7/libbeat/common" mem "github.com/elastic/beats/v7/libbeat/metric/system/memory" "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/metricbeat/mb/parse" - - "github.com/pkg/errors" + linux "github.com/elastic/beats/v7/metricbeat/module/linux/memory" + "github.com/elastic/beats/v7/metricbeat/module/system" ) func init() { @@ -38,11 +42,18 @@ func init() { // MetricSet for fetching system memory metrics. type MetricSet struct { mb.BaseMetricSet + IsFleet bool } // New is a mb.MetricSetFactory that returns a memory.MetricSet. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { - return &MetricSet{base}, nil + + systemModule, ok := base.Module().(*system.Module) + if !ok { + return nil, fmt.Errorf("unexpected module type") + } + + return &MetricSet{BaseMetricSet: base, IsFleet: systemModule.IsAgent}, nil } // Fetch fetches memory metrics from the OS. @@ -103,70 +114,18 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { "pages": vmstat.SwapRa, "cached": vmstat.SwapRaHit, } - pageStats := common.MapStr{ - "pgscan_kswapd": common.MapStr{ - "pages": vmstat.PgscanKswapd, - }, - "pgscan_direct": common.MapStr{ - "pages": vmstat.PgscanDirect, - }, - "pgfree": common.MapStr{ - "pages": vmstat.Pgfree, - }, - "pgsteal_kswapd": common.MapStr{ - "pages": vmstat.PgstealKswapd, - }, - "pgsteal_direct": common.MapStr{ - "pages": vmstat.PgstealDirect, - }, - } - // This is similar to the vmeff stat gathered by sar - // these ratios calculate thhe efficiency of page reclaim - if vmstat.PgscanDirect != 0 { - pageStats["direct_efficiency"] = common.MapStr{ - "pct": common.Round(float64(vmstat.PgstealDirect)/float64(vmstat.PgscanDirect), common.DefaultDecimalPlacesCount), - } - } + } - if vmstat.PgscanKswapd != 0 { - pageStats["kswapd_efficiency"] = common.MapStr{ - "pct": common.Round(float64(vmstat.PgstealKswapd)/float64(vmstat.PgscanKswapd), common.DefaultDecimalPlacesCount), - } + // for backwards compatibility, only report if we're not in fleet mode + if !m.IsFleet { + err := linux.FetchLinuxMemStats(memory) + if err != nil { + return errors.Wrap(err, "error getting page stats") } - - memory["page_stats"] = pageStats } memory["swap"] = swap - hugePagesStat, err := mem.GetHugeTLBPages() - if err != nil { - return errors.Wrap(err, "hugepages") - } - if hugePagesStat != nil { - mem.AddHugeTLBPagesPercentage(hugePagesStat) - thp := common.MapStr{ - "total": hugePagesStat.Total, - "used": common.MapStr{ - "bytes": hugePagesStat.TotalAllocatedSize, - "pct": hugePagesStat.UsedPercent, - }, - "free": hugePagesStat.Free, - "reserved": hugePagesStat.Reserved, - "surplus": hugePagesStat.Surplus, - "default_size": hugePagesStat.DefaultSize, - } - if vmstat != nil { - thp["swap"] = common.MapStr{ - "out": common.MapStr{ - "pages": vmstat.ThpSwpout, - "fallback": vmstat.ThpSwpoutFallback, - }, - } - } - memory["hugepages"] = thp - } - r.Event(mb.Event{ MetricSetFields: memory, }) diff --git a/metricbeat/module/system/process/_meta/data.json b/metricbeat/module/system/process/_meta/data.json index 78ece4aa114..84b74110683 100644 --- a/metricbeat/module/system/process/_meta/data.json +++ b/metricbeat/module/system/process/_meta/data.json @@ -1,29 +1,26 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "dataset": "system.process", "duration": 115000, "module": "system" }, "metricset": { - "name": "process" + "name": "process", + "period": 10000 }, "process": { "args": [ - "/usr/bin/dockerd", - "-H", - "unix://" + "/usr/lib/systemd/systemd", + "--switched-root", + "--system", + "--deserialize", + "28" ], - "executable": "/usr/bin/dockerd-ce", - "name": "dockerd", - "pgid": 2080, - "pid": 2080, - "ppid": 1, - "working_directory": "/" + "name": "systemd", + "pgid": 1, + "pid": 1, + "ppid": 0 }, "service": { "type": "system" @@ -32,11 +29,11 @@ "process": { "cgroup": { "blkio": { - "id": "docker.service", - "path": "/system.slice/docker.service", + "id": "init.scope", + "path": "/init.scope", "total": { - "bytes": 844576104448, - "ios": 54869430 + "bytes": 7453696, + "ios": 548 } }, "cpu": { @@ -49,8 +46,8 @@ }, "shares": 1024 }, - "id": "docker.service", - "path": "/system.slice/docker.service", + "id": "init.scope", + "path": "/init.scope", "rt": { "period": { "us": 0 @@ -68,38 +65,38 @@ } }, "cpuacct": { - "id": "docker.service", - "path": "/system.slice/docker.service", + "id": "init.scope", + "path": "/init.scope", "percpu": { - "1": 7058282754012, - "2": 7053634662537, - "3": 7069386293853, - "4": 7050055153087 + "1": 3930656993407, + "2": 4025787490535, + "3": 4064460082910, + "4": 3387847262532 }, "stats": { "system": { - "ns": 7202920000000 + "ns": 4996000000000 }, "user": { - "ns": 19573240000000 + "ns": 10329380000000 } }, "total": { - "ns": 28231358863489 + "ns": 15408751829384 } }, - "id": "docker.service", + "id": "init.scope", "memory": { - "id": "docker.service", + "id": "init.scope", "kmem": { "failures": 0, "limit": { "bytes": 9223372036854771712 }, "usage": { - "bytes": 21139456, + "bytes": 9404416, "max": { - "bytes": 480030720 + "bytes": 14987264 } } }, @@ -121,95 +118,88 @@ "bytes": 9223372036854771712 }, "usage": { - "bytes": 3337703424, + "bytes": 29437952, "max": { - "bytes": 5245300736 + "bytes": 70705152 } } }, "memsw": { "failures": 0, "limit": { - "bytes": 0 + "bytes": 9223372036854771712 }, "usage": { - "bytes": 0, + "bytes": 30392320, "max": { - "bytes": 0 + "bytes": 70705152 } } }, - "path": "/system.slice/docker.service", + "path": "/init.scope", "stats": { "active_anon": { - "bytes": 779677696 + "bytes": 3444736 }, "active_file": { - "bytes": 1753378816 + "bytes": 10563584 }, "cache": { - "bytes": 2127810560 + "bytes": 10752000 }, "hierarchical_memory_limit": { "bytes": 9223372036854771712 }, "hierarchical_memsw_limit": { - "bytes": 0 + "bytes": 9223372036854771712 }, "inactive_anon": { - "bytes": 409075712 + "bytes": 6197248 }, "inactive_file": { - "bytes": 374431744 + "bytes": 327680 }, - "major_page_faults": 53164, + "major_page_faults": 198, "mapped_file": { - "bytes": 7491584 + "bytes": 9867264 }, - "page_faults": 21923702, - "pages_in": 57261049, - "pages_out": 56521859, + "page_faults": 3626304, + "pages_in": 1095732, + "pages_out": 1090806, "rss": { - "bytes": 1188753408 + "bytes": 9592832 }, "rss_huge": { - "bytes": 56623104 + "bytes": 0 }, "swap": { - "bytes": 0 + "bytes": 675840 }, "unevictable": { "bytes": 0 } } }, - "path": "/system.slice/docker.service" + "path": "/init.scope" }, - "cmdline": "/usr/bin/dockerd -H unix://", + "cmdline": "/usr/lib/systemd/systemd --switched-root --system --deserialize 28", "cpu": { - "start_time": "2019-01-08T17:06:39.000Z", + "start_time": "2020-08-27T01:05:16.000Z", "total": { "norm": { - "pct": 0.0049 + "pct": 0.0056 }, - "pct": 0.0195, - "value": 27827810 + "pct": 0.0222, + "value": 15389060 } }, - "fd": { - "limit": { - "hard": 1048576, - "soft": 1048576 - }, - "open": 68 - }, "memory": { "rss": { - "bytes": 1187336192, - "pct": 0.0716 + "bytes": 12853248, + "pct": 0.0008 }, - "share": 8253440, - "size": 4438523904 + "share": 7118848, + "size": 176881664 }, "state": "sleeping" } @@ -217,4 +207,4 @@ "user": { "name": "root" } -} +} \ No newline at end of file diff --git a/metricbeat/module/system/process/_meta/fields.yml b/metricbeat/module/system/process/_meta/fields.yml index 51ab8b81e09..fe941031551 100644 --- a/metricbeat/module/system/process/_meta/fields.yml +++ b/metricbeat/module/system/process/_meta/fields.yml @@ -96,17 +96,27 @@ type: long format: bytes description: > - The Resident Set Size. The amount of memory the process occupied in main memory (RAM). On Windows this represents the current working set size, in bytes. + The Resident Set Size. The amount of memory the process occupied in main memory (RAM). On Windows this represents the current working set size, in bytes. Not available on Windows. - name: rss.pct type: scaled_float format: percent description: > - The percentage of memory the process occupied in main memory (RAM). + The percentage of memory the process occupied in main memory (RAM). Not available on Windows. + - name: wss.bytes + type: long + format: bytes + description: > + The Working Set Size. The amount of memory the process occupied in main memory (RAM). Windows only. + - name: wss.pct + type: scaled_float + format: percent + description: > + The percentage of memory the process occupied in main memory (RAM). Windows only. - name: share type: long format: bytes description: > - The shared memory the process uses. + The shared memory the process uses. Not available on Windows. - name: fd type: group description: > diff --git a/metricbeat/module/system/process/process.go b/metricbeat/module/system/process/process.go index 9c754177372..804c62d06d6 100644 --- a/metricbeat/module/system/process/process.go +++ b/metricbeat/module/system/process/process.go @@ -46,9 +46,10 @@ func init() { // MetricSet that fetches process metrics. type MetricSet struct { mb.BaseMetricSet - stats *process.Stats - cgroup *cgroup.Reader - perCPU bool + stats *process.Stats + cgroup *cgroup.Reader + perCPU bool + IsAgent bool } // New creates and returns a new MetricSet. @@ -58,6 +59,11 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { return nil, err } + systemModule, ok := base.Module().(*system.Module) + if !ok { + return nil, fmt.Errorf("unexpected module type") + } + m := &MetricSet{ BaseMetricSet: base, stats: &process.Stats{ @@ -67,7 +73,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { CacheCmdLine: config.CacheCmdLine, IncludeTop: config.IncludeTop, }, - perCPU: config.IncludePerCPU, + perCPU: config.IncludePerCPU, + IsAgent: systemModule.IsAgent, } err := m.stats.Init() if err != nil { @@ -75,11 +82,6 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } if runtime.GOOS == "linux" { - systemModule, ok := base.Module().(*system.Module) - if !ok { - return nil, fmt.Errorf("unexpected module type") - } - if config.Cgroups == nil || *config.Cgroups { debugf("process cgroup data collection is enabled, using hostfs='%v'", systemModule.HostFS) m.cgroup, err = cgroup.NewReader(systemModule.HostFS, true) @@ -148,6 +150,22 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { rootFields.Put("process.args", args) } + // This is a temporary fix until we make these changes global across libbeat + // This logic should happen in libbeat getProcessEvent() + + // There's some more Windows memory quirks we need to deal with. + // "rss" is a linux concept, but "wss" is a direct match on Windows. + // "share" is also unavailable on Windows. + + if m.IsAgent { + if runtime.GOOS == "windows" { + proc.Delete("memory.share") + if setSize := getAndRemove(proc, "memory.rss"); setSize != nil { + proc.Put("memory.wss", setSize) + } + } + } + e := mb.Event{ RootFields: rootFields, MetricSetFields: proc, diff --git a/metricbeat/module/system/process_summary/_meta/data.json b/metricbeat/module/system/process_summary/_meta/data.json index 363e5f77a0d..159defcb769 100644 --- a/metricbeat/module/system/process_summary/_meta/data.json +++ b/metricbeat/module/system/process_summary/_meta/data.json @@ -1,16 +1,13 @@ { "@timestamp": "2017-10-12T08:05:34.853Z", - "agent": { - "hostname": "host.example.com", - "name": "host.example.com" - }, "event": { "dataset": "system.process.summary", "duration": 115000, "module": "system" }, "metricset": { - "name": "process_summary" + "name": "process_summary", + "period": 10000 }, "service": { "type": "system" @@ -19,13 +16,13 @@ "process": { "summary": { "dead": 0, - "idle": 112, - "running": 0, - "sleeping": 288, + "idle": 63, + "running": 3, + "sleeping": 117, "stopped": 0, - "total": 401, - "unknown": 0, - "zombie": 1 + "total": 185, + "unknown": 2, + "zombie": 0 } } } diff --git a/metricbeat/module/system/process_summary/process_summary.go b/metricbeat/module/system/process_summary/process_summary.go index 071a7b2a660..5edb31f9182 100644 --- a/metricbeat/module/system/process_summary/process_summary.go +++ b/metricbeat/module/system/process_summary/process_summary.go @@ -20,6 +20,8 @@ package process_summary import ( + "runtime" + "github.com/pkg/errors" "github.com/elastic/beats/v7/libbeat/common" @@ -79,7 +81,7 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { state := sigar.ProcState{} err = state.Get(pid) if err != nil { - summary.unknown += 1 + summary.unknown++ continue } @@ -104,15 +106,25 @@ func (m *MetricSet) Fetch(r mb.ReporterV2) error { } } - event := common.MapStr{ - "total": len(pids), - "sleeping": summary.sleeping, - "running": summary.running, - "idle": summary.idle, - "stopped": summary.stopped, - "zombie": summary.zombie, - "unknown": summary.unknown, - "dead": summary.dead, + event := common.MapStr{} + if runtime.GOOS == "windows" { + event = common.MapStr{ + "total": len(pids), + "sleeping": summary.sleeping, + "running": summary.running, + "unknown": summary.unknown, + } + } else { + event = common.MapStr{ + "total": len(pids), + "sleeping": summary.sleeping, + "running": summary.running, + "idle": summary.idle, + "stopped": summary.stopped, + "zombie": summary.zombie, + "unknown": summary.unknown, + "dead": summary.dead, + } } r.Event(mb.Event{ diff --git a/metricbeat/module/system/process_summary/process_summary_test.go b/metricbeat/module/system/process_summary/process_summary_test.go index 55afce8f190..c95e361224c 100644 --- a/metricbeat/module/system/process_summary/process_summary_test.go +++ b/metricbeat/module/system/process_summary/process_summary_test.go @@ -20,6 +20,7 @@ package process_summary import ( + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -53,18 +54,26 @@ func TestFetch(t *testing.T) { event, ok := summary.(common.MapStr) require.True(t, ok) - assert.Contains(t, event, "total") - assert.Contains(t, event, "sleeping") - assert.Contains(t, event, "running") - assert.Contains(t, event, "idle") - assert.Contains(t, event, "stopped") - assert.Contains(t, event, "zombie") - assert.Contains(t, event, "unknown") + if runtime.GOOS == "windows" { + assert.Contains(t, event, "total") + assert.Contains(t, event, "sleeping") + assert.Contains(t, event, "running") + assert.Contains(t, event, "unknown") + total := event["sleeping"].(int) + event["running"].(int) + event["unknown"].(int) + assert.Equal(t, event["total"].(int), total) + } else { + assert.Contains(t, event, "total") + assert.Contains(t, event, "sleeping") + assert.Contains(t, event, "running") + assert.Contains(t, event, "idle") + assert.Contains(t, event, "stopped") + assert.Contains(t, event, "zombie") + assert.Contains(t, event, "unknown") + total := event["sleeping"].(int) + event["running"].(int) + event["idle"].(int) + + event["stopped"].(int) + event["zombie"].(int) + event["unknown"].(int) - total := event["sleeping"].(int) + event["running"].(int) + event["idle"].(int) + - event["stopped"].(int) + event["zombie"].(int) + event["unknown"].(int) - - assert.Equal(t, event["total"].(int), total) + assert.Equal(t, event["total"].(int), total) + } } func getConfig() map[string]interface{} { diff --git a/metricbeat/module/system/system.go b/metricbeat/module/system/system.go index edb9f55bf6b..f1f060a90bc 100644 --- a/metricbeat/module/system/system.go +++ b/metricbeat/module/system/system.go @@ -21,10 +21,12 @@ import ( "flag" "sync" + "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/metricbeat/mb" ) var ( + // HostFS is an alternate mountpoint for the filesytem root, for when metricbeat is running inside a container. HostFS = flag.String("system.hostfs", "", "mountpoint of the host's filesystem for use in monitoring a host from within a container") ) @@ -37,16 +39,48 @@ func init() { } } +// Module represents the system module type Module struct { mb.BaseModule - HostFS string // Mountpoint of the host's filesystem for use in monitoring inside a container. + HostFS string // Mountpoint of the host's filesystem for use in monitoring inside a container. + IsAgent bool // Looks to see if metricbeat is running under agent. Useful if we have breaking changes in one but not the other. } +// NewModule instatiates the system module func NewModule(base mb.BaseModule) (mb.Module, error) { // This only needs to be configured once for all system modules. once.Do(func() { initModule() }) - return &Module{BaseModule: base, HostFS: *HostFS}, nil + return &Module{BaseModule: base, HostFS: *HostFS, IsAgent: checkMgmtFlags()}, nil +} + +// checkMgmtFlags checks to see if metricbeat is running under Agent +// The management setting is stored in the main Beat runtime object, but we can't see that from a module +// So instead we check the CLI flags, since Agent starts metricbeat with "-E", "management.mode=x-pack-fleet", "-E", "management.enabled=true" +func checkMgmtFlags() bool { + type management struct { + Mode string `config:"management.mode"` + Enabled bool `config:"management.enabled"` + } + var managementSettings management + + cfgFlag := flag.Lookup("E") + if cfgFlag == nil { + return false + } + + CfgObject, _ := cfgFlag.Value.(*common.SettingsFlag) + cliCfg := CfgObject.Config() + + err := cliCfg.Unpack(&managementSettings) + if err != nil { + return false + } + + if managementSettings.Enabled == true && managementSettings.Mode == "x-pack-fleet" { + return true + } + return false } diff --git a/metricbeat/module/system/test_system.py b/metricbeat/module/system/test_system.py index f689b99fb4c..0cfb820c18a 100644 --- a/metricbeat/module/system/test_system.py +++ b/metricbeat/module/system/test_system.py @@ -7,13 +7,23 @@ import unittest -SYSTEM_CPU_FIELDS = ["cores", "idle.pct", "iowait.pct", "irq.pct", "nice.pct", - "softirq.pct", "steal.pct", "system.pct", "user.pct", "total.pct"] +SYSTEM_CPU_FIELDS_LINUX = ["cores", "idle.pct", "iowait.pct", "irq.pct", "nice.pct", + "softirq.pct", "steal.pct", "system.pct", "user.pct", "total.pct"] -SYSTEM_CPU_FIELDS_ALL = ["cores", "idle.pct", "idle.ticks", "iowait.pct", "iowait.ticks", "irq.pct", "irq.ticks", "nice.pct", "nice.ticks", - "softirq.pct", "softirq.ticks", "steal.pct", "steal.ticks", "system.pct", "system.ticks", "user.pct", "user.ticks", - "idle.norm.pct", "iowait.norm.pct", "irq.norm.pct", "nice.norm.pct", "softirq.norm.pct", - "steal.norm.pct", "system.norm.pct", "user.norm.pct", "total.norm.pct", "total.value"] +SYSTEM_CPU_FIELDS_WINDOWS = ["cores", "idle.pct", "system.pct", "user.pct", "total.pct"] + +SYSTEM_CPU_FIELDS_DARWIN = ["cores", "idle.pct", "system.pct", "user.pct", "total.pct", "nice.pct"] + +SYSTEM_CPU_FIELDS_LINUX_ALL = ["cores", "idle.pct", "idle.ticks", "iowait.pct", "iowait.ticks", "irq.pct", "irq.ticks", "nice.pct", "nice.ticks", + "softirq.pct", "softirq.ticks", "steal.pct", "steal.ticks", "system.pct", "system.ticks", "user.pct", "user.ticks", + "idle.norm.pct", "iowait.norm.pct", "irq.norm.pct", "nice.norm.pct", "softirq.norm.pct", + "steal.norm.pct", "system.norm.pct", "user.norm.pct", "total.norm.pct", "total.value"] + +SYSTEM_CPU_FIELDS_WINDOWS_ALL = ["cores", "idle.pct", "idle.ticks", "system.pct", "system.ticks", "user.pct", "user.ticks", + "idle.norm.pct", "system.norm.pct", "user.norm.pct", "total.norm.pct", "total.value"] + +SYSTEM_CPU_FIELDS_DARWIN_ALL = ["cores", "idle.pct", "idle.ticks", "nice.pct", "nice.ticks", "system.pct", "system.ticks", "user.pct", "user.ticks", + "idle.norm.pct", "nice.norm.pct", "system.norm.pct", "user.norm.pct", "total.norm.pct", "total.value"] SYSTEM_LOAD_FIELDS = ["cores", "1", "5", "15", "norm.1", "norm.5", "norm.15"] @@ -37,6 +47,10 @@ "free_files", "mount_point", "total", "used.bytes", "used.pct"] +SYSTEM_FILESYSTEM_FIELDS_WINDOWS = ["available", "device_name", "type", "files", + "mount_point", "total", "used.bytes", + "used.pct"] + SYSTEM_FSSTAT_FIELDS = ["count", "total_files", "total_size"] SYSTEM_MEMORY_FIELDS = ["swap", "actual.free", "free", "total", "used.bytes", "used.pct", "actual.used.bytes", @@ -82,7 +96,12 @@ def test_cpu(self): if "system" in evt: cpu = evt["system"]["cpu"] - self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS), cpu.keys()) + if sys.platform.startswith("linux"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_LINUX), cpu.keys()) + elif sys.platform.startswith("darwin"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_DARWIN), cpu.keys()) + elif sys.platform.startswith("win"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_WINDOWS), cpu.keys()) else: host_cpu = evt["host"]["cpu"] self.assertCountEqual(self.de_dot(SYSTEM_CPU_HOST_FIELDS), host_cpu.keys()) @@ -111,7 +130,12 @@ def test_cpu_ticks_option(self): for evt in output: self.assert_fields_are_documented(evt) cpuStats = evt["system"]["cpu"] - self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_ALL), cpuStats.keys()) + if sys.platform.startswith("linux"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_LINUX_ALL), cpuStats.keys()) + elif sys.platform.startswith("win"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_WINDOWS_ALL), cpuStats.keys()) + elif sys.platform.startswith("darwin"): + self.assertCountEqual(self.de_dot(SYSTEM_CPU_FIELDS_DARWIN_ALL), cpuStats.keys()) @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_core(self): @@ -261,7 +285,10 @@ def test_filesystem(self): for evt in output: self.assert_fields_are_documented(evt) filesystem = evt["system"]["filesystem"] - self.assertCountEqual(self.de_dot(SYSTEM_FILESYSTEM_FIELDS), filesystem.keys()) + if sys.platform.startswith("win"): + self.assertCountEqual(self.de_dot(SYSTEM_FILESYSTEM_FIELDS_WINDOWS), filesystem.keys()) + else: + self.assertCountEqual(self.de_dot(SYSTEM_FILESYSTEM_FIELDS), filesystem.keys()) @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd|openbsd", sys.platform), "os") def test_fsstat(self): @@ -378,13 +405,17 @@ def test_process_summary(self): assert isinstance(summary["total"], int) assert isinstance(summary["sleeping"], int) assert isinstance(summary["running"], int) - assert isinstance(summary["idle"], int) - assert isinstance(summary["stopped"], int) - assert isinstance(summary["zombie"], int) assert isinstance(summary["unknown"], int) - assert summary["total"] == summary["sleeping"] + summary["running"] + \ - summary["idle"] + summary["stopped"] + summary["zombie"] + summary["unknown"] + if not sys.platform.startswith("win"): + assert isinstance(summary["idle"], int) + assert isinstance(summary["stopped"], int) + assert isinstance(summary["zombie"], int) + assert summary["total"] == summary["sleeping"] + summary["running"] + \ + summary["idle"] + summary["stopped"] + summary["zombie"] + summary["unknown"] + + if sys.platform.startswith("windows"): + assert summary["total"] == summary["sleeping"] + summary["running"] + summary["unknown"] @unittest.skipUnless(re.match("(?i)win|linux|darwin|freebsd", sys.platform), "os") def test_process(self): diff --git a/metricbeat/modules.d/linux.yml.disabled b/metricbeat/modules.d/linux.yml.disabled index a01989aa0c9..22e675cafff 100644 --- a/metricbeat/modules.d/linux.yml.disabled +++ b/metricbeat/modules.d/linux.yml.disabled @@ -5,8 +5,10 @@ period: 10s metricsets: - "pageinfo" + - "memory" # - ksm # - conntrack + # - iostat enabled: true #hostfs: /hostfs diff --git a/metricbeat/tests/system/test_processors.py b/metricbeat/tests/system/test_processors.py index 2f7d131d119..a680c049d67 100644 --- a/metricbeat/tests/system/test_processors.py +++ b/metricbeat/tests/system/test_processors.py @@ -12,13 +12,13 @@ def test_drop_fields(self): self.render_config_template( modules=[{ "name": "system", - "metricsets": ["cpu"], + "metricsets": ["network"], "period": "1s" }], processors=[{ "drop_fields": { "when": "range.system.cpu.system.pct.lt: 0.1", - "fields": ["system.cpu.load"], + "fields": ["system.network.in"], }, }] ) @@ -27,7 +27,7 @@ def test_drop_fields(self): proc.check_kill_and_wait() output = self.read_output_json() - self.assertEqual(len(output), 1) + self.assertGreater(len(output), 1) evt = output[0] self.assert_fields_are_documented(evt) @@ -37,12 +37,9 @@ def test_drop_fields(self): 'agent', '@timestamp', 'system', 'metricset.module', 'metricset.rtt', 'metricset.name', 'host', 'service', 'ecs', 'event' ]), evt.keys()) - cpu = evt["system"]["cpu"] - print(list(cpu.keys())) - self.assertCountEqual(self.de_dot([ - "system", "cores", "user", "softirq", "iowait", - "idle", "irq", "steal", "nice", "total" - ]), cpu.keys()) + network = evt["system"]["network"] + print(list(network.keys())) + self.assertCountEqual(self.de_dot(["name", "out", "in"]), network.keys()) def test_dropfields_with_condition(self): """ diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 17c9234ea9f..bd99c2cf508 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -902,8 +902,10 @@ metricbeat.modules: period: 10s metricsets: - "pageinfo" + - "memory" # - ksm # - conntrack + # - iostat enabled: true #hostfs: /hostfs