Skip to content

Commit

Permalink
Merge pull request #8 from yimuniao/master
Browse files Browse the repository at this point in the history 
add allowlist in cadvisor file system extractor to avoid extra logs
  • Loading branch information
@pxaws
pxaws authored Jun 26, 2020
2 parents f127a75 + 7b02ce8 commit 6d42494
Show file tree
Hide file tree
Showing 2 changed files with 232 additions and 1 deletion.
21 changes: 20 additions & 1 deletion plugins/inputs/cadvisor/extractors/fs_extractor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,16 @@ import (
. "github.com/aws/amazon-cloudwatch-agent/internal/containerinsightscommon"
cinfo "github.com/google/cadvisor/info/v1"
"log"
"regexp"
"time"
)

const (
allowList = "^tmpfs$|^/dev/|^overlay$"
)

type FileSystemMetricExtractor struct {
allowListRegexP *regexp.Regexp
}

func (f *FileSystemMetricExtractor) HasValue(info *cinfo.ContainerInfo) bool {
Expand All @@ -22,11 +28,16 @@ func (f *FileSystemMetricExtractor) GetValue(info *cinfo.ContainerInfo, containe

containerType = getFSMetricType(containerType)
stats := GetStats(info)

for _, v := range stats.Filesystem {
metric := newCadvisorMetric(containerType)
if v.Device == "" {
continue
}
if f.allowListRegexP != nil && !f.allowListRegexP.MatchString(v.Device) {
continue
}

metric.tags[DiskDev] = v.Device
metric.tags[FSType] = v.Type

Expand All @@ -52,7 +63,15 @@ func (f *FileSystemMetricExtractor) CleanUp(now time.Time) {
}

func NewFileSystemMetricExtractor() *FileSystemMetricExtractor {
return &FileSystemMetricExtractor{}
fse := &FileSystemMetricExtractor{}
if p, err := regexp.Compile(allowList); err == nil {
log.Printf("I! NewFileSystemMetricExtractor set regex succeed")
fse.allowListRegexP = p
} else {
log.Printf("E! NewFileSystemMetricExtractor set regex failed: %v", err)
}

return fse
}

func getFSMetricType(containerType string) string {
Expand Down
212 changes: 212 additions & 0 deletions plugins/inputs/cadvisor/extractors/fs_extractor_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,175 @@ import (

. "github.com/aws/amazon-cloudwatch-agent/internal/containerinsightscommon"
cinfo "github.com/google/cadvisor/info/v1"
"github.com/stretchr/testify/assert"
)

const (
testAllowList = `
{
"/kubepods/besteffort/podaf16b540-4ae2-11e9-977b-0672b6c6fc94/573ee6cd04a6208af809b2329652c74386f1992faca8662c733d7f250014e718": {
"id": "573ee6cd04a6208af809b2329652c74386f1992faca8662c733d7f250014e718",
"name": "/kubepods/besteffort/podaf16b540-4ae2-11e9-977b-0672b6c6fc94/573ee6cd04a6208af809b2329652c74386f1992faca8662c733d7f250014e718",
"aliases": [
"k8s_ubuntu_stress-1-core-mh2pn_default_af16b540-4ae2-11e9-977b-0672b6c6fc94_0",
"573ee6cd04a6208af809b2329652c74386f1992faca8662c733d7f250014e718"
],
"namespace": "docker",
"spec": {
"creation_time": "2019-03-20T07:35:09.746280405Z",
"labels": {
"annotation.io.kubernetes.container.hash": "70bfcd85",
"annotation.io.kubernetes.container.restartCount": "0",
"annotation.io.kubernetes.container.terminationMessagePath": "/dev/termination-log",
"annotation.io.kubernetes.container.terminationMessagePolicy": "File",
"annotation.io.kubernetes.pod.terminationGracePeriod": "30",
"io.kubernetes.container.logpath": "/var/log/pods/af16b540-4ae2-11e9-977b-0672b6c6fc94/ubuntu/0.log",
"io.kubernetes.container.name": "ubuntu",
"io.kubernetes.docker.type": "container",
"io.kubernetes.pod.name": "stress-1-core-mh2pn",
"io.kubernetes.pod.namespace": "default",
"io.kubernetes.pod.uid": "af16b540-4ae2-11e9-977b-0672b6c6fc94",
"io.kubernetes.sandbox.id": "a5bb552d7fb8e5014468756f165732e0c6bcd9dcbd229efc51afc014317d20d6"
},
"has_cpu": true,
"cpu": {
"limit": 2,
"max_limit": 0,
"mask": "0-3",
"period": 100000
},
"has_memory": true,
"memory": {
"limit": 9223372036854771712,
"reservation": 9223372036854771712,
"swap_limit": 9223372036854771712
},
"has_network": false,
"has_filesystem": true,
"has_diskio": false,
"has_custom_metrics": false,
"image": "ubuntu@sha256:017eef0b616011647b269b5c65826e2e2ebddbe5d1f8c1e56b3599fb14fabec8"
},
"stats": [
{
"timestamp": "2019-04-09T22:26:42.984081498Z",
"filesystem": [
{
"device": "tmpfs",
"type": "vfs",
"capacity": 21462233088,
"usage": 25661440,
"base_usage": 25640960,
"available": 0,
"has_inodes": false,
"inodes": 67,
"inodes_free": 0,
"reads_completed": 0,
"reads_merged": 0,
"sectors_read": 0,
"read_time": 0,
"writes_completed": 0,
"writes_merged": 0,
"sectors_written": 0,
"write_time": 0,
"io_in_progress": 0,
"io_time": 0,
"weighted_io_time": 0
},
{
"device": "/dev/xvda1",
"type": "vfs",
"capacity": 21462233088,
"usage": 25661440,
"base_usage": 25640960,
"available": 0,
"has_inodes": false,
"inodes": 67,
"inodes_free": 0,
"reads_completed": 0,
"reads_merged": 0,
"sectors_read": 0,
"read_time": 0,
"writes_completed": 0,
"writes_merged": 0,
"sectors_written": 0,
"write_time": 0,
"io_in_progress": 0,
"io_time": 0,
"weighted_io_time": 0
},
{
"device": "overlay",
"type": "vfs",
"capacity": 21462233088,
"usage": 25661440,
"base_usage": 25640960,
"available": 0,
"has_inodes": false,
"inodes": 67,
"inodes_free": 0,
"reads_completed": 0,
"reads_merged": 0,
"sectors_read": 0,
"read_time": 0,
"writes_completed": 0,
"writes_merged": 0,
"sectors_written": 0,
"write_time": 0,
"io_in_progress": 0,
"io_time": 0,
"weighted_io_time": 0
},
{
"device": "/dev",
"type": "vfs",
"capacity": 21462233088,
"usage": 25661440,
"base_usage": 25640960,
"available": 0,
"has_inodes": false,
"inodes": 67,
"inodes_free": 0,
"reads_completed": 0,
"reads_merged": 0,
"sectors_read": 0,
"read_time": 0,
"writes_completed": 0,
"writes_merged": 0,
"sectors_written": 0,
"write_time": 0,
"io_in_progress": 0,
"io_time": 0,
"weighted_io_time": 0
},
{
"device": "overlaytest",
"type": "vfs",
"capacity": 21462233088,
"usage": 25661440,
"base_usage": 25640960,
"available": 0,
"has_inodes": false,
"inodes": 67,
"inodes_free": 0,
"reads_completed": 0,
"reads_merged": 0,
"sectors_read": 0,
"read_time": 0,
"writes_completed": 0,
"writes_merged": 0,
"sectors_written": 0,
"write_time": 0,
"io_in_progress": 0,
"io_time": 0,
"weighted_io_time": 0
}
]
}
]
}
}
`
)

func TestFSStats(t *testing.T) {
Expand Down Expand Up @@ -49,3 +218,46 @@ func TestFSStats(t *testing.T) {
}
AssertContainsTaggedField(t, cMetrics[0], expectedFields, expectedTags)
}

func TestAllowList(t *testing.T) {
extractor := NewFileSystemMetricExtractor()
assert.Equal(t, true, extractor.allowListRegexP.MatchString("/dev/shm"))
assert.Equal(t, true, extractor.allowListRegexP.MatchString("tmpfs"))
assert.Equal(t, true, extractor.allowListRegexP.MatchString("overlay"))
assert.Equal(t, false, extractor.allowListRegexP.MatchString("overlaytest"))
assert.Equal(t, false, extractor.allowListRegexP.MatchString("/dev"))
}

func TestFSStatsWithAllowList(t *testing.T) {
var result []*cinfo.ContainerInfo
containers := map[string]*cinfo.ContainerInfo{}
err := json.Unmarshal([]byte(testAllowList), &containers)

if err != nil {
log.Printf("Fail to read request body: %s", err)
}

for _, containerInfo := range containers {
result = append(result, containerInfo)
}

var b bytes.Buffer
enc := json.NewEncoder(&b)
enc.Encode(result)
containerType := TypeContainer
extractor := NewFileSystemMetricExtractor()

var cMetrics []*CAdvisorMetric
if extractor.HasValue(result[0]) {
cMetrics = extractor.GetValue(result[0], containerType)
}
for _, cadvisorMetric := range cMetrics {
log.Printf("cadvisor Metrics received:\n %v \n", *cadvisorMetric)
}
// There are 3 valid device names which pass the allowlist in testAllowList json.
assert.Equal(t, 3, len(cMetrics))
assert.Equal(t, "tmpfs", cMetrics[0].tags["device"])
assert.Equal(t, "/dev/xvda1", cMetrics[1].tags["device"])
assert.Equal(t, "overlay", cMetrics[2].tags["device"])

}

0 comments on commit 6d42494

Please sign in to comment.