From 6f8ba2a6d733a8d03db05b5a338d08db9ca36b5d Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 28 Nov 2023 19:49:02 +0200 Subject: [PATCH 01/20] add hard memory limit --- .../module/awsfargate/task_stats/container.go | 2 ++ .../module/awsfargate/task_stats/task_stats.go | 6 ++++++ .../module/awsfargate/task_stats/task_stats_test.go | 12 +++++++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/container.go b/x-pack/metricbeat/module/awsfargate/task_stats/container.go index 3479fc580e87..f1ecaa276741 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/container.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/container.go @@ -15,6 +15,7 @@ type container struct { Name string Image string Labels map[string]string + Limits Limits `json:"Limits"` } func getContainerMetadata(c *container) *container { @@ -23,6 +24,7 @@ func getContainerMetadata(c *container) *container { Image: c.Image, Name: helpers.ExtractContainerName([]string{c.Name}), Labels: deDotLabels(c.Labels), + Limits: c.Limits, } } diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go index 21eab0f0bc74..847ad10b0c8f 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go @@ -79,9 +79,15 @@ type TaskMetadata struct { Revision string `json:"Revision"` DesiredStatus string `json:"DesiredStatus"` KnownStatus string `json:"KnownStatus"` + Limit Limits `json:"Limits"` Containers []*container `json:"Containers"` } +// Limits is a struct that represents the memory limit from ${ECS_CONTAINER_METADATA_URI_V4}/task +type Limits struct { + Memory uint64 `json:"Memory"` +} + // New creates a new instance of the MetricSet. New is responsible for unpacking // any MetricSet specific configuration options if there are any. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go index 7756a15f1e77..3b7594bfb142 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go @@ -48,6 +48,10 @@ var ( "Revision": "7", "DesiredStatus": "RUNNING", "KnownStatus": "ACTIVATING", + "Limits": { + "CPU": 1, + "Memory": 7168 + }, "Containers": [{ "DockerId": "query-metadata-1", "Name": "query-metadata", @@ -57,7 +61,11 @@ var ( "com.amazonaws.ecs.container-name": "query-metadata", "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/febee046097849aba589d4435207c04a", "com.amazonaws.ecs.task-definition-family": "query-metadata", - "com.amazonaws.ecs.task-definition-version": "7"} + "com.amazonaws.ecs.task-definition-version": "7"}, + "Limits": { + "CPU": 2, + "Memory": 3328 + } }] }` ) @@ -87,11 +95,13 @@ func TestGetTask(t *testing.T) { assert.Equal(t, "RUNNING", taskOutput.DesiredStatus) assert.Equal(t, "ACTIVATING", taskOutput.KnownStatus) + assert.Equal(t, uint64(7168), taskOutput.Limit.Memory) assert.Equal(t, 1, len(taskOutput.Containers)) assert.Equal(t, "query-metadata-1", taskOutput.Containers[0].DockerId) assert.Equal(t, "query-metadata", taskOutput.Containers[0].Name) assert.Equal(t, "mreferre/eksutils", taskOutput.Containers[0].Image) assert.Equal(t, 5, len(taskOutput.Containers[0].Labels)) + assert.Equal(t, uint64(3328), taskOutput.Containers[0].Limits.Memory) } func TestGetStatsList(t *testing.T) { From 6225b60eaea43af64a2029d8d0254bdecc9020a4 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 28 Nov 2023 19:55:28 +0200 Subject: [PATCH 02/20] remove cpu usage percentage --- .../module/awsfargate/task_stats/_meta/data.json | 1 - .../module/awsfargate/task_stats/_meta/fields.yml | 5 ----- .../awsfargate/task_stats/_meta/testdata/task.json | 12 +++++++++--- .../metricbeat/module/awsfargate/task_stats/data.go | 1 - .../module/awsfargate/task_stats/memory.go | 2 -- 5 files changed, 9 insertions(+), 12 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json index 002da6310183..2202251a02d8 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json @@ -108,7 +108,6 @@ }, "usage": { "max": 15294464, - "pct": 0.003136136404770672, "total": 12349440 } }, diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml index b0dddd97691d..0772a3a82fb6 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml @@ -251,11 +251,6 @@ format: bytes description: > Max memory usage. - - name: pct - type: scaled_float - format: percent - description: > - Memory usage percentage. - name: total type: long format: bytes diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json index 37399e027a93..60769755f4d6 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json @@ -5,8 +5,11 @@ "Revision": "7", "DesiredStatus": "RUNNING", "KnownStatus": "ACTIVATING", + "Limits": { + "Memory": 7168 + }, "Containers": [{ - "DockerId": "1234", + "DockerId": "query-metadata-1", "Name": "query-metadata", "Image": "mreferre/eksutils", "Labels": { @@ -14,6 +17,9 @@ "com.amazonaws.ecs.container-name": "query-metadata", "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/febee046097849aba589d4435207c04a", "com.amazonaws.ecs.task-definition-family": "query-metadata", - "com.amazonaws.ecs.task-definition-version": "7"} + "com.amazonaws.ecs.task-definition-version": "7"}, + "Limits": { + "Memory": 3328 + } }] -} +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go index d1a4eb3277e9..881b4a1c041c 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/data.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -163,7 +163,6 @@ func createMemoryFields(stats *Stats) mapstr.M { }, "usage": mapstr.M{ "total": stats.memoryStats.Usage, - "pct": stats.memoryStats.UsageP, "max": stats.memoryStats.MaxUsage, }, } diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/memory.go b/x-pack/metricbeat/module/awsfargate/task_stats/memory.go index 0dfb68d9317a..ec581a2d3c69 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/memory.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/memory.go @@ -13,7 +13,6 @@ type memoryStats struct { TotalRss uint64 TotalRssP float64 Usage uint64 - UsageP float64 //Raw stats from the cgroup subsystem Stats map[string]uint64 //Windows-only memory stats @@ -30,7 +29,6 @@ func getMemoryStats(taskStats types.StatsJSON) memoryStats { MaxUsage: taskStats.Stats.MemoryStats.MaxUsage, TotalRssP: float64(totalRSS) / float64(taskStats.Stats.MemoryStats.Limit), Usage: taskStats.Stats.MemoryStats.Usage, - UsageP: float64(taskStats.Stats.MemoryStats.Usage) / float64(taskStats.Stats.MemoryStats.Limit), Stats: taskStats.Stats.MemoryStats.Stats, //Windows memory statistics Commit: taskStats.Stats.MemoryStats.Commit, From e747537f1d8c4a54b04ea7dae6a2f29fad6ce176 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 28 Nov 2023 19:56:32 +0200 Subject: [PATCH 03/20] replace io/ioutil with io due to deprecation --- .../awsfargate/task_stats/task_stats_test.go | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go index 3b7594bfb142..374df91a9e34 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go @@ -6,7 +6,7 @@ package task_stats import ( "bytes" - "io/ioutil" + "io" "net/http" "testing" @@ -49,9 +49,8 @@ var ( "DesiredStatus": "RUNNING", "KnownStatus": "ACTIVATING", "Limits": { - "CPU": 1, - "Memory": 7168 - }, + "Memory": 7168 + }, "Containers": [{ "DockerId": "query-metadata-1", "Name": "query-metadata", @@ -63,16 +62,15 @@ var ( "com.amazonaws.ecs.task-definition-family": "query-metadata", "com.amazonaws.ecs.task-definition-version": "7"}, "Limits": { - "CPU": 2, "Memory": 3328 } - }] - }` + }] + }` ) func TestGetTaskStats(t *testing.T) { taskStatsResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + Body: io.NopCloser(bytes.NewReader([]byte(taskStatsJson))), } taskStatsOutput, err := getTaskStats(taskStatsResp) @@ -82,7 +80,7 @@ func TestGetTaskStats(t *testing.T) { func TestGetTask(t *testing.T) { taskResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader([]byte(taskRespJson))), + Body: io.NopCloser(bytes.NewReader([]byte(taskRespJson))), } taskOutput, err := getTask(taskResp) @@ -106,14 +104,14 @@ func TestGetTask(t *testing.T) { func TestGetStatsList(t *testing.T) { taskStatsResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + Body: io.NopCloser(bytes.NewReader([]byte(taskStatsJson))), } taskStatsOutput, err := getTaskStats(taskStatsResp) assert.NoError(t, err) taskResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader([]byte(taskRespJson))), + Body: io.NopCloser(bytes.NewReader([]byte(taskRespJson))), } taskOutput, err := getTask(taskResp) @@ -125,7 +123,7 @@ func TestGetStatsList(t *testing.T) { func TestGetCPUStats(t *testing.T) { taskStatsResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + Body: io.NopCloser(bytes.NewReader([]byte(taskStatsJson))), } taskStatsOutput, err := getTaskStats(taskStatsResp) From 34dcac115cdcf0ed7ab74557668a83d99eafd874 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 28 Nov 2023 20:22:20 +0200 Subject: [PATCH 04/20] make update --- metricbeat/docs/fields.asciidoc | 12 ------------ x-pack/metricbeat/module/awsfargate/fields.go | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 32d73808e314..f6dc296d1e8a 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -5501,18 +5501,6 @@ format: bytes -- -*`awsfargate.task_stats.memory.usage.pct`*:: -+ --- -Memory usage percentage. - - -type: scaled_float - -format: percent - --- - *`awsfargate.task_stats.memory.usage.total`*:: + -- diff --git a/x-pack/metricbeat/module/awsfargate/fields.go b/x-pack/metricbeat/module/awsfargate/fields.go index ead4ebc95252..3cc23fae8254 100644 --- a/x-pack/metricbeat/module/awsfargate/fields.go +++ b/x-pack/metricbeat/module/awsfargate/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAwsfargate returns asset data. // This is the base64 encoded zlib format compressed contents of module/awsfargate. func AssetAwsfargate() string { - return "eJzsWk1z2zYQvftX7Hg6kzaJ5UumBx0yoyrOjA92PLbTHOkVuJRRkQADgFGUtv+9A5CUaH6JomyXzohHEth9b/ftggR4AgtajQGXOkA1R0NHAIabkMZwPPlyAx/Tu8dHAD5ppnhsuBRjeH8EAHC3mXcHkfSTkIDJMCRmNNjp2UOIyCjONARKRmBQL+wd9NEgkPBjyYUZHQEoCgk1jWFGBo8AAk6hr8fO1wkIjKgE1V5mFdMY5komcXanOK04lUlhkAtSoxBnFOoRk5GHEf6QApfaI6Y9FibakFpPzc0vaLWUyi/cfxCOs+kNTNOpzlcv1/mQkwcWdkCwttAbg03NCSrRx7tL6+T6srdfnwIuuDV4EmDEw1VvFBtLULK0B6ZvpDSX/UNTAJWZSkdXsNnRnjZodMVXUeUVT+8LDwDuNmbucroazP2mGM09GliSItBMYUx+Wp7FunXAnQ345e+z6Y03/XR5Ozm/PLv2Ls5uJx8mtxPv8/W59+e7f0/t2NN0bG1x51e5yPOrXLUP0pXWlleqjLYkVMJTrE/49YrHMSn5W607F7p9fK2TXjGSu+A+CcMD/qDX7OTjfekhwHRd/xvrgExJrR0aDSj8PJb6LSzvObsH+ppgqMHIQlG4GL2BV6ev4E3hNvdHzfHySXNFvpNcoh+P1e09QWYbUtsQSOWU7EKcitaVro17C8KFkEvxJPic5f7oWJzU4imXewXNdSIMjwimV5/zsh6VxteVVdH3gpSgcBQzUxmSw9AMQ/K9IJRYNyiQKkIzhpgUI1E3YksE7XWVTsY5gQzAkeIiAwc6RkZlYhUKQqroBfAAixND/oN8mK2cUkQSzUjZCTaRTCqqpLHC1nC2KKt4QzWUYt6PgEXgbHePv15pQ4MLvTQYZmzSJNhIZ1C3URmilFr47CupjPVTSioD6lw0A0k0qaHFPYu2hbatEBz8IWqnwmFfxTimz9KCusTd1cb/HvPbdYUmGudb4Q5CJyXMfWSRs/K5XnDZ7z3mA9cLOD/91PclRhGW39ravXcMz4SxJEpCNOQ7Hxr8RHExd5EJeZBW1z1t3pHrEtUAvkhAxnWFBNuKqSMNe12u89ibyAbubGWoN+BcuG1GOpL6w5pwhPbjozbbSTvT2Tn+LFGKhMnyENv2RkyKOgEXlmhS3zgjz7byJ0aa9gS3ZhiZO4bzT6Doa0La6Le2LwsUMsVdn8Mc+BK5eWbUOU7QsY2zRWClwQV8TSih9Ast47UTFze9Pk+PTWTTdlOna1KtPbB5Nd6ymPgUK2K2z43h99G7ft2yh8TXKlG8tgIftYM7Jz9FC+/PZKg93DIyJF5gG89ycejjhz5eQ2T3Pp7qaWCNfLvI1wJPogjV6nnexlH8TF3dfgDJmBTaKT9Vd3dv6HmyXmabLyTm0OoPrb6GyO6t3m0EDaTTVxpqg8pz7BFFstLmO273XLi5fTd7ykfjD53L2V9Uu6mWPvBac14Y40UYx1zMswnHr4/7Rfgal1mssoP09B8Y2+xcnHT2dGSfAheGVICsWnqbPxeiiDdvGu6xvE6dZeO2/qwApIAvXPhyWRfsLmtnk7zhmdegtDybDeV4Y8LFAOBeES6yNHdKxhq+4t/QkLeUamGFq8mM2jtMA59tXDrwyLBAhgU0mU4cAuThiMmkdj+7Y1vcCu4j8hCcE1LNu/Qhb6uzpwtd1hyd+2Z0Sje3wD16wPXNzYNuVQXwEks/Y6RIux9jrBpB8x81RzRFDvVnM9BViNDpjGYHLhcNLHLrrWdO7njnKRTz2Z0bPYZmIvw+AMVc4PecTcMxHgxfI+lhXpsuYOjFWhP8HK8gY9eV0esRFzOZVD7DmjW97dWyicTj9fx/atmXPyC4YDKy66az2VzVvpJxXPt50/0MvhuizNMGWYxsQXXFnmMjpaTa7/eANmipeftO0R1SNuAZwtWOqSpkmZiDkg9KfhFK/i8AAP//k1OOZw==" + return "eJzsWk1z2zYQvftX7Hg6kzaJ5UumBx0yoyrOjA92PLbTHOkVuJRRkQADgFGUtv+9AxCUaH5IFGW7ckY6isDue7tvHylCJzCjxRBwriNUUzR0BGC4iWkIx6MvN/Ax//b4CCAkzRRPDZdiCO+PAADuVvvuIJFhFhMwGcfEjAa73V+EhIziTEOkZAIG9cx+gyEaBBJhKrkwgyMARTGhpiFMyOARQMQpDvXQ5ToBgQlVoNqPWaQ0hKmSWeq/KW8rb2VSGOSC1CDGCcV6wGQSYII/pMC5DojpgMWZNqSWW4vwM1rMpQpL3z8ox9n4Bsb5VperV+piycmDCFsgWEbojcG25gSV6JPdtXV0fdk7b0gRF9wGPIkw4fGiN4pVJKhE2gHTN1Kay/6lKYHyofLVNWx2daANGl3LVVZ5LdP70gWAu1WYu4KuBnO/GkZzjwbmpAg0U5hSmI9neW4dcBcDfvn7bHwTjD9d3o7OL8+ug4uz29GH0e0o+Hx9Hvz57t9Tu/Y0X9s43MWnOuTFpzq1D9qVz1ZQmYx1TaiVpzyf8OsVT1NS8rfGdK50u+RaNr0WpEjBQxKGR/yB12yV433lIsB4Of+r6IBMSa0dGg0owqKW+i3M7zm7B/qaYazByNJQuBq9gVenr+BN6WseDtrrFZLmikInuUw/HqvbewIfG/LYEEnllOxKnIvWja6t+xqEMyHn4knwucj90bE0a8RTHfcamutMGJ4QjK8+F2M9qKxvGqty7hkpQfEgZaa2pIChGcYUBlEssWlRJFWCZggpKUaiacWGCtrPVb4ZpwQyAkeKCw8OdIqMqsRqFIRUyQvgARYnxvwHhTBZOKWILJmQshtsI5lUVGtjja3hbFZV8YpqLMW0HwGLwMXuXn+90Ib2rvTSYOzZ5E2wlfZQN1HZRymt4bOrpDzrp5SUB+pStAPJNKl9q7uvtoW2aRAc/H3UTo3DropxTJ/FgrrU3c3G/17z2+WEZhqnG+HuhU4qmPvIomAVcj3jst9zzAeuZ3B++qnvQ4wirD61rc/esTwjxrIki9FQ6HJoCDPFxdRVJuZRPl33tHpGbmpUC/gyAZk2DRJsGqaONOznctnH3kRWcCcLQ70BF8JdF6QjqT9sCEdoNz5q9Tppazpb159lSpEwvg+ptTdiUjQJuHSLJvWNMwqslT8x0twT3D3DyCIxnH8CRV8z0ka/tb4sUMgcd3MPC+Bz5OaZURc4Qae2zhaBlQYX8DWjjPJfaJ7XVlzc9uY+PTaRle3mSZek1npg+914w80kpFQRsz43hN8H7/q5ZQ+JL1WieOMEPqqDuyQ/hYX3Z7KvHm4ZGRIv0MZ9Lw4+fvDxBiLb+3iupz0z8s0iXwo8SxJUi+d5GkfxM7m6/QEkU1Jot/xU7u6e0ItmvUybLzXmYPUHq28gsr3VuxdBe+L0NUNtUXmBPaFE1my+4+ueC7e378ue6tH4w+Ry8hc1vlTLLwRre15aEySYplxM/Ybj18f9KnyNc18rf5Ce/wfGmp2rk/ZXB/YqcGFIRcjqo7f650KS8PaXhjvcXscusnGv/qwApIAvXIRy3lTsLvfONnnDM9+D8vFsD1TgTQlnewD3inDm29ypGUv4in9DQ8FcqpkVriYzWO8wLXw2cenAw2MBjwU0mU4cIuTxgMms8X12R1vcCO4j8hhcElLtb+ljvm7Onq503hxd+nZ0Srdb4A4ecH1z88Ct6gBe4uh7Roq0+2OMVSNo/qPhiKbMoflsBroKETqd0WzB5aKFRRF97ZmTO955CsV8dudGj6GZBL/vgWIu8HvBpuUYD/Zd5w24C7yCjLXkwesBFxOZ1X7BtMth01NZG4nHs8t/GtlXn725YDKxtxwXs30gQiXTtPGXQffj626IfKYVshTZjJrmpMBGSkm128n6Omh5eHs77g7JL3iGcq3HVBeyzMxByQclvwgl/xcAAP//8Plhvw==" } From ad0a9e80ddd6235ae3b6fd2875885c6c67cc5845 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Thu, 30 Nov 2023 06:36:21 +0200 Subject: [PATCH 05/20] fix integration test --- .../module/awsfargate/task_stats/_meta/testdata/task.json | 2 +- .../module/awsfargate/task_stats/task_stats_integration_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json index 60769755f4d6..d06b46e0df09 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json @@ -9,7 +9,7 @@ "Memory": 7168 }, "Containers": [{ - "DockerId": "query-metadata-1", + "DockerId": "1234", "Name": "query-metadata", "Image": "mreferre/eksutils", "Labels": { diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go index f5c32de5deec..29b347aa6617 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go @@ -42,7 +42,7 @@ func TestFetch(t *testing.T) { assert.NoError(t, err) formattedStats := getStatsList(taskStatsOutput, taskOutput) - assert.Equal(t, 1, len(formattedStats)) + assert.Equal(t, 0, len(formattedStats)) event := createEvent(&formattedStats[0]) // Build a metricset to test the event From 4082276e6735870c856f6e1fe356423cc91500ab Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Thu, 30 Nov 2023 10:21:01 +0200 Subject: [PATCH 06/20] defer body close --- x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go | 4 ++++ .../awsfargate/task_stats/task_stats_integration_test.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go index 847ad10b0c8f..ea97bac5a525 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go @@ -137,6 +137,8 @@ func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { if err != nil { return nil, fmt.Errorf("http.Get failed: %w", err) } + defer taskStatsResp.Body.Close() + taskStatsOutput, err := getTaskStats(taskStatsResp) if err != nil { return nil, fmt.Errorf("getTaskStats failed: %w", err) @@ -151,6 +153,8 @@ func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { if err != nil { return nil, fmt.Errorf("http.Get failed: %w", err) } + defer taskResp.Body.Close() + taskOutput, err := getTask(taskResp) if err != nil { return nil, fmt.Errorf("getTask failed: %w", err) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go index 29b347aa6617..13925cfbbc22 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go @@ -31,9 +31,11 @@ func TestFetch(t *testing.T) { taskStatsResp, err := buildResponse("./_meta/testdata/task_stats.json") assert.NoError(t, err) + defer taskStatsResp.Body.Close() byteTaskResp, err := buildResponse("./_meta/testdata/task.json") assert.NoError(t, err) + defer byteTaskResp.Body.Close() taskStatsOutput, err := getTaskStats(taskStatsResp) assert.NoError(t, err) @@ -69,9 +71,11 @@ func TestData(t *testing.T) { taskStatsResp, err := buildResponse("./_meta/testdata/task_stats.json") assert.NoError(t, err) + defer taskStatsResp.Body.Close() byteTaskResp, err := buildResponse("./_meta/testdata/task.json") assert.NoError(t, err) + defer byteTaskResp.Body.Close() taskStatsOutput, err := getTaskStats(taskStatsResp) assert.NoError(t, err) From 69eabe12287aa33d545c9cc61965438556612fe9 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Thu, 30 Nov 2023 16:29:22 +0200 Subject: [PATCH 07/20] fix integration test --- .../module/awsfargate/task_stats/task_stats_integration_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go index 13925cfbbc22..11101a9899a6 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go @@ -44,7 +44,7 @@ func TestFetch(t *testing.T) { assert.NoError(t, err) formattedStats := getStatsList(taskStatsOutput, taskOutput) - assert.Equal(t, 0, len(formattedStats)) + assert.Equal(t, 1, len(formattedStats)) event := createEvent(&formattedStats[0]) // Build a metricset to test the event From 325497d37d323bd8998158f4ce69494860ba458c Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 17:29:58 +0200 Subject: [PATCH 08/20] remove redundant limits json --- x-pack/metricbeat/module/awsfargate/task_stats/container.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/container.go b/x-pack/metricbeat/module/awsfargate/task_stats/container.go index f1ecaa276741..9c4d7d0d0e61 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/container.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/container.go @@ -15,7 +15,7 @@ type container struct { Name string Image string Labels map[string]string - Limits Limits `json:"Limits"` + Limits Limits } func getContainerMetadata(c *container) *container { From 5c5a5e29990a095915dc80ad42b99109da714949 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 17:36:51 +0200 Subject: [PATCH 09/20] explicitly state limits is the hard memory limit --- x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go index ea97bac5a525..39ed9ba85bb9 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go @@ -83,7 +83,7 @@ type TaskMetadata struct { Containers []*container `json:"Containers"` } -// Limits is a struct that represents the memory limit from ${ECS_CONTAINER_METADATA_URI_V4}/task +// Limits is a struct that represents the memory limit from ${ECS_CONTAINER_METADATA_URI_V4}/task, which is the Hard Memory Limit set in AWS ECS type Limits struct { Memory uint64 `json:"Memory"` } From e573be48aac6a933f91f7db4335bafb689aa167a Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 17:44:57 +0200 Subject: [PATCH 10/20] use t.Cleanup in favor of defer --- .../task_stats/task_stats_integration_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go index 11101a9899a6..ec5e4ddfab37 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go @@ -31,11 +31,9 @@ func TestFetch(t *testing.T) { taskStatsResp, err := buildResponse("./_meta/testdata/task_stats.json") assert.NoError(t, err) - defer taskStatsResp.Body.Close() byteTaskResp, err := buildResponse("./_meta/testdata/task.json") assert.NoError(t, err) - defer byteTaskResp.Body.Close() taskStatsOutput, err := getTaskStats(taskStatsResp) assert.NoError(t, err) @@ -53,6 +51,11 @@ func TestFetch(t *testing.T) { // The goal here is to make sure every element inside the // event has a matching field ("no field left behind"). mbtest.TestMetricsetFieldsDocumented(t, metricSet, []mb.Event{event}) + + t.Cleanup(func() { + taskStatsResp.Body.Close() + byteTaskResp.Body.Close() + }) } func TestData(t *testing.T) { @@ -71,11 +74,9 @@ func TestData(t *testing.T) { taskStatsResp, err := buildResponse("./_meta/testdata/task_stats.json") assert.NoError(t, err) - defer taskStatsResp.Body.Close() byteTaskResp, err := buildResponse("./_meta/testdata/task.json") assert.NoError(t, err) - defer byteTaskResp.Body.Close() taskStatsOutput, err := getTaskStats(taskStatsResp) assert.NoError(t, err) @@ -89,6 +90,11 @@ func TestData(t *testing.T) { standardizeEvent := m.StandardizeEvent(event) mbtest.WriteEventToDataJSON(t, standardizeEvent, "") + + t.Cleanup(func() { + taskStatsResp.Body.Close() + byteTaskResp.Body.Close() + }) } // buildResponse is a test helper that loads the content of `filename` and returns From e4dab69c138fd0e991ef4a30aa13f46d39c4f3c2 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 19:07:53 +0200 Subject: [PATCH 11/20] add memory root field --- x-pack/metricbeat/module/awsfargate/task_stats/data.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go index 881b4a1c041c..8fc89f1991dc 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/data.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -87,8 +87,9 @@ func createRootFields(stats *Stats, regionName string) mapstr.M { "image": mapstr.M{ "name": stats.Container.Image, }, - "name": stats.Container.Name, - "labels": stats.Container.Labels, + "name": stats.Container.Name, + "labels": stats.Container.Labels, + "memory_hard_limit": stats.Container.Limits, }, } From e1d2e6269d8c63fc3fbab000b8ab4cb4392927e2 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 19:07:53 +0200 Subject: [PATCH 12/20] add memory root field --- x-pack/metricbeat/module/awsfargate/task_stats/data.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go index 881b4a1c041c..96b2e8400838 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/data.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -87,8 +87,9 @@ func createRootFields(stats *Stats, regionName string) mapstr.M { "image": mapstr.M{ "name": stats.Container.Image, }, - "name": stats.Container.Name, - "labels": stats.Container.Labels, + "name": stats.Container.Name, + "labels": stats.Container.Labels, + "memory_hard_limit": stats.Container.Limits.Memory, }, } From 5f200544815da03033250310ef3be492981ed7d7 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 20:21:19 +0200 Subject: [PATCH 13/20] add changelog entry --- CHANGELOG.next.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c5fc424fe6ee..86f9f4d505f4 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -124,6 +124,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix memory leak on Windows {issue}37142[37142] {pull}37171[37171] - Fix unintended skip in metric collection on Azure Monitor {issue}37204[37204] {pull}37203[37203] - Fix the "api-version query parameter (?api-version=) is required for all requests" error in Azure Billing. {pull}37158[37158] +- Add memory hard limit from container metadata and remove usage percentage in AWS Fargate. {pull}37194[37194] *Osquerybeat* From ab9f5aaf0ccbb9db7b52b06422d4aa66f04ada95 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 21:25:24 +0200 Subject: [PATCH 14/20] make update --- metricbeat/docs/fields.asciidoc | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 80f0086ed2e6..1e5124fb8f80 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4986,6 +4986,18 @@ type: keyword -- +*`awsfargate.task_stats.memory_hard_limit`*:: ++ +-- +The Hard Memory Limit for the task from Amazon ECS. + + +type: scaled_float + +format: bytes + +-- + [float] === cpu From db2cf8a6012c5ea862d62c222a22a2c256c90e2c Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 21:44:30 +0200 Subject: [PATCH 15/20] make update --- metricbeat/docs/fields.asciidoc | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 1e5124fb8f80..80f0086ed2e6 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4986,18 +4986,6 @@ type: keyword -- -*`awsfargate.task_stats.memory_hard_limit`*:: -+ --- -The Hard Memory Limit for the task from Amazon ECS. - - -type: scaled_float - -format: bytes - --- - [float] === cpu From 005ca10b48ce619bf14449067084af22d54b27ce Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 21:51:36 +0200 Subject: [PATCH 16/20] document hard_memory_limit field --- .../module/awsfargate/task_stats/_meta/fields.yml | 4 ++++ x-pack/metricbeat/module/awsfargate/task_stats/data.go | 10 +++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml index 0772a3a82fb6..dd440c9a6c99 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml @@ -22,6 +22,10 @@ type: keyword description: > The known status for the task from Amazon ECS. + - name: hard_memory_limit + type: scaled_float + description: > + The Hard Memory Limit for the task from Amazon ECS. - name: cpu type: group description: Runtime CPU metrics. diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go index 96b2e8400838..45581855a36f 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/data.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -57,6 +57,11 @@ func createEvent(stats *Stats) mb.Event { _, _ = e.MetricSetFields.Put("task_known_status", taskKnownStatus) } + memoryHardLimit := stats.Container.Limits.Memory + if memoryHardLimit != 0 { + _, _ = e.MetricSetFields.Put("memory_hard_limit", taskKnownStatus) + } + _, _ = e.MetricSetFields.Put("identifier", generateIdentifier(stats.Container.Name, stats.Container.DockerId)) return e } @@ -87,9 +92,8 @@ func createRootFields(stats *Stats, regionName string) mapstr.M { "image": mapstr.M{ "name": stats.Container.Image, }, - "name": stats.Container.Name, - "labels": stats.Container.Labels, - "memory_hard_limit": stats.Container.Limits.Memory, + "name": stats.Container.Name, + "labels": stats.Container.Labels, }, } From 0dd595fcf772cc425f9621668fa683b3b82f5794 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 21:54:56 +0200 Subject: [PATCH 17/20] make update --- metricbeat/docs/fields.asciidoc | 10 ++++++++++ x-pack/metricbeat/module/awsfargate/fields.go | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 80f0086ed2e6..097ebc20a162 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4986,6 +4986,16 @@ type: keyword -- +*`awsfargate.task_stats.hard_memory_limit`*:: ++ +-- +The Hard Memory Limit for the task from Amazon ECS. + + +type: scaled_float + +-- + [float] === cpu diff --git a/x-pack/metricbeat/module/awsfargate/fields.go b/x-pack/metricbeat/module/awsfargate/fields.go index 3cc23fae8254..79acef65e798 100644 --- a/x-pack/metricbeat/module/awsfargate/fields.go +++ b/x-pack/metricbeat/module/awsfargate/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAwsfargate returns asset data. // This is the base64 encoded zlib format compressed contents of module/awsfargate. func AssetAwsfargate() string { - return "eJzsWk1z2zYQvftX7Hg6kzaJ5UumBx0yoyrOjA92PLbTHOkVuJRRkQADgFGUtv+9AxCUaH5IFGW7ckY6isDue7tvHylCJzCjxRBwriNUUzR0BGC4iWkIx6MvN/Ax//b4CCAkzRRPDZdiCO+PAADuVvvuIJFhFhMwGcfEjAa73V+EhIziTEOkZAIG9cx+gyEaBBJhKrkwgyMARTGhpiFMyOARQMQpDvXQ5ToBgQlVoNqPWaQ0hKmSWeq/KW8rb2VSGOSC1CDGCcV6wGQSYII/pMC5DojpgMWZNqSWW4vwM1rMpQpL3z8ox9n4Bsb5VperV+piycmDCFsgWEbojcG25gSV6JPdtXV0fdk7b0gRF9wGPIkw4fGiN4pVJKhE2gHTN1Kay/6lKYHyofLVNWx2daANGl3LVVZ5LdP70gWAu1WYu4KuBnO/GkZzjwbmpAg0U5hSmI9neW4dcBcDfvn7bHwTjD9d3o7OL8+ug4uz29GH0e0o+Hx9Hvz57t9Tu/Y0X9s43MWnOuTFpzq1D9qVz1ZQmYx1TaiVpzyf8OsVT1NS8rfGdK50u+RaNr0WpEjBQxKGR/yB12yV433lIsB4Of+r6IBMSa0dGg0owqKW+i3M7zm7B/qaYazByNJQuBq9gVenr+BN6WseDtrrFZLmikInuUw/HqvbewIfG/LYEEnllOxKnIvWja6t+xqEMyHn4knwucj90bE0a8RTHfcamutMGJ4QjK8+F2M9qKxvGqty7hkpQfEgZaa2pIChGcYUBlEssWlRJFWCZggpKUaiacWGCtrPVb4ZpwQyAkeKCw8OdIqMqsRqFIRUyQvgARYnxvwHhTBZOKWILJmQshtsI5lUVGtjja3hbFZV8YpqLMW0HwGLwMXuXn+90Ib2rvTSYOzZ5E2wlfZQN1HZRymt4bOrpDzrp5SUB+pStAPJNKl9q7uvtoW2aRAc/H3UTo3DropxTJ/FgrrU3c3G/17z2+WEZhqnG+HuhU4qmPvIomAVcj3jst9zzAeuZ3B++qnvQ4wirD61rc/esTwjxrIki9FQ6HJoCDPFxdRVJuZRPl33tHpGbmpUC/gyAZk2DRJsGqaONOznctnH3kRWcCcLQ70BF8JdF6QjqT9sCEdoNz5q9Tppazpb159lSpEwvg+ptTdiUjQJuHSLJvWNMwqslT8x0twT3D3DyCIxnH8CRV8z0ka/tb4sUMgcd3MPC+Bz5OaZURc4Qae2zhaBlQYX8DWjjPJfaJ7XVlzc9uY+PTaRle3mSZek1npg+914w80kpFQRsz43hN8H7/q5ZQ+JL1WieOMEPqqDuyQ/hYX3Z7KvHm4ZGRIv0MZ9Lw4+fvDxBiLb+3iupz0z8s0iXwo8SxJUi+d5GkfxM7m6/QEkU1Jot/xU7u6e0ItmvUybLzXmYPUHq28gsr3VuxdBe+L0NUNtUXmBPaFE1my+4+ueC7e378ue6tH4w+Ry8hc1vlTLLwRre15aEySYplxM/Ybj18f9KnyNc18rf5Ce/wfGmp2rk/ZXB/YqcGFIRcjqo7f650KS8PaXhjvcXscusnGv/qwApIAvXIRy3lTsLvfONnnDM9+D8vFsD1TgTQlnewD3inDm29ypGUv4in9DQ8FcqpkVriYzWO8wLXw2cenAw2MBjwU0mU4cIuTxgMms8X12R1vcCO4j8hhcElLtb+ljvm7Onq503hxd+nZ0Srdb4A4ecH1z88Ct6gBe4uh7Roq0+2OMVSNo/qPhiKbMoflsBroKETqd0WzB5aKFRRF97ZmTO955CsV8dudGj6GZBL/vgWIu8HvBpuUYD/Zd5w24C7yCjLXkwesBFxOZ1X7BtMth01NZG4nHs8t/GtlXn725YDKxtxwXs30gQiXTtPGXQffj626IfKYVshTZjJrmpMBGSkm128n6Omh5eHs77g7JL3iGcq3HVBeyzMxByQclvwgl/xcAAP//8Plhvw==" + return "eJzsWt9v2zYQfs9fcQgGdGsb56XYgx8KeG6KBVjSIEnXR+VMnW3OEqmSVF132/8+kKJsRT9sWU4yJ7AfLfLu++6++ySLPoEZLfqAcz1GNUFDRwCGm4j6cDz4cgMfs2+PjwBC0kzxxHAp+vD+CADgbrXvDmIZphEBk1FEzGiw2/1FiMkozjSMlYzBoJ7ZbzBEg0AiTCQXpncEoCgi1NSHERk8AhhzikLdd7lOQGBMJaj2YxYJ9WGiZJr4b4rbiluZFAa5INWLcESR7jEZBxjjDylwrgNiOmBRqg2p5dY8/IwWc6nCwvf3ynE2vIFhttXl6pQ6X3JyL8IWCJYROmOwrTlBJbpkd20dXF92zhvSmAtuA56MMebRojOKVSQoRdoB0zdSmsvupSmA8qGy1RVsdnWgDRpdyVVUeSXT+8IFgLtVmLucrgYzXQ2jmaKBOSkCzRQmFGbjWZxbB9zFgJ/+PhveBMNPl7eD88uz6+Di7HbwYXA7CD5fnwd/vvv31K49zdbWDnf+KQ95/ilP7b12ZbMVlCZjXRMq5SnOJ/x8xZOElPylNp0r3S65lk2vBMlT8JCE4WN+z2u2yvG+dBFguJz/VXRApqTWDo0GFGFeS/0W5lPOpkBfU4w0GFkYClejN/Dq9BW8KXzNw15zvULSXFHoJJfqh2N1OyXwsSGLDWOpnJJdiTPRutG1dV+DcCbkXDwKPhe5O7opqjCIKZZqEUQ85qYWnWYYURiMI4nlBS0g/o4qhAuXA/6wOTrgZElai6xsSxVI16kwPCYYXn3O7adXWl83/sXcM1KCol7CytxbFchlkCpG04eEFCNRt2JDGe3nKtuMEwI5BkeKCw8OdIKMysQqFIRU8TPgARYnRvwHhTBaOKWINB6RshtsI5lUVGljha3hbFaethXVSIpJNwIWgYvdvv56oQ3tXemlwcizyZpgK+2hbqKyj1Jaw2dXSXnWjykpD9SlaAaSalL7VndfbQtt0yA4+PuonQqHXRXjmD6JBbWpu5uN/73mt8sJTTVONsLdC52UMHeRRc4q5HrGZbfnmA9cz+D89FPXhxhFWH66XJ+9ZXkGjKVxGqGh0OXQEKaKi4mrTMTH2XRNafUsX9eoBvBFAjKpGyTYNEwtadjP5bKPnYms4I4WhjoDzoW7LkhLUr/ZEI7QbnzU6rXX1nS2rj9LlSJhfB8Sa2/EpKgTcOEWTeobZxRYK39kpJknuHuGkXliOP8Eir6mpI1+a31ZoJAZ7voe5sDnyM0To85xgk5snS0CKw0u4GtKKWW/JD2vrbi47fV9emgiK9vNki5JrfXA5rvxhptJSIkiZn2uD7/23nVzyw4SX6pE8doJfFAHd0lehIV3Z7KvHm4ZGRLP0MZ9Lw4+fvDxGiLb+3impz0z8s0iXwo8jWNUi6d5Gkfxklzd/gCSCSm0W16Uu7sn9LxZz9PmC405WP3B6muIbG/17kXQnjh9xVAbVJ5jz47Vur3u8cdlHV/2lI/w7yeXo7+o9qVadiFY2/PCmiDGJOFi4jccvz7uVuFrnPta+QP/7L861uxcnbS/2rNXgQtDaoysOnqrf1jE1YNMeIjb69BFNu7VnxWAFPCFi1DO64rd5t7ZJG944ntQNp7NgXK8CeFsD+BeEc58m1s1Ywlf8W9oKJhLNbPC1WR66x2mgc8mLi14eCzgsYAm04rDGHnUYzKtfZ/d0hY3gvuIPAKXhFTzW/q6PwzAE5TOm6NL34xO6WYL3MEDrm9u7rlVFcBzHH3PSJF2f+CxagTNf9Qc0RQ51J/NQFshQqszmi24XDSwyKOvPXNyxzuPoZjP7tzoITQT4/c9UMwFfs/ZNBzjwb7rvAZ3jleQsZbce93jYiTTyi+YZjlseiprIvFwdvlPLfvyszcXTMb2luNiNg9EqGSS1P4yaH983Q6Rz7RCliCbUd2c5NhIKal2O1lfBy0Lb2/H7SH5BU9QrvWYqkKWqTko+aDkZ6Hk/wIAAP//mzKQEw==" } From 6e13185246735fa32a6238eec65b9befbf07f0c0 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 22:08:08 +0200 Subject: [PATCH 18/20] fix memory field --- x-pack/metricbeat/module/awsfargate/task_stats/data.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go index 45581855a36f..0db9f40135d0 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/data.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -59,7 +59,7 @@ func createEvent(stats *Stats) mb.Event { memoryHardLimit := stats.Container.Limits.Memory if memoryHardLimit != 0 { - _, _ = e.MetricSetFields.Put("memory_hard_limit", taskKnownStatus) + _, _ = e.MetricSetFields.Put("memory_hard_limit", memoryHardLimit) } _, _ = e.MetricSetFields.Put("identifier", generateIdentifier(stats.Container.Name, stats.Container.DockerId)) From 51d6b155bbc4e5eb15629c352c7cbb5c1f82d4a1 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 22:38:34 +0200 Subject: [PATCH 19/20] fix field name --- x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml index dd440c9a6c99..b79be57b70c8 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml @@ -22,7 +22,7 @@ type: keyword description: > The known status for the task from Amazon ECS. - - name: hard_memory_limit + - name: memory_hard_limit type: scaled_float description: > The Hard Memory Limit for the task from Amazon ECS. From a72e8e72cfd25679f3cb14ffca8e3f71d14de3c8 Mon Sep 17 00:00:00 2001 From: lucian-ioan Date: Tue, 5 Dec 2023 22:44:14 +0200 Subject: [PATCH 20/20] make update --- metricbeat/docs/fields.asciidoc | 2 +- x-pack/metricbeat/module/awsfargate/fields.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 097ebc20a162..f4fbe02155c3 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4986,7 +4986,7 @@ type: keyword -- -*`awsfargate.task_stats.hard_memory_limit`*:: +*`awsfargate.task_stats.memory_hard_limit`*:: + -- The Hard Memory Limit for the task from Amazon ECS. diff --git a/x-pack/metricbeat/module/awsfargate/fields.go b/x-pack/metricbeat/module/awsfargate/fields.go index 79acef65e798..e7a68db138f6 100644 --- a/x-pack/metricbeat/module/awsfargate/fields.go +++ b/x-pack/metricbeat/module/awsfargate/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAwsfargate returns asset data. // This is the base64 encoded zlib format compressed contents of module/awsfargate. func AssetAwsfargate() string { - return "eJzsWt9v2zYQfs9fcQgGdGsb56XYgx8KeG6KBVjSIEnXR+VMnW3OEqmSVF132/8+kKJsRT9sWU4yJ7AfLfLu++6++ySLPoEZLfqAcz1GNUFDRwCGm4j6cDz4cgMfs2+PjwBC0kzxxHAp+vD+CADgbrXvDmIZphEBk1FEzGiw2/1FiMkozjSMlYzBoJ7ZbzBEg0AiTCQXpncEoCgi1NSHERk8AhhzikLdd7lOQGBMJaj2YxYJ9WGiZJr4b4rbiluZFAa5INWLcESR7jEZBxjjDylwrgNiOmBRqg2p5dY8/IwWc6nCwvf3ynE2vIFhttXl6pQ6X3JyL8IWCJYROmOwrTlBJbpkd20dXF92zhvSmAtuA56MMebRojOKVSQoRdoB0zdSmsvupSmA8qGy1RVsdnWgDRpdyVVUeSXT+8IFgLtVmLucrgYzXQ2jmaKBOSkCzRQmFGbjWZxbB9zFgJ/+PhveBMNPl7eD88uz6+Di7HbwYXA7CD5fnwd/vvv31K49zdbWDnf+KQ95/ilP7b12ZbMVlCZjXRMq5SnOJ/x8xZOElPylNp0r3S65lk2vBMlT8JCE4WN+z2u2yvG+dBFguJz/VXRApqTWDo0GFGFeS/0W5lPOpkBfU4w0GFkYClejN/Dq9BW8KXzNw15zvULSXFHoJJfqh2N1OyXwsSGLDWOpnJJdiTPRutG1dV+DcCbkXDwKPhe5O7opqjCIKZZqEUQ85qYWnWYYURiMI4nlBS0g/o4qhAuXA/6wOTrgZElai6xsSxVI16kwPCYYXn3O7adXWl83/sXcM1KCol7CytxbFchlkCpG04eEFCNRt2JDGe3nKtuMEwI5BkeKCw8OdIKMysQqFIRU8TPgARYnRvwHhTBaOKWINB6RshtsI5lUVGljha3hbFaethXVSIpJNwIWgYvdvv56oQ3tXemlwcizyZpgK+2hbqKyj1Jaw2dXSXnWjykpD9SlaAaSalL7VndfbQtt0yA4+PuonQqHXRXjmD6JBbWpu5uN/73mt8sJTTVONsLdC52UMHeRRc4q5HrGZbfnmA9cz+D89FPXhxhFWH66XJ+9ZXkGjKVxGqGh0OXQEKaKi4mrTMTH2XRNafUsX9eoBvBFAjKpGyTYNEwtadjP5bKPnYms4I4WhjoDzoW7LkhLUr/ZEI7QbnzU6rXX1nS2rj9LlSJhfB8Sa2/EpKgTcOEWTeobZxRYK39kpJknuHuGkXliOP8Eir6mpI1+a31ZoJAZ7voe5sDnyM0To85xgk5snS0CKw0u4GtKKWW/JD2vrbi47fV9emgiK9vNki5JrfXA5rvxhptJSIkiZn2uD7/23nVzyw4SX6pE8doJfFAHd0lehIV3Z7KvHm4ZGRLP0MZ9Lw4+fvDxGiLb+3impz0z8s0iXwo8jWNUi6d5Gkfxklzd/gCSCSm0W16Uu7sn9LxZz9PmC405WP3B6muIbG/17kXQnjh9xVAbVJ5jz47Vur3u8cdlHV/2lI/w7yeXo7+o9qVadiFY2/PCmiDGJOFi4jccvz7uVuFrnPta+QP/7L861uxcnbS/2rNXgQtDaoysOnqrf1jE1YNMeIjb69BFNu7VnxWAFPCFi1DO64rd5t7ZJG944ntQNp7NgXK8CeFsD+BeEc58m1s1Ywlf8W9oKJhLNbPC1WR66x2mgc8mLi14eCzgsYAm04rDGHnUYzKtfZ/d0hY3gvuIPAKXhFTzW/q6PwzAE5TOm6NL34xO6WYL3MEDrm9u7rlVFcBzHH3PSJF2f+CxagTNf9Qc0RQ51J/NQFshQqszmi24XDSwyKOvPXNyxzuPoZjP7tzoITQT4/c9UMwFfs/ZNBzjwb7rvAZ3jleQsZbce93jYiTTyi+YZjlseiprIvFwdvlPLfvyszcXTMb2luNiNg9EqGSS1P4yaH983Q6Rz7RCliCbUd2c5NhIKal2O1lfBy0Lb2/H7SH5BU9QrvWYqkKWqTko+aDkZ6Hk/wIAAP//mzKQEw==" + return "eJzsWt9v2zYQfs9fcQgGdGsb56XYgx8KeG6KBVjSIEnXR+VMnW3OEqmSVF132/8+kKJsWT9sWU4yJ4gfLfLu++6++ySLPoEZLfqAcz1GNUFDRwCGm4j6cDz4cgMfs2+PjwBC0kzxxHAp+vD+CADgbrXvDmIZphEBk1FEzGiw2/1FiMkozjSMlYzBoJ7ZbzBEg0AiTCQXpncEoCgi1NSHERk8AhhzikLdd7lOQGBMJaj2YxYJ9WGiZJr4b4rbiluZFAa5INWLcESR7jEZBxjjDylwrgNiOmBRqg2p5dY8/IwWc6nCwvdr5Tgb3sAw2+pydUqdLzlZi7ADgmWEzhhsa05QiS7ZXVsH15ed84Y05oLbgCdjjHm06IxiFQlKkfbA9I2U5rJ7aQqgfKhsdQWbXR1og0ZXchVVXsn0vnAB4G4V5i6nq8FMV8NopmhgTopAM4UJhdl4FufWAXcx4Ke/z4Y3wfDT5e3g/PLsOrg4ux18GNwOgs/X58Gf7/49tWtPs7W1w51/ykOef8pTu9aubLaC0mRsakKlPMX5hJ+veJKQkr/UpnOl2yfXsumVIHkKHpIwfMzXvGanHO9LFwGGy/lfRQdkSmrt0GhAEea11G9hPuVsCvQ1xUiDkYWhcDV6A69OX8Gbwtc87DXXKyTNFYVOcqm+P1a3UwIfG7LYMJbKKdmVOBOtG11b9w0IZ0LOxYPgc5G7o4splmoRTFGFQcRjbmrRaYYRhcE4klhe0ALi76hCuHCJ4A+bowNOlqS1yMq2VIF0nQrDY4Lh1efcfnql9XXjX8w9IyUo6iWszL1VgVwGqWI0fUhIMRJ1K7aU0X6uss04IZBjcKS48OBAJ8ioTKxCQUgVPwEeYHFixH9QCKOFU4pI4xEpu8E2kklFlTZW2BrOZuVpW1GNpJh0I2ARuNjt668X2tDBlV4ajDybrAm20h7qNiqHKKUNfPaVlGf9kJLyQF2KZiCpJnVodffVttC2DYKDf4jaqXDYVzGO6aNYUJu6u9n432t+u5zQVONkK9yD0EkJcxdZ5KxCrmdcdnuO+cD1DM5PP3V9iFGE5afLzdlblmfAWBqnERoKXQ4NYaq4mLjKRHycTdeUVs/ydY1qAF8kIJO6QYJtw9SShv1cLvvYmcgK7mhhqDPgXLibgrQk9ZsN4Qjtx0etXnvtTGfn+rNUKRLG9yGx9kZMijoBF27RpL5xRoG18gdGmnmCu2cYmSeG80+g6GtK2ui31pcFCpnhru9hDnyO3Dwy6hwn6MTW2SKw0uACvqaUUvZL0vPaiYvbXt+n+yayst0s6ZLURg9svhtvuZmElChi1uf68GvvXTe37CDxpUoUr53Ae3Vwl+RZWHh3Jofq4ZaRIfEEbdz34sXHX3y8hsjuPp7p6cCMfLvIlwJP4xjV4nGexlE8J1e3P4BkQgrtlmfl7u4JPW/W07T5QmNerP7F6muI7G717kXQgTh9xVAbVL5+ttbtdY8/Luv4sqd8hL+eXI7+otqXatmFYGPPC2uCGJOEi4nfcPz6uFuFr3Hua+UP/LP/6lizc3XS/mrPXgUuDKkxsurorf5hEVcPMuE+bq9DF9m4V39WAFLAFy5COa8rdpt7Z5O84ZHvQdl4NgfK8SaEswOAe0U4821u1YwlfMW/oaFgLtXMCleT6W12mAY+27i04OGxgMcCmkwrDmPkUY/JtPZ9dktb3AruI/IIXBJSzW/p6/4wAI9QOm+OLn0zOqWbLXAPD7i+uVlzqyqApzj6npEi7f7AY9UImv+oOaIpcqg/m4G2QoRWZzQ7cLloYJFH33jm5I53HkIxn9250X1oJsbvB6CYC/yes2k4xoND13kN7hyvIGMtufe6x8VIppVfMM1y2PZU1kTi/uzyn1r25WdvLpiM7S3HxWweiFDJJKn9ZdD++LodIp9phSxBNqO6OcmxkVJS7XeyvglaFt7ejttD8gseoVybMVWFLFPzouQXJT8JJf8XAAD//5wYkBM=" }