From e908c0213cb63b8b193e3fd1aede902cdbb62f39 Mon Sep 17 00:00:00 2001 From: Lucian Ioan <59661554+lucian-ioan@users.noreply.github.com> Date: Wed, 6 Dec 2023 00:02:47 +0200 Subject: [PATCH] [AWS Fargate] Add memory hard limit from container metadata and remove usage percentage (#37194) --- CHANGELOG.next.asciidoc | 1 + metricbeat/docs/fields.asciidoc | 22 +++++++--------- x-pack/metricbeat/module/awsfargate/fields.go | 2 +- .../awsfargate/task_stats/_meta/data.json | 1 - .../awsfargate/task_stats/_meta/fields.yml | 9 +++---- .../task_stats/_meta/testdata/task.json | 10 +++++-- .../module/awsfargate/task_stats/container.go | 2 ++ .../module/awsfargate/task_stats/data.go | 6 ++++- .../module/awsfargate/task_stats/memory.go | 2 -- .../awsfargate/task_stats/task_stats.go | 10 +++++++ .../task_stats/task_stats_integration_test.go | 10 +++++++ .../awsfargate/task_stats/task_stats_test.go | 26 ++++++++++++------- 12 files changed, 68 insertions(+), 33 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index c2d577a3879b..39f80047af3c 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -124,6 +124,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix memory leak on Windows {issue}37142[37142] {pull}37171[37171] - Fix unintended skip in metric collection on Azure Monitor {issue}37204[37204] {pull}37203[37203] - Fix the "api-version query parameter (?api-version=) is required for all requests" error in Azure Billing. {pull}37158[37158] +- Add memory hard limit from container metadata and remove usage percentage in AWS Fargate. {pull}37194[37194] *Osquerybeat* diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 1c29445adbaf..f4fbe02155c3 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -4986,6 +4986,16 @@ type: keyword -- +*`awsfargate.task_stats.memory_hard_limit`*:: ++ +-- +The Hard Memory Limit for the task from Amazon ECS. + + +type: scaled_float + +-- + [float] === cpu @@ -5501,18 +5511,6 @@ format: bytes -- -*`awsfargate.task_stats.memory.usage.pct`*:: -+ --- -Memory usage percentage. - - -type: scaled_float - -format: percent - --- - *`awsfargate.task_stats.memory.usage.total`*:: + -- diff --git a/x-pack/metricbeat/module/awsfargate/fields.go b/x-pack/metricbeat/module/awsfargate/fields.go index ead4ebc95252..e7a68db138f6 100644 --- a/x-pack/metricbeat/module/awsfargate/fields.go +++ b/x-pack/metricbeat/module/awsfargate/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAwsfargate returns asset data. // This is the base64 encoded zlib format compressed contents of module/awsfargate. func AssetAwsfargate() string { - return "eJzsWk1z2zYQvftX7Hg6kzaJ5UumBx0yoyrOjA92PLbTHOkVuJRRkQADgFGUtv+9A5CUaH6JomyXzohHEth9b/ftggR4AgtajQGXOkA1R0NHAIabkMZwPPlyAx/Tu8dHAD5ppnhsuBRjeH8EAHC3mXcHkfSTkIDJMCRmNNjp2UOIyCjONARKRmBQL+wd9NEgkPBjyYUZHQEoCgk1jWFGBo8AAk6hr8fO1wkIjKgE1V5mFdMY5komcXanOK04lUlhkAtSoxBnFOoRk5GHEf6QApfaI6Y9FibakFpPzc0vaLWUyi/cfxCOs+kNTNOpzlcv1/mQkwcWdkCwttAbg03NCSrRx7tL6+T6srdfnwIuuDV4EmDEw1VvFBtLULK0B6ZvpDSX/UNTAJWZSkdXsNnRnjZodMVXUeUVT+8LDwDuNmbucroazP2mGM09GliSItBMYUx+Wp7FunXAnQ345e+z6Y03/XR5Ozm/PLv2Ls5uJx8mtxPv8/W59+e7f0/t2NN0bG1x51e5yPOrXLUP0pXWlleqjLYkVMJTrE/49YrHMSn5W607F7p9fK2TXjGSu+A+CcMD/qDX7OTjfekhwHRd/xvrgExJrR0aDSj8PJb6LSzvObsH+ppgqMHIQlG4GL2BV6ev4E3hNvdHzfHySXNFvpNcoh+P1e09QWYbUtsQSOWU7EKcitaVro17C8KFkEvxJPic5f7oWJzU4imXewXNdSIMjwimV5/zsh6VxteVVdH3gpSgcBQzUxmSw9AMQ/K9IJRYNyiQKkIzhpgUI1E3YksE7XWVTsY5gQzAkeIiAwc6RkZlYhUKQqroBfAAixND/oN8mK2cUkQSzUjZCTaRTCqqpLHC1nC2KKt4QzWUYt6PgEXgbHePv15pQ4MLvTQYZmzSJNhIZ1C3URmilFr47CupjPVTSioD6lw0A0k0qaHFPYu2hbatEBz8IWqnwmFfxTimz9KCusTd1cb/HvPbdYUmGudb4Q5CJyXMfWSRs/K5XnDZ7z3mA9cLOD/91PclRhGW39ravXcMz4SxJEpCNOQ7Hxr8RHExd5EJeZBW1z1t3pHrEtUAvkhAxnWFBNuKqSMNe12u89ibyAbubGWoN+BcuG1GOpL6w5pwhPbjozbbSTvT2Tn+LFGKhMnyENv2RkyKOgEXlmhS3zgjz7byJ0aa9gS3ZhiZO4bzT6Doa0La6Le2LwsUMsVdn8Mc+BK5eWbUOU7QsY2zRWClwQV8TSih9Ast47UTFze9Pk+PTWTTdlOna1KtPbB5Nd6ymPgUK2K2z43h99G7ft2yh8TXKlG8tgIftYM7Jz9FC+/PZKg93DIyJF5gG89ycejjhz5eQ2T3Pp7qaWCNfLvI1wJPogjV6nnexlH8TF3dfgDJmBTaKT9Vd3dv6HmyXmabLyTm0OoPrb6GyO6t3m0EDaTTVxpqg8pz7BFFstLmO273XLi5fTd7ykfjD53L2V9Uu6mWPvBac14Y40UYx1zMswnHr4/7Rfgal1mssoP09B8Y2+xcnHT2dGSfAheGVICsWnqbPxeiiDdvGu6xvE6dZeO2/qwApIAvXPhyWRfsLmtnk7zhmdegtDybDeV4Y8LFAOBeES6yNHdKxhq+4t/QkLeUamGFq8mM2jtMA59tXDrwyLBAhgU0mU4cAuThiMmkdj+7Y1vcCu4j8hCcE1LNu/Qhb6uzpwtd1hyd+2Z0Sje3wD16wPXNzYNuVQXwEks/Y6RIux9jrBpB8x81RzRFDvVnM9BViNDpjGYHLhcNLHLrrWdO7njnKRTz2Z0bPYZmIvw+AMVc4PecTcMxHgxfI+lhXpsuYOjFWhP8HK8gY9eV0esRFzOZVD7DmjW97dWyicTj9fx/atmXPyC4YDKy66az2VzVvpJxXPt50/0MvhuizNMGWYxsQXXFnmMjpaTa7/eANmipeftO0R1SNuAZwtWOqSpkmZiDkg9KfhFK/i8AAP//k1OOZw==" + return "eJzsWt9v2zYQfs9fcQgGdGsb56XYgx8KeG6KBVjSIEnXR+VMnW3OEqmSVF132/8+kKJsWT9sWU4yJ4gfLfLu++6++ySLPoEZLfqAcz1GNUFDRwCGm4j6cDz4cgMfs2+PjwBC0kzxxHAp+vD+CADgbrXvDmIZphEBk1FEzGiw2/1FiMkozjSMlYzBoJ7ZbzBEg0AiTCQXpncEoCgi1NSHERk8AhhzikLdd7lOQGBMJaj2YxYJ9WGiZJr4b4rbiluZFAa5INWLcESR7jEZBxjjDylwrgNiOmBRqg2p5dY8/IwWc6nCwvdr5Tgb3sAw2+pydUqdLzlZi7ADgmWEzhhsa05QiS7ZXVsH15ed84Y05oLbgCdjjHm06IxiFQlKkfbA9I2U5rJ7aQqgfKhsdQWbXR1og0ZXchVVXsn0vnAB4G4V5i6nq8FMV8NopmhgTopAM4UJhdl4FufWAXcx4Ke/z4Y3wfDT5e3g/PLsOrg4ux18GNwOgs/X58Gf7/49tWtPs7W1w51/ykOef8pTu9aubLaC0mRsakKlPMX5hJ+veJKQkr/UpnOl2yfXsumVIHkKHpIwfMzXvGanHO9LFwGGy/lfRQdkSmrt0GhAEea11G9hPuVsCvQ1xUiDkYWhcDV6A69OX8Gbwtc87DXXKyTNFYVOcqm+P1a3UwIfG7LYMJbKKdmVOBOtG11b9w0IZ0LOxYPgc5G7o4splmoRTFGFQcRjbmrRaYYRhcE4klhe0ALi76hCuHCJ4A+bowNOlqS1yMq2VIF0nQrDY4Lh1efcfnql9XXjX8w9IyUo6iWszL1VgVwGqWI0fUhIMRJ1K7aU0X6uss04IZBjcKS48OBAJ8ioTKxCQUgVPwEeYHFixH9QCKOFU4pI4xEpu8E2kklFlTZW2BrOZuVpW1GNpJh0I2ARuNjt668X2tDBlV4ajDybrAm20h7qNiqHKKUNfPaVlGf9kJLyQF2KZiCpJnVodffVttC2DYKDf4jaqXDYVzGO6aNYUJu6u9n432t+u5zQVONkK9yD0EkJcxdZ5KxCrmdcdnuO+cD1DM5PP3V9iFGE5afLzdlblmfAWBqnERoKXQ4NYaq4mLjKRHycTdeUVs/ydY1qAF8kIJO6QYJtw9SShv1cLvvYmcgK7mhhqDPgXLibgrQk9ZsN4Qjtx0etXnvtTGfn+rNUKRLG9yGx9kZMijoBF27RpL5xRoG18gdGmnmCu2cYmSeG80+g6GtK2ui31pcFCpnhru9hDnyO3Dwy6hwn6MTW2SKw0uACvqaUUvZL0vPaiYvbXt+n+yayst0s6ZLURg9svhtvuZmElChi1uf68GvvXTe37CDxpUoUr53Ae3Vwl+RZWHh3Jofq4ZaRIfEEbdz34sXHX3y8hsjuPp7p6cCMfLvIlwJP4xjV4nGexlE8J1e3P4BkQgrtlmfl7u4JPW/W07T5QmNerP7F6muI7G717kXQgTh9xVAbVL5+ttbtdY8/Luv4sqd8hL+eXI7+otqXatmFYGPPC2uCGJOEi4nfcPz6uFuFr3Hua+UP/LP/6lizc3XS/mrPXgUuDKkxsurorf5hEVcPMuE+bq9DF9m4V39WAFLAFy5COa8rdpt7Z5O84ZHvQdl4NgfK8SaEswOAe0U4821u1YwlfMW/oaFgLtXMCleT6W12mAY+27i04OGxgMcCmkwrDmPkUY/JtPZ9dktb3AruI/IIXBJSzW/p6/4wAI9QOm+OLn0zOqWbLXAPD7i+uVlzqyqApzj6npEi7f7AY9UImv+oOaIpcqg/m4G2QoRWZzQ7cLloYJFH33jm5I53HkIxn9250X1oJsbvB6CYC/yes2k4xoND13kN7hyvIGMtufe6x8VIppVfMM1y2PZU1kTi/uzyn1r25WdvLpiM7S3HxWweiFDJJKn9ZdD++LodIp9phSxBNqO6OcmxkVJS7XeyvglaFt7ejttD8gseoVybMVWFLFPzouQXJT8JJf8XAAD//5wYkBM=" } diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json index 002da6310183..2202251a02d8 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/data.json @@ -108,7 +108,6 @@ }, "usage": { "max": 15294464, - "pct": 0.003136136404770672, "total": 12349440 } }, diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml index b0dddd97691d..b79be57b70c8 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/fields.yml @@ -22,6 +22,10 @@ type: keyword description: > The known status for the task from Amazon ECS. + - name: memory_hard_limit + type: scaled_float + description: > + The Hard Memory Limit for the task from Amazon ECS. - name: cpu type: group description: Runtime CPU metrics. @@ -251,11 +255,6 @@ format: bytes description: > Max memory usage. - - name: pct - type: scaled_float - format: percent - description: > - Memory usage percentage. - name: total type: long format: bytes diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json index 37399e027a93..d06b46e0df09 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json +++ b/x-pack/metricbeat/module/awsfargate/task_stats/_meta/testdata/task.json @@ -5,6 +5,9 @@ "Revision": "7", "DesiredStatus": "RUNNING", "KnownStatus": "ACTIVATING", + "Limits": { + "Memory": 7168 + }, "Containers": [{ "DockerId": "1234", "Name": "query-metadata", @@ -14,6 +17,9 @@ "com.amazonaws.ecs.container-name": "query-metadata", "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/febee046097849aba589d4435207c04a", "com.amazonaws.ecs.task-definition-family": "query-metadata", - "com.amazonaws.ecs.task-definition-version": "7"} + "com.amazonaws.ecs.task-definition-version": "7"}, + "Limits": { + "Memory": 3328 + } }] -} +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/container.go b/x-pack/metricbeat/module/awsfargate/task_stats/container.go index 3479fc580e87..9c4d7d0d0e61 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/container.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/container.go @@ -15,6 +15,7 @@ type container struct { Name string Image string Labels map[string]string + Limits Limits } func getContainerMetadata(c *container) *container { @@ -23,6 +24,7 @@ func getContainerMetadata(c *container) *container { Image: c.Image, Name: helpers.ExtractContainerName([]string{c.Name}), Labels: deDotLabels(c.Labels), + Limits: c.Limits, } } diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/data.go b/x-pack/metricbeat/module/awsfargate/task_stats/data.go index d1a4eb3277e9..0db9f40135d0 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/data.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/data.go @@ -57,6 +57,11 @@ func createEvent(stats *Stats) mb.Event { _, _ = e.MetricSetFields.Put("task_known_status", taskKnownStatus) } + memoryHardLimit := stats.Container.Limits.Memory + if memoryHardLimit != 0 { + _, _ = e.MetricSetFields.Put("memory_hard_limit", memoryHardLimit) + } + _, _ = e.MetricSetFields.Put("identifier", generateIdentifier(stats.Container.Name, stats.Container.DockerId)) return e } @@ -163,7 +168,6 @@ func createMemoryFields(stats *Stats) mapstr.M { }, "usage": mapstr.M{ "total": stats.memoryStats.Usage, - "pct": stats.memoryStats.UsageP, "max": stats.memoryStats.MaxUsage, }, } diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/memory.go b/x-pack/metricbeat/module/awsfargate/task_stats/memory.go index 0dfb68d9317a..ec581a2d3c69 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/memory.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/memory.go @@ -13,7 +13,6 @@ type memoryStats struct { TotalRss uint64 TotalRssP float64 Usage uint64 - UsageP float64 //Raw stats from the cgroup subsystem Stats map[string]uint64 //Windows-only memory stats @@ -30,7 +29,6 @@ func getMemoryStats(taskStats types.StatsJSON) memoryStats { MaxUsage: taskStats.Stats.MemoryStats.MaxUsage, TotalRssP: float64(totalRSS) / float64(taskStats.Stats.MemoryStats.Limit), Usage: taskStats.Stats.MemoryStats.Usage, - UsageP: float64(taskStats.Stats.MemoryStats.Usage) / float64(taskStats.Stats.MemoryStats.Limit), Stats: taskStats.Stats.MemoryStats.Stats, //Windows memory statistics Commit: taskStats.Stats.MemoryStats.Commit, diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go index 21eab0f0bc74..39ed9ba85bb9 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats.go @@ -79,9 +79,15 @@ type TaskMetadata struct { Revision string `json:"Revision"` DesiredStatus string `json:"DesiredStatus"` KnownStatus string `json:"KnownStatus"` + Limit Limits `json:"Limits"` Containers []*container `json:"Containers"` } +// Limits is a struct that represents the memory limit from ${ECS_CONTAINER_METADATA_URI_V4}/task, which is the Hard Memory Limit set in AWS ECS +type Limits struct { + Memory uint64 `json:"Memory"` +} + // New creates a new instance of the MetricSet. New is responsible for unpacking // any MetricSet specific configuration options if there are any. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { @@ -131,6 +137,8 @@ func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { if err != nil { return nil, fmt.Errorf("http.Get failed: %w", err) } + defer taskStatsResp.Body.Close() + taskStatsOutput, err := getTaskStats(taskStatsResp) if err != nil { return nil, fmt.Errorf("getTaskStats failed: %w", err) @@ -145,6 +153,8 @@ func (m *MetricSet) queryTaskMetadataEndpoints() ([]Stats, error) { if err != nil { return nil, fmt.Errorf("http.Get failed: %w", err) } + defer taskResp.Body.Close() + taskOutput, err := getTask(taskResp) if err != nil { return nil, fmt.Errorf("getTask failed: %w", err) diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go index f5c32de5deec..ec5e4ddfab37 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_integration_test.go @@ -51,6 +51,11 @@ func TestFetch(t *testing.T) { // The goal here is to make sure every element inside the // event has a matching field ("no field left behind"). mbtest.TestMetricsetFieldsDocumented(t, metricSet, []mb.Event{event}) + + t.Cleanup(func() { + taskStatsResp.Body.Close() + byteTaskResp.Body.Close() + }) } func TestData(t *testing.T) { @@ -85,6 +90,11 @@ func TestData(t *testing.T) { standardizeEvent := m.StandardizeEvent(event) mbtest.WriteEventToDataJSON(t, standardizeEvent, "") + + t.Cleanup(func() { + taskStatsResp.Body.Close() + byteTaskResp.Body.Close() + }) } // buildResponse is a test helper that loads the content of `filename` and returns diff --git a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go index 7756a15f1e77..374df91a9e34 100644 --- a/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go +++ b/x-pack/metricbeat/module/awsfargate/task_stats/task_stats_test.go @@ -6,7 +6,7 @@ package task_stats import ( "bytes" - "io/ioutil" + "io" "net/http" "testing" @@ -48,6 +48,9 @@ var ( "Revision": "7", "DesiredStatus": "RUNNING", "KnownStatus": "ACTIVATING", + "Limits": { + "Memory": 7168 + }, "Containers": [{ "DockerId": "query-metadata-1", "Name": "query-metadata", @@ -57,14 +60,17 @@ var ( "com.amazonaws.ecs.container-name": "query-metadata", "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:111122223333:task/default/febee046097849aba589d4435207c04a", "com.amazonaws.ecs.task-definition-family": "query-metadata", - "com.amazonaws.ecs.task-definition-version": "7"} - }] - }` + "com.amazonaws.ecs.task-definition-version": "7"}, + "Limits": { + "Memory": 3328 + } + }] + }` ) func TestGetTaskStats(t *testing.T) { taskStatsResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + Body: io.NopCloser(bytes.NewReader([]byte(taskStatsJson))), } taskStatsOutput, err := getTaskStats(taskStatsResp) @@ -74,7 +80,7 @@ func TestGetTaskStats(t *testing.T) { func TestGetTask(t *testing.T) { taskResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader([]byte(taskRespJson))), + Body: io.NopCloser(bytes.NewReader([]byte(taskRespJson))), } taskOutput, err := getTask(taskResp) @@ -87,23 +93,25 @@ func TestGetTask(t *testing.T) { assert.Equal(t, "RUNNING", taskOutput.DesiredStatus) assert.Equal(t, "ACTIVATING", taskOutput.KnownStatus) + assert.Equal(t, uint64(7168), taskOutput.Limit.Memory) assert.Equal(t, 1, len(taskOutput.Containers)) assert.Equal(t, "query-metadata-1", taskOutput.Containers[0].DockerId) assert.Equal(t, "query-metadata", taskOutput.Containers[0].Name) assert.Equal(t, "mreferre/eksutils", taskOutput.Containers[0].Image) assert.Equal(t, 5, len(taskOutput.Containers[0].Labels)) + assert.Equal(t, uint64(3328), taskOutput.Containers[0].Limits.Memory) } func TestGetStatsList(t *testing.T) { taskStatsResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + Body: io.NopCloser(bytes.NewReader([]byte(taskStatsJson))), } taskStatsOutput, err := getTaskStats(taskStatsResp) assert.NoError(t, err) taskResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader([]byte(taskRespJson))), + Body: io.NopCloser(bytes.NewReader([]byte(taskRespJson))), } taskOutput, err := getTask(taskResp) @@ -115,7 +123,7 @@ func TestGetStatsList(t *testing.T) { func TestGetCPUStats(t *testing.T) { taskStatsResp := &http.Response{ - Body: ioutil.NopCloser(bytes.NewReader([]byte(taskStatsJson))), + Body: io.NopCloser(bytes.NewReader([]byte(taskStatsJson))), } taskStatsOutput, err := getTaskStats(taskStatsResp)