Skip to content

Commit

Permalink
Merge pull request apache#12408: [BEAM-10602] Display Python streamin…
Browse files Browse the repository at this point in the history 
…g metrics in Grafana dashboard
  • Loading branch information
@mxm
mxm authored Jul 31, 2020
2 parents 835805d + ec6fb19 commit cdc2475
Showing 1 changed file with 56 additions and 33 deletions.
89 changes: 56 additions & 33 deletions .test-infra/metrics/grafana/dashboards/perftests_metrics/ParDo_Load_Tests.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"iteration": 1588593729528,
"iteration": 1596019686624,
"links": [],
"panels": [
{
Expand Down Expand Up @@ -78,7 +78,7 @@
],
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_1\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"",
"query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"",
"rawQuery": true,
"refId": "A",
"resultFormat": "time_series",
Expand All @@ -103,7 +103,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$sdk | ParDo | 2GB, 100 byte records, 10 iterations",
"title": "$sdk | ParDo | 2GB, 100 byte records, 5 iterations, 10 counter operations, 3 counters, parallelism 5",
"tooltip": {
"shared": true,
"sort": 0,
Expand Down Expand Up @@ -158,7 +158,7 @@
"y": 0
},
"hiddenSeries": false,
"id": 3,
"id": 4,
"interval": "1d",
"legend": {
"avg": false,
Expand Down Expand Up @@ -195,15 +195,14 @@
},
{
"params": [
"metric"
"null"
],
"type": "field"
"type": "fill"
}
],
"measurement": "python_batch_cogbk_2",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_2\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"",
"query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_5\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"",
"rawQuery": true,
"refId": "A",
"resultFormat": "time_series",
Expand All @@ -228,7 +227,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$sdk | ParDo | 2GB, 100 byte records, 200 iterations",
"title": "$sdk | ParDo | 2GB, 100 byte records, 5 iterations, 10 counter operations, 3 counters, parallelism 5",
"tooltip": {
"shared": true,
"sort": 0,
Expand Down Expand Up @@ -283,7 +282,7 @@
"y": 8
},
"hiddenSeries": false,
"id": 4,
"id": 5,
"interval": "1d",
"legend": {
"avg": false,
Expand All @@ -310,7 +309,7 @@
"steppedLine": false,
"targets": [
{
"alias": "$tag_metric",
"alias": "",
"groupBy": [
{
"params": [
Expand All @@ -320,15 +319,14 @@
},
{
"params": [
"metric"
"null"
],
"type": "field"
"type": "fill"
}
],
"measurement": "python_batch_cogbk_3",
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_3\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"",
"query": "SELECT mean(value)/1000 FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /latency/ AND $timeFilter GROUP BY time($__interval), \"metric\"",
"rawQuery": true,
"refId": "A",
"resultFormat": "time_series",
Expand All @@ -353,7 +351,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$sdk | ParDo | 2GB, 100 byte records, 10 counter increments",
"title": "Latency $sdk | ParDo | 2GB, 100 byte records, 5 iterations, 10 counter operations, 3 counters, parallelism 5 (latency)",
"tooltip": {
"shared": true,
"sort": 0,
Expand Down Expand Up @@ -408,7 +406,7 @@
"y": 8
},
"hiddenSeries": false,
"id": 5,
"id": 3,
"interval": "1d",
"legend": {
"avg": false,
Expand All @@ -435,12 +433,24 @@
"steppedLine": false,
"targets": [
{
"alias": "$tag_metric",
"groupBy": [],
"measurement": "python_batch_cogbk_4",
"alias": "",
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"orderByTime": "ASC",
"policy": "default",
"query": "SELECT mean(\"value\") FROM \"${sdk}_${processingType}_pardo_4\" WHERE \"metric\" =~ /runtime/ AND $timeFilter GROUP BY time($__interval), \"metric\"",
"query": "SELECT min, max, sum / count\nFROM\n(\n SELECT max(value)/1000 as min FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_min/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as max FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_max/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as sum FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_sum/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n),\n(\n SELECT max(value)/1000 as count FROM \"${sdk}_${processingType}_pardo_6\" WHERE \"metric\" =~ /loadgenerator\\/impulse.*_count/ AND $timeFilter GROUP BY time($__interval), \"metric\"\n)\n\n",
"rawQuery": true,
"refId": "A",
"resultFormat": "time_series",
Expand All @@ -451,6 +461,10 @@
"value"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
Expand All @@ -461,7 +475,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "$sdk | ParDo | 2GB, 100 byte records, 100 counter increments",
"title": "Checkpoint duration $sdk | ParDo | 2GB, 100 byte records, 5 iterations, 10 counter operations, 3 counters, parallelism 5",
"tooltip": {
"shared": true,
"sort": 0,
Expand Down Expand Up @@ -505,15 +519,18 @@
"refresh": false,
"schemaVersion": 22,
"style": "dark",
"tags": ["performance tests"],
"tags": [
"performance tests"
],
"templating": {
"list": [
{
"allValue": null,
"current": {
"selected": false,
"text": "batch",
"value": "batch"
"tags": [],
"text": "streaming",
"value": "streaming"
},
"hide": 0,
"includeAll": false,
Expand All @@ -522,12 +539,14 @@
"name": "processingType",
"options": [
{
"selected": true,
"$$hashKey": "object:283",
"selected": false,
"text": "batch",
"value": "batch"
},
{
"selected": false,
"$$hashKey": "object:284",
"selected": true,
"text": "streaming",
"value": "streaming"
}
Expand All @@ -540,8 +559,9 @@
"allValue": null,
"current": {
"selected": false,
"text": "java",
"value": "java"
"tags": [],
"text": "python",
"value": "python"
},
"hide": 0,
"includeAll": false,
Expand All @@ -550,16 +570,19 @@
"name": "sdk",
"options": [
{
"selected": true,
"$$hashKey": "object:272",
"selected": false,
"text": "java",
"value": "java"
},
{
"selected": false,
"$$hashKey": "object:273",
"selected": true,
"text": "python",
"value": "python"
},
{
"$$hashKey": "object:274",
"selected": false,
"text": "go",
"value": "go"
Expand Down Expand Up @@ -596,5 +619,5 @@
"variables": {
"list": []
},
"version": 1
}
"version": 2
}

0 comments on commit cdc2475

Please sign in to comment.