From f4c04e1a28abe9fc819388e046946ee77a5bb809 Mon Sep 17 00:00:00 2001 From: David Neuman Date: Fri, 22 Jan 2016 13:53:51 -0700 Subject: [PATCH 1/2] updated scripted dashboards to use continuous queries --- .../grafana/traffic_ops_cachegroup.js | 2 +- .../grafana/traffic_ops_deliveryservice.js | 12 ++++++------ traffic_stats/grafana/traffic_ops_scripted.js | 18 +++++++++--------- traffic_stats/grafana/traffic_ops_server.js | 4 ++-- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/traffic_stats/grafana/traffic_ops_cachegroup.js b/traffic_stats/grafana/traffic_ops_cachegroup.js index 8cf15d660..52ee42bae 100644 --- a/traffic_stats/grafana/traffic_ops_cachegroup.js +++ b/traffic_stats/grafana/traffic_ops_cachegroup.js @@ -116,7 +116,7 @@ dashboard.title = which; "targets": [ { "rawQuery": true, - "query": "SELECT sum(value)*1000/6 FROM \"bandwidth\" WHERE cachegroup='" + which + "' and $timeFilter GROUP BY time(60s), hostname", + "query": "SELECT sum(value)*1000 FROM \"monthly\".\"bandwidth.1min\" WHERE cachegroup='" + which + "' and $timeFilter GROUP BY time(60s), hostname", "alias": "$tag_hostname" } ], diff --git a/traffic_stats/grafana/traffic_ops_deliveryservice.js b/traffic_stats/grafana/traffic_ops_deliveryservice.js index fd7c5c5e6..4f093c389 100644 --- a/traffic_stats/grafana/traffic_ops_deliveryservice.js +++ b/traffic_stats/grafana/traffic_ops_deliveryservice.js @@ -108,7 +108,7 @@ dashboard.title = which; "targets": [ { "measurement": "bw", - "query": "SELECT mean(value)*1000 FROM \"kbps\" WHERE deliveryservice='" + which + "' and cachegroup = 'total' and $timeFilter GROUP BY time(60s), deliveryservice ORDER BY asc", + "query": "SELECT mean(value)*1000 FROM \"monthly\".\"kbps.ds.1min\" WHERE deliveryservice='" + which + "' and cachegroup = 'total' and $timeFilter GROUP BY time(60s), deliveryservice ORDER BY asc", "rawQuery": true, "tags": { "deliveryservice": which @@ -198,24 +198,24 @@ dashboard.title = which; "tags": { "deliveryservice": which }, - "query": "SELECT mean(value) FROM \"tps_2xx\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time($interval) ORDER BY asc", + "query": "SELECT mean(value) FROM \"monthly\".\"tps_2xx.ds.1min\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time(60s) ORDER BY asc", "hide": false, "rawQuery": true }, { "target": "", "rawQuery": true, - "query": "SELECT mean(value) FROM \"tps_3xx\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time($interval) ORDER BY asc" + "query": "SELECT mean(value) FROM \"monthly\".\"tps_3xx.ds.1min\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time(60s) ORDER BY asc" }, { "target": "", "rawQuery": true, - "query": "SELECT mean(value) FROM \"tps_4xx\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time($interval) ORDER BY asc" + "query": "SELECT mean(value) FROM \"monthly\".\"tps_4xx.ds.1min\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time(60s) ORDER BY asc" }, { "target": "", "rawQuery": true, - "query": "SELECT mean(value) FROM \"tps_5xx\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time($interval) ORDER BY asc" + "query": "SELECT mean(value) FROM \"monthly\".\"tps_5xx.ds.1min\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time(60s) ORDER BY asc" } ], "aliasColors": {}, @@ -273,7 +273,7 @@ dashboard.title = which; "steppedLine": false, "targets": [ { - "query": "SELECT sum(value)*1000/6 FROM \"kbps\" WHERE deliveryservice='" + which + "' and cachegroup != 'total' and $timeFilter GROUP BY time(60s), cachegroup", + "query": "SELECT mean(value)*1000 FROM \"monthly\".\"kbps.ds.1min\" WHERE deliveryservice='" + which + "' and cachegroup != 'total' and $timeFilter GROUP BY time(60s), cachegroup", "rawQuery": true, "alias": "$tag_cachegroup" } diff --git a/traffic_stats/grafana/traffic_ops_scripted.js b/traffic_stats/grafana/traffic_ops_scripted.js index f8b8b493a..07f9f55ce 100644 --- a/traffic_stats/grafana/traffic_ops_scripted.js +++ b/traffic_stats/grafana/traffic_ops_scripted.js @@ -112,7 +112,7 @@ if (type == "deliveryservice") { "targets": [ { "measurement": "bw", - "query": "SELECT mean(value)*1000 FROM \"kbps\" WHERE deliveryservice='" + which + "' and cachegroup = 'total' and $timeFilter GROUP BY time(60s), deliveryservice ORDER BY asc", + "query": "SELECT mean(value)*1000 FROM \"monthly\".\"kbps.ds.1min\" WHERE deliveryservice='" + which + "' and cachegroup = 'total' and $timeFilter GROUP BY time(60s), deliveryservice ORDER BY asc", "rawQuery": true, "tags": { "deliveryservice": which @@ -201,24 +201,24 @@ if (type == "deliveryservice") { "tags": { "deliveryservice": which }, - "query": "SELECT mean(value) FROM \"tps_2xx\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time($interval) ORDER BY asc", + "query": "SELECT mean(value) FROM \"monthly\".\"tps_2xx.ds.1min\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time(60s) ORDER BY asc", "hide": false, "rawQuery": true }, { "target": "", "rawQuery": true, - "query": "SELECT mean(value) FROM \"tps_3xx\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time($interval) ORDER BY asc" + "query": "SELECT mean(value) FROM \"monthly\".\"tps_3xx.ds.1min\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time(60s) ORDER BY asc" }, { "target": "", "rawQuery": true, - "query": "SELECT mean(value) FROM \"tps_4xx\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time($interval) ORDER BY asc" + "query": "SELECT mean(value) FROM \"monthly\".\"tps_4xx.ds.1min\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time(60s) ORDER BY asc" }, { "target": "", "rawQuery": true, - "query": "SELECT mean(value) FROM \"tps_5xx\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time($interval) ORDER BY asc" + "query": "SELECT mean(value) FROM \"monthly\".\"tps_5xx.ds.1min\" WHERE $timeFilter AND deliveryservice='" + which + "' GROUP BY time(60s) ORDER BY asc" } ], "aliasColors": {}, @@ -276,7 +276,7 @@ if (type == "deliveryservice") { "steppedLine": false, "targets": [ { - "query": "SELECT sum(value)*1000/6 FROM \"kbps\" WHERE deliveryservice='" + which + "' and cachegroup != 'total' and $timeFilter GROUP BY time(60s), cachegroup", + "query": "SELECT mean(value)*1000 FROM \"monthly\".\"kbps.ds.1min\" WHERE deliveryservice='" + which + "' and cachegroup != 'total' and $timeFilter GROUP BY time(60s), cachegroup", "rawQuery": true } ], @@ -359,7 +359,7 @@ else if ( type == "cachegroup" ) { "targets": [ { "rawQuery": true, - "query": "SELECT sum(value)*1000/6 FROM \"bandwidth\" WHERE cachegroup='" + which + "' and $timeFilter GROUP BY time(60s), hostname" + "query": "SELECT mean(value)*1000 FROM \"monthly\".\"bandwidth.1min\" WHERE cachegroup='" + which + "' and $timeFilter GROUP BY time(60s), hostname" } ], "aliasColors": {}, @@ -434,7 +434,7 @@ else if ( type == "server" ) { { "measurement": "bandwidth", "tags": {}, - "query": "SELECT sum(value)*1000/6 FROM \"bandwidth\" WHERE hostname='" + which + "' and $timeFilter GROUP BY time(60s)", + "query": "SELECT mean(value)*1000 FROM \"monthly\".\"bandwidth.1min\" WHERE hostname='" + which + "' and $timeFilter GROUP BY time(60s)", "rawQuery": true } ], @@ -506,7 +506,7 @@ else if ( type == "server" ) { { "measurement": "bandwidth", "tags": {}, - "query": "SELECT sum(value)*1000/6 FROM \"ats.proxy.process.http.current_client_connections\" WHERE hostname='" + which + "' and $timeFilter GROUP BY time(60s)", + "query": "SELECT mean(value)*1000 FROM \"monthly\".\"connections.1min\" WHERE hostname='" + which + "' and $timeFilter GROUP BY time(60s)", "rawQuery": true } ], diff --git a/traffic_stats/grafana/traffic_ops_server.js b/traffic_stats/grafana/traffic_ops_server.js index b61c4317b..0811fc47b 100644 --- a/traffic_stats/grafana/traffic_ops_server.js +++ b/traffic_stats/grafana/traffic_ops_server.js @@ -118,7 +118,7 @@ dashboard.title = which; { "measurement": "bandwidth", "tags": {}, - "query": "SELECT sum(value)*1000/6 FROM \"bandwidth\" WHERE hostname='" + which + "' and $timeFilter GROUP BY time(60s)", + "query": "SELECT mean(value)*1000 FROM \"monthly\".\"bandwidth.1min\" WHERE hostname='" + which + "' and $timeFilter GROUP BY time(60s)", "rawQuery": true } ], @@ -190,7 +190,7 @@ dashboard.title = which; { "measurement": "bandwidth", "tags": {}, - "query": "SELECT sum(value)*1000/6 FROM \"ats.proxy.process.http.current_client_connections\" WHERE hostname='" + which + "' and $timeFilter GROUP BY time(60s)", + "query": "SELECT mean(value)*1000 FROM \"monthly\".\"connections.1min\" WHERE hostname='" + which + "' and $timeFilter GROUP BY time(60s)", "rawQuery": true } ], From 03e3b5fb9becdb5beb42decc0006fa6fdc4c8d88 Mon Sep 17 00:00:00 2001 From: David Neuman Date: Mon, 25 Jan 2016 10:23:49 -0700 Subject: [PATCH 2/2] update graphs to combine 3 api calls into one and to use continuous queries. This allows us to display more consistent data to the user --- .../Extensions/TrafficStats/API/CacheStats.pm | 84 +++++++++++++------ traffic_ops/app/lib/TrafficOpsRoutes.pm | 4 +- .../templates/visual_status/graphs.html.ep | 83 ++++++------------ 3 files changed, 87 insertions(+), 84 deletions(-) diff --git a/traffic_ops/app/lib/Extensions/TrafficStats/API/CacheStats.pm b/traffic_ops/app/lib/Extensions/TrafficStats/API/CacheStats.pm index f8f53f705..c30b738c1 100644 --- a/traffic_ops/app/lib/Extensions/TrafficStats/API/CacheStats.pm +++ b/traffic_ops/app/lib/Extensions/TrafficStats/API/CacheStats.pm @@ -111,42 +111,78 @@ sub get_stat { return $summary_content->{results}[0]{series}[0]{values}[0][1]; } - return "err"; + return ""; } -sub current_bandwidth { +sub current_stats { my $self = shift; - my $cdn = $self->param('cdnName'); - my $query = "SELECT sum(value)/6 FROM \"bandwidth\" WHERE time < now() - 60s and time > now() - 120s"; - if ($cdn) { - $query = "SELECT sum(value)/6 FROM \"bandwidth\" WHERE time < now() - 60s and time > now() - 120s and cdn = \'$cdn\'"; + my @stats; + my $current_bw = $self->get_current_bandwidth(); + my $conns = $self->get_current_connections(); + my $capacity = $self->get_current_capacity(); + my $rs = $self->db->resultset('Cdn'); + while ( my $cdn = $rs->next ) { + my $cdn_name = $cdn->name; + my $bw = $current_bw->{$cdn_name}; + my $conn = $conns->{$cdn_name}; + my $cap = $capacity->{$cdn_name}; + push(@stats, ({cdn => $cdn_name, bandwidth => $bw, connections => $conn, capacity => $cap})); } - my $bandwidth = $self->get_stat("cache_stats", $query); - return $self->success({"bandwidth" => $bandwidth/1000000}); + push(@stats, ({cdn => "total", bandwidth => $current_bw->{"total"}, connections => $conns->{"total"}})); + return $self->success({"currentStats" => \@stats}); } -sub current_connections { +sub get_current_bandwidth { my $self = shift; - my $cdn = $self->param('cdnName'); - my $query = "select sum(value)/6 from \"ats.proxy.process.http.current_client_connections\" where time > now() - 120s and time < now() - 60s"; - if ($cdn) { - $query = "select sum(value)/6 from \"ats.proxy.process.http.current_client_connections\" where time > now() - 120s and time < now() - 60s and cdn = \'$cdn\'"; + my $bw; + my $total_bw = 0; + my $rs = $self->db->resultset('Cdn'); + while ( my $cdn = $rs->next ) { + my $cdn_name = $cdn->name; + my $query = "SELECT last(value) FROM \"monthly\".\"bandwidth.cdn.1min\" WHERE cdn = \'$cdn_name\'"; + my $bandwidth = $self->get_stat("cache_stats", $query); + if ($bandwidth) { + $bw->{$cdn_name} = $bandwidth/1000000; + $total_bw += $bandwidth; + } } - my $connections = $self->get_stat("cache_stats", $query); - return $self->success({"connections" => $connections}); + $bw->{"total"} = $total_bw/1000000; + return $bw; } -sub current_capacity { +sub get_current_connections { my $self = shift; - my $cdn = $self->param('cdnName'); - my $query = "select sum(value)/6 from \"maxKbps\" where time > now() - 120s and time < now() - 60s"; - if ($cdn) { - $query = "select sum(value)/6 from \"maxKbps\" where time > now() - 120s and time < now() - 60s and cdn = \'$cdn\'"; + my $conn; + my $total_conn = 0; + my $rs = $self->db->resultset('Cdn'); + while ( my $cdn = $rs->next ) { + my $cdn_name = $cdn->name; + my $query = "select last(value) from \"monthly\".\"connections.cdn.1min\" where cdn = \'$cdn_name\'"; + my $connections = $self->get_stat("cache_stats", $query); + if ($connections) { + $conn->{$cdn_name} = $connections; + $total_conn += $connections; + } + } + $conn->{"total"} = $total_conn; + return $conn; +} + +sub get_current_capacity { + my $self = shift; + my $cap; + my $rs = $self->db->resultset('Cdn'); + while ( my $cdn = $rs->next ) { + my $cdn_name = $cdn->name; + my $query = "select sum(value)/6 from \"maxKbps\" where time > now() - 120s and time < now() - 60s and cdn = \'$cdn_name\'"; + my $capacity = $self->get_stat("cache_stats", $query); + if ($capacity) { + $capacity = $capacity/1000000; #convert to Gbps + $capacity = $capacity * 0.85; # need a better way to figure out percentage of max besides hard-coding + $cap->{$cdn_name} = $capacity; + } } - my $capacity = $self->get_stat("cache_stats", $query); - $capacity = $capacity/1000000; #convert to Gbps - $capacity = $capacity * 0.85; # need a better way to figure out percentage of max besides hard-coding - return $self->success({"capacity" => $capacity}); + return $cap; } sub daily_summary { diff --git a/traffic_ops/app/lib/TrafficOpsRoutes.pm b/traffic_ops/app/lib/TrafficOpsRoutes.pm index f8a9bbcf6..38e67014e 100644 --- a/traffic_ops/app/lib/TrafficOpsRoutes.pm +++ b/traffic_ops/app/lib/TrafficOpsRoutes.pm @@ -778,10 +778,8 @@ sub traffic_stats_routes { $r->get( "/api/$version/deliveryservice_stats" => [ format => [qw(json)] ] )->over( authenticated => 1 ) ->to( 'DeliveryServiceStats#index', namespace => $namespace ); $r->get( "/api/$version/cache_stats" => [ format => [qw(json)] ] )->over( authenticated => 1 )->to( 'CacheStats#index', namespace => $namespace ); - $r->get( "internal/api/$version/current_bandwidth" => [ format => [qw(json)] ] )->to( 'CacheStats#current_bandwidth', namespace => $namespace ); - $r->get( "internal/api/$version/current_connections" => [ format => [qw(json)] ] )->to( 'CacheStats#current_connections', namespace => $namespace ); - $r->get( "internal/api/$version/current_capacity" => [ format => [qw(json)] ] )->to( 'CacheStats#current_capacity', namespace => $namespace ); $r->get( "internal/api/$version/daily_summary" => [ format => [qw(json)] ] )->to( 'CacheStats#daily_summary', namespace => $namespace ); + $r->get( "internal/api/$version/current_stats" => [ format => [qw(json)] ] )->to( 'CacheStats#current_stats', namespace => $namespace ); } sub catch_all { diff --git a/traffic_ops/app/templates/visual_status/graphs.html.ep b/traffic_ops/app/templates/visual_status/graphs.html.ep index 7775153f5..04eca4858 100644 --- a/traffic_ops/app/templates/visual_status/graphs.html.ep +++ b/traffic_ops/app/templates/visual_status/graphs.html.ep @@ -28,67 +28,36 @@ $(function () {