Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enable metrics collection for multiple provider pods #108

Merged
merged 5 commits into from
May 17, 2023
Merged

Enable metrics collection for multiple provider pods #108

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
28c4213
Gather Prometheus metrics for one or multiple provider pods
May 11, 2023
297b326
Metrics collection restults now work for multiple pods
May 11, 2023
103d5f8
Lower cyclomatic complexity by refactoring to smaller functions
May 11, 2023
a68754c
Aggragated metrics do not have pod names, no need to print them out
May 11, 2023
ee69b3b
Adding missing labels to the aggregated memory and CPU metrics
May 11, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions cmd/perf/internal/common/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type Result struct {
Metric string
MetricUnit string
Peak, Average float64
PodName string
}

// ConstructPrometheusClient creates a Prometheus API Client
Expand All @@ -66,8 +67,10 @@ func ConstructTimeRange(startTime, endTime time.Time, stepDuration time.Duration
}

// ConstructResult creates a Result object from collected data
func ConstructResult(value model.Value, metric, unit string) (*Result, error) {
result := &Result{}
func ConstructResult(value model.Value, metric, unit string, podName string) (*Result, error) {
result := &Result{
PodName: podName,
}
matrix, ok := value.(model.Matrix)
if !ok {
return nil, errors.New("model cannot cast to matrix")
Expand Down Expand Up @@ -101,6 +104,9 @@ func CalculateAverageAndPeak(data []Data) (float64, float64) {

// Print reports the results
func (r Result) Print() {
if r.PodName != "" {
log.Info(fmt.Sprintf("Pod: %s", r.PodName))
}
log.Info(fmt.Sprintf("Average %s: %f %s \n", r.Metric, r.Average, r.MetricUnit))
log.Info(fmt.Sprintf("Peak %s: %f %s \n", r.Metric, r.Peak, r.MetricUnit))
}
80 changes: 58 additions & 22 deletions cmd/perf/internal/quantify.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package internal
import (
"context"
"fmt"
"strings"
"time"

"github.com/pkg/errors"
Expand All @@ -34,7 +35,7 @@ import (

// QuantifyOptions represents the options of quantify command
type QuantifyOptions struct {
providerPod string
providerPods []string
providerNamespace string
mrPaths map[string]int
cmd *cobra.Command
Expand All @@ -56,12 +57,12 @@ func NewCmdQuantify() *cobra.Command {
"reports them. When you execute this tool an end-to-end experiment will run.",
Example: "provider-scale --mrs ./internal/providerScale/manifests/virtualnetwork.yaml=2 " +
"--mrs ./internal/providerScale/manifests/loadbalancer.yaml=2" +
"--provider-pod crossplane-provider-jet-azure " +
"--provider-pods crossplane-provider-jet-azure " +
"--provider-namespace crossplane-system",
RunE: o.Run,
}

o.cmd.Flags().StringVar(&o.providerPod, "provider-pod", "", "Pod name of provider")
o.cmd.Flags().StringSliceVarP(&o.providerPods, "provider-pods", "p", []string{}, "Names of the provider pods. Multiple names can be specified, separated by commas (spaces are ignored).")
o.cmd.Flags().StringVar(&o.providerNamespace, "provider-namespace", "crossplane-system",
"Namespace name of provider")
o.cmd.Flags().StringToIntVar(&o.mrPaths, "mrs", nil, "Managed resource templates that will be deployed")
Expand All @@ -71,7 +72,7 @@ func NewCmdQuantify() *cobra.Command {
o.cmd.Flags().StringVar(&o.nodeIP, "node", "", "Node IP")
o.cmd.Flags().DurationVar(&o.timeout, "timeout", 120*time.Minute, "Timeout for the experiment")

if err := o.cmd.MarkFlagRequired("provider-pod"); err != nil {
if err := o.cmd.MarkFlagRequired("provider-pods"); err != nil {
panic(err)
}
if err := o.cmd.MarkFlagRequired("mrs"); err != nil {
Expand Down Expand Up @@ -114,28 +115,63 @@ func (o *QuantifyOptions) Run(_ *cobra.Command, _ []string) error {
log.Infof("Results\n------------------------------------------------------------\n")
log.Infof("Experiment Duration: %f seconds\n", o.endTime.Sub(o.startTime).Seconds())
time.Sleep(60 * time.Second)
queryResultMemory, err := o.CollectData(fmt.Sprintf(`sum(node_namespace_pod_container:container_memory_working_set_bytes{pod="%s", namespace="%s"})`,
o.providerPod, o.providerNamespace))
if err != nil {
return errors.Wrap(err, "cannot collect memory data")
}
memoryResult, err := common.ConstructResult(queryResultMemory, "Memory", "Bytes")
if err != nil {
return errors.Wrap(err, "cannot construct memory results")
}
qureyResultCPURate, err := o.CollectData(fmt.Sprintf(`instance:node_cpu_utilisation:rate5m{instance="%s"} * 100`, o.nodeIP))

err := o.processPods(timeToReadinessResults)
if err != nil {
return errors.Wrap(err, "cannot collect cpu data")
return errors.Wrap(err, "cannot process pods")
}
cpuRateResult, err := common.ConstructResult(qureyResultCPURate, "CPU", "Rate")
if err != nil {
return errors.Wrap(err, "cannot construct cpu results")
return nil
}

// processPods calculated metrics for provider pods
func (o *QuantifyOptions) processPods(timeToReadinessResults []common.Result) error {
// Initialize aggregated results
var aggregatedMemoryResult = &common.Result{Metric: "Memory", MetricUnit: "Bytes"}
var aggregatedCPURateResult = &common.Result{Metric: "CPU", MetricUnit: "Rate"}

for _, providerPod := range o.providerPods {
providerPod = strings.TrimSpace(providerPod)
queryResultMemory, err := o.CollectData(fmt.Sprintf(`sum(node_namespace_pod_container:container_memory_working_set_bytes{pod="%s", namespace="%s"})`,
providerPod, o.providerNamespace))
if err != nil {
return errors.Wrap(err, "cannot collect memory data")
}
memoryResult, err := common.ConstructResult(queryResultMemory, "Memory", "Bytes", providerPod)
if err != nil {
return errors.Wrap(err, "cannot construct memory results")
}
// Update aggregated memory result
aggregatedMemoryResult.Average += memoryResult.Average
if memoryResult.Peak > aggregatedMemoryResult.Peak {
aggregatedMemoryResult.Peak = memoryResult.Peak
}

queryResultCPURate, err := o.CollectData(fmt.Sprintf(`instance:node_cpu_utilisation:rate5m{instance="%s"} * 100`, o.nodeIP))
if err != nil {
return errors.Wrap(err, "cannot collect cpu data")
}
cpuRateResult, err := common.ConstructResult(queryResultCPURate, "CPU", "Rate", providerPod)
if err != nil {
return errors.Wrap(err, "cannot construct cpu results")
}
// Update aggregated CPU rate result
aggregatedCPURateResult.Average += cpuRateResult.Average
if cpuRateResult.Peak > aggregatedCPURateResult.Peak {
aggregatedCPURateResult.Peak = cpuRateResult.Peak
}

for _, timeToReadinessResult := range timeToReadinessResults {
timeToReadinessResult.Print()
}
memoryResult.Print()
cpuRateResult.Print()
}
for _, timeToReadinessResult := range timeToReadinessResults {
timeToReadinessResult.Print()

if len(o.providerPods) > 1 {
log.Infof("\nAggregated Results\n------------------------------------------------------------\n")
aggregatedMemoryResult.Print()
aggregatedCPURateResult.Print()
}
memoryResult.Print()
cpuRateResult.Print()
return nil
}

Expand Down