Merge pull request #41 from achinthagunasekara/dev-agunasekara-collec…

…t-service-times

Collect service times metrics from Squid

.gitignore

@@ -1,2 +1,3 @@
 bin/*
 squid-exporter
+*.swp

README.md

@@ -89,6 +89,14 @@ Features:
   -  [x] Swap
   -  [ ] Page Faults
   -  [ ] Others
+- [ ] Expose Squid service times
+  - [x] HTTP requests
+  - [x] Cache misses
+  - [x] Cache hits
+  - [x] Near hits
+  - [ ] Not-Modified replies
+  - [x] DNS lookups
+  - [ ] ICP queries
 - [ ] Histograms
 - [ ] Other metrics
 - [x] Squid Authentication (Basic Auth)

VERSION

@@ -1 +1 @@
-1.8.3
+1.9.0

collector/client.go

@@ -25,6 +25,7 @@ type CacheObjectClient struct {
 /*SquidClient provides functionality to fetch squid metrics */
 type SquidClient interface {
 	GetCounters() (types.Counters, error)
+	GetServiceTimes() (types.Counters, error)
 }
 
 const (
@@ -49,15 +50,14 @@ func NewCacheObjectClient(hostname string, port int, login string, password stri
 	}
 }
 
-/*GetCounters fetches counters from squid cache manager */
-func (c *CacheObjectClient) GetCounters() (types.Counters, error) {
-	conn, err := connect(c.hostname, c.port)
+func readFromSquid(hostname string, port int, basicAuthString string, endpoint string) (*bufio.Reader, error) {
+	conn, err := connect(hostname, port)
 
 	if err != nil {
-		return types.Counters{}, err
+		return nil, err
 	}
 
-	r, err := get(conn, "counters", c.basicAuthString)
+	r, err := get(conn, endpoint, basicAuthString)
 
 	if err != nil {
 		return nil, err
@@ -67,21 +67,39 @@ func (c *CacheObjectClient) GetCounters() (types.Counters, error) {
 		return nil, fmt.Errorf("Non success code %d while fetching metrics", r.StatusCode)
 	}
 
-	var counters types.Counters
-
-	// TODO: Move to another func
-	reader := bufio.NewReader(r.Body)
+	return bufio.NewReader(r.Body), err
+}
 
+func readLines(reader *bufio.Reader, lines chan<- string) {
 	for {
 		line, err := reader.ReadString('\n')
 
 		if err == io.EOF {
 			break
 		}
 		if err != nil {
-			return nil, err
+			log.Printf("error reading from the bufio.Reader: %v", err)
+			break
 		}
 
+		lines <- line
+	}
+	close(lines)
+}
+
+/*GetCounters fetches counters from squid cache manager */
+func (c *CacheObjectClient) GetCounters() (types.Counters, error) {
+	var counters types.Counters
+
+	reader, err := readFromSquid(c.hostname, c.port, c.basicAuthString, "counters")
+	if err != nil {
+		return nil, fmt.Errorf("error getting counters: %v", err)
+	}
+
+	lines := make(chan string)
+	go readLines(reader, lines)
+
+	for line := range lines {
 		c, err := decodeCounterStrings(line)
 		if err != nil {
 			log.Println(err)
@@ -93,6 +111,30 @@ func (c *CacheObjectClient) GetCounters() (types.Counters, error) {
 	return counters, err
 }
 
+/*GetServiceTimes fetches service times from squid cache manager */
+func (c *CacheObjectClient) GetServiceTimes() (types.Counters, error) {
+	var serviceTimes types.Counters
+
+	reader, err := readFromSquid(c.hostname, c.port, c.basicAuthString, "service_times")
+	if err != nil {
+		return nil, fmt.Errorf("error getting service times: %v", err)
+	}
+
+	lines := make(chan string)
+	go readLines(reader, lines)
+
+	for line := range lines {
+		s, err := decodeServiceTimeStrings(line)
+		if err != nil {
+			log.Println(err)
+		} else {
+			serviceTimes = append(serviceTimes, s)
+		}
+	}
+
+	return serviceTimes, err
+}
+
 func connect(hostname string, port int) (net.Conn, error) {
 	return net.Dial("tcp", fmt.Sprintf("%s:%d", hostname, port))
 }
@@ -132,5 +174,34 @@ func decodeCounterStrings(line string) (types.Counter, error) {
 		}
 	}
 
-	return types.Counter{}, errors.New("could not parse line: " + line)
+	return types.Counter{}, errors.New("counter - could not parse line: " + line)
+}
+
+func decodeServiceTimeStrings(line string) (types.Counter, error) {
+	if equal := strings.Index(line, ":"); equal >= 0 {
+		if key := strings.TrimSpace(line[:equal]); len(key) > 0 {
+			value := ""
+			if len(line) > equal {
+				value = strings.TrimSpace(line[equal+1:])
+			}
+			key = strings.Replace(key, " ", "_", -1)
+			key = strings.Replace(key, "(", "", -1)
+			key = strings.Replace(key, ")", "", -1)
+
+			if equalTwo := strings.Index(value, "%"); equalTwo >= 0 {
+				if keyTwo := strings.TrimSpace(value[:equalTwo]); len(keyTwo) > 0 {
+					if len(value) > equalTwo {
+						value = strings.Split(strings.TrimSpace(value[equalTwo+1:]), " ")[0]
+					}
+					key = key + "_" + keyTwo
+				}
+			}
+
+			if value, err := strconv.ParseFloat(value, 64); err == nil {
+				return types.Counter{Key: key, Value: value}, nil
+			}
+		}
+	}
+
+	return types.Counter{}, errors.New("servicec times - could not parse line: " + line)
 }

collector/metrics.go

@@ -16,7 +16,8 @@ const (
 )
 
 var (
-	counters descMap
+	counters     descMap
+	serviceTimes descMap
 )
 
 /*Exporter entry point to squid exporter */
@@ -33,6 +34,7 @@ type Exporter struct {
 /*New initializes a new exporter */
 func New(hostname string, port int, login string, password string, labels config.Labels) *Exporter {
 	counters = generateSquidCounters(labels.Keys)
+	serviceTimes = generateSquidServiceTimes(labels.Keys)
 	c := NewCacheObjectClient(hostname, port, login, password)
 
 	return &Exporter{
@@ -59,6 +61,10 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) {
 		ch <- v
 	}
 
+	for _, v := range serviceTimes {
+		ch <- v
+	}
+
 }
 
 /*Collect fetches metrics from squid manager and pushes them to promethus */
@@ -74,7 +80,20 @@ func (e *Exporter) Collect(c chan<- prometheus.Metric) {
 		}
 	} else {
 		e.up.With(prometheus.Labels{"host": e.hostname}).Set(0)
-		log.Println("Could not fetch metrics from squid instance: ", err)
+		log.Println("Could not fetch counter metrics from squid instance: ", err)
 	}
+
+	insts, err = e.client.GetServiceTimes()
+
+	if err == nil {
+		for i := range insts {
+			if d, ok := serviceTimes[insts[i].Key]; ok {
+				c <- prometheus.MustNewConstMetric(d, prometheus.CounterValue, insts[i].Value, e.labels.Values...)
+			}
+		}
+	} else {
+		log.Println("Could not fetch service times metrics from squid instance: ", err)
+	}
+
 	e.up.Collect(c)
 }

collector/service_times.go

@@ -0,0 +1,143 @@
+package collector
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+type squidServiceTimes struct {
+	Section     string
+	Counter     string
+	Suffix      string
+	Description string
+}
+
+var squidServiceTimess = []squidServiceTimes{
+	{"HTTP_Requests", "All", "5", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "10", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "15", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "20", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "25", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "30", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "35", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "40", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "45", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "50", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "55", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "60", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "65", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "70", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "75", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "80", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "85", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "90", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "95", "Service Time Percentiles 5min"},
+	{"HTTP_Requests", "All", "100", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "5", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "10", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "15", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "20", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "25", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "30", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "35", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "40", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "45", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "50", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "55", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "60", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "65", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "70", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "75", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "80", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "85", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "90", "Service Time Percentiles 5min"},
+	{"Cache_Misses", "", "95", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "5", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "10", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "15", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "20", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "25", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "30", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "35", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "40", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "45", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "50", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "55", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "60", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "65", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "70", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "75", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "80", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "85", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "90", "Service Time Percentiles 5min"},
+	{"Cache_Hits", "", "95", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "5", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "10", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "15", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "20", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "25", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "30", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "35", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "40", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "45", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "50", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "55", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "60", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "65", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "70", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "75", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "80", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "85", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "90", "Service Time Percentiles 5min"},
+	{"Near_Hits", "", "95", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "5", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "10", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "15", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "20", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "25", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "30", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "35", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "40", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "45", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "50", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "55", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "60", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "65", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "70", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "75", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "80", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "85", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "90", "Service Time Percentiles 5min"},
+	{"DNS_Lookups", "", "95", "Service Time Percentiles 5min"},
+}
+
+func generateSquidServiceTimes(labels []string) descMap {
+	serviceTimes := descMap{}
+
+	for i := range squidServiceTimess {
+		serviceTime := squidServiceTimess[i]
+
+		var key string
+		var name string
+
+		if serviceTime.Counter != "" {
+			key = fmt.Sprintf("%s_%s_%s", serviceTime.Section, serviceTime.Counter, serviceTime.Suffix)
+			name = prometheus.BuildFQName(namespace, strings.Replace(serviceTime.Section, ".", "_", -1),
+				fmt.Sprintf("%s_%s", serviceTime.Counter, serviceTime.Suffix))
+		} else {
+			key = fmt.Sprintf("%s_%s", serviceTime.Section, serviceTime.Suffix)
+			name = prometheus.BuildFQName(namespace, strings.Replace(serviceTime.Section, ".", "_", -1),
+				fmt.Sprintf("%s", serviceTime.Suffix))
+		}
+
+		serviceTimes[key] = prometheus.NewDesc(
+			name,
+			serviceTime.Description,
+			labels, nil,
+		)
+	}
+
+	return serviceTimes
+}