diff --git a/metricbeat/docker-compose.yml b/metricbeat/docker-compose.yml index 98788275a6ca..ce9b1e2e6f39 100644 --- a/metricbeat/docker-compose.yml +++ b/metricbeat/docker-compose.yml @@ -5,11 +5,12 @@ services: depends_on: - apache - couchbase - - mongodb - haproxy - kafka + - mongodb - mysql - nginx + - phpfpm - postgresql - prometheus - redis @@ -33,6 +34,8 @@ services: - MYSQL_DSN=root:test@tcp(mysql:3306)/ - MYSQL_HOST=mysql - MYSQL_PORT=3306 + - PHPFPM_HOST=phpfpm + - PHPFPM_PORT=81 - POSTGRESQL_DSN=postgres://postgresql:5432?sslmode=disable - POSTGRESQL_HOST=postgresql - POSTGRESQL_PORT=5432 @@ -78,6 +81,9 @@ services: haproxy: build: ${PWD}/module/haproxy/_meta + phpfpm: + build: ${PWD}/module/php_fpm/_meta + postgresql: image: postgres:9.5.3 diff --git a/metricbeat/docker-entrypoint.sh b/metricbeat/docker-entrypoint.sh index 25087fc7fa36..0f623b44b007 100755 --- a/metricbeat/docker-entrypoint.sh +++ b/metricbeat/docker-entrypoint.sh @@ -26,8 +26,10 @@ waitFor ${APACHE_HOST} ${APACHE_PORT} Apache waitFor ${COUCHBASE_HOST} ${COUCHBASE_PORT} Couchbase waitFor ${HAPROXY_HOST} ${HAPROXY_PORT} HAProxy waitFor ${KAFKA_HOST} ${KAFKA_PORT} Kafka +waitFor ${MONGODB_HOST} ${MONGODB_PORT} MongoDB waitFor ${MYSQL_HOST} ${MYSQL_PORT} MySQL waitFor ${NGINX_HOST} ${NGINX_PORT} Nginx +waitFor ${PHPFPM_HOST} ${PHPFPM_PORT} PHP_FPM waitFor ${POSTGRESQL_HOST} ${POSTGRESQL_PORT} Postgresql waitFor ${PROMETHEUS_HOST} ${PROMETHEUS_PORT} Prometheus waitFor ${REDIS_HOST} ${REDIS_PORT} Redis diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 72222092b270..59865e61a660 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -3826,7 +3826,7 @@ experimental[] [float] -=== php_fpm.pool.pool +=== php_fpm.pool.name type: keyword diff --git a/metricbeat/metricbeat.template-es2x.json b/metricbeat/metricbeat.template-es2x.json index 67d90128a557..3c50d6f915c6 100644 --- a/metricbeat/metricbeat.template-es2x.json +++ b/metricbeat/metricbeat.template-es2x.json @@ -2182,7 +2182,7 @@ } } }, - "pool": { + "name": { "ignore_above": 1024, "index": "not_analyzed", "type": "string" diff --git a/metricbeat/metricbeat.template.json b/metricbeat/metricbeat.template.json index 5d4b27fb5153..2d2182ffc29b 100644 --- a/metricbeat/metricbeat.template.json +++ b/metricbeat/metricbeat.template.json @@ -2161,7 +2161,7 @@ } } }, - "pool": { + "name": { "ignore_above": 1024, "type": "keyword" }, diff --git a/metricbeat/module/php_fpm/_meta/Dockerfile b/metricbeat/module/php_fpm/_meta/Dockerfile index 7e70cdb2b8ad..783a3db96dcf 100644 --- a/metricbeat/module/php_fpm/_meta/Dockerfile +++ b/metricbeat/module/php_fpm/_meta/Dockerfile @@ -2,3 +2,5 @@ FROM richarvey/nginx-php-fpm RUN echo "pm.status_path = /status" >> /etc/php7/php-fpm.d/www.conf ADD ./php-fpm.conf /etc/nginx/sites-enabled + +EXPOSE 81 diff --git a/metricbeat/module/php_fpm/php_fpm.go b/metricbeat/module/php_fpm/php_fpm.go deleted file mode 100644 index e20d352e3dba..000000000000 --- a/metricbeat/module/php_fpm/php_fpm.go +++ /dev/null @@ -1,59 +0,0 @@ -package php_fpm - -import ( - "fmt" - "io" - "net/http" - - "github.com/elastic/beats/metricbeat/mb" - "github.com/elastic/beats/metricbeat/mb/parse" -) - -const ( - defaultScheme = "http" - defaultPath = "/status" -) - -// HostParser is used for parsing the configured php-fpm hosts. -var HostParser = parse.URLHostParserBuilder{ - DefaultScheme: defaultScheme, - DefaultPath: defaultPath, - QueryParams: "json", - PathConfigKey: "status_path", -}.Build() - -// StatsClient provides access to php-fpm stats api -type StatsClient struct { - address string - user string - password string - http *http.Client -} - -// NewStatsClient creates a new StatsClient -func NewStatsClient(m mb.BaseMetricSet) *StatsClient { - return &StatsClient{ - address: m.HostData().SanitizedURI, - user: m.HostData().User, - password: m.HostData().Password, - http: &http.Client{Timeout: m.Module().Config().Timeout}, - } -} - -// Fetch php-fpm stats -func (c *StatsClient) Fetch() (io.ReadCloser, error) { - req, err := http.NewRequest("GET", c.address, nil) - if c.user != "" || c.password != "" { - req.SetBasicAuth(c.user, c.password) - } - resp, err := c.http.Do(req) - if err != nil { - return nil, fmt.Errorf("error making http request: %v", err) - } - - if resp.StatusCode != 200 { - return nil, fmt.Errorf("HTTP error %d: %s", resp.StatusCode, resp.Status) - } - - return resp.Body, nil -} diff --git a/metricbeat/module/php_fpm/php_fpm_test.go b/metricbeat/module/php_fpm/php_fpm_test.go deleted file mode 100644 index e5e6fe1342d3..000000000000 --- a/metricbeat/module/php_fpm/php_fpm_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package php_fpm - -import ( - "testing" - - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - - "github.com/stretchr/testify/assert" -) - -func TestHostParser(t *testing.T) { - tests := []struct { - host, expected string - }{ - {"localhost", "http://localhost/status?json="}, - {"localhost:123", "http://localhost:123/status?json="}, - {"http://localhost:123", "http://localhost:123/status?json="}, - } - - m := mbtest.NewTestModule(t, map[string]interface{}{}) - - for _, test := range tests { - hi, err := HostParser(m, test.host) - if err != nil { - t.Error("failed on", test.host, err) - continue - } - assert.Equal(t, test.expected, hi.URI) - } -} diff --git a/metricbeat/module/php_fpm/pool/_meta/data.json b/metricbeat/module/php_fpm/pool/_meta/data.json index 06cb28526330..8616a772903d 100644 --- a/metricbeat/module/php_fpm/pool/_meta/data.json +++ b/metricbeat/module/php_fpm/pool/_meta/data.json @@ -1,25 +1,28 @@ { - "@timestamp": "2017-01-18T23:57:23.960Z", - "beat": { - "hostname": "host.example.com", - "name": "host.example.com" - }, - "metricset": { - "host": "localhost:8081", - "module": "php_fpm", - "name": "pool", - "rtt": 1237 - }, - "php_fpm": { - "pool": { - "connections.accepted": 803, - "connections.queued": 0, - "hostname": "localhost:8081", - "pool": "www", - "processes.active": 1, - "processes.idle": 2, - "requests.slow": 0 - } - }, - "type": "metricsets" -} + "@timestamp": "2016-05-23T08:05:34.853Z", + "beat": { + "hostname": "host.example.com", + "name": "host.example.com" + }, + "metricset": { + "host": "127.0.0.1:81", + "module": "php_fpm", + "name": "pool", + "rtt": 115 + }, + "php_fpm": { + "pool": { + "connections": { + "accepted": 13, + "queued": 0 + }, + "pool": "www", + "processes": { + "active": 1, + "idle": 2 + }, + "slow_requests": 0 + } + }, + "type": "metricsets" +} \ No newline at end of file diff --git a/metricbeat/module/php_fpm/pool/_meta/fields.yml b/metricbeat/module/php_fpm/pool/_meta/fields.yml index ba22858f91ed..c4e2a096b0b7 100644 --- a/metricbeat/module/php_fpm/pool/_meta/fields.yml +++ b/metricbeat/module/php_fpm/pool/_meta/fields.yml @@ -4,7 +4,7 @@ `pool` contains the metrics that were obtained from the PHP-FPM process pool. fields: - - name: pool + - name: name type: keyword description: > The name of the pool. diff --git a/metricbeat/module/php_fpm/pool/data.go b/metricbeat/module/php_fpm/pool/data.go index 48a797f36cf1..7c6b1d888953 100644 --- a/metricbeat/module/php_fpm/pool/data.go +++ b/metricbeat/module/php_fpm/pool/data.go @@ -1,18 +1,21 @@ package pool -type poolStats struct { - Pool string `json:"pool"` - ProcessManager string `json:"process manager"` - StartTime int `json:"start time"` - StartSince int `json:"start since"` - AcceptedConn int `json:"accepted conn"` - ListenQueue int `json:"listen queue"` - MaxListenQueue int `json:"max listen queue"` - ListenQueueLen int `json:"listen queue len"` - IdleProcesses int `json:"idle processes"` - ActiveProcesses int `json:"active processes"` - TotalProcesses int `json:"total processes"` - MaxActiveProcesses int `json:"max active processes"` - MaxChildrenReached int `json:"max children reached"` - SlowRequests int `json:"slow requests"` -} +import ( + s "github.com/elastic/beats/metricbeat/schema" + c "github.com/elastic/beats/metricbeat/schema/mapstriface" +) + +var ( + schema = s.Schema{ + "name": c.Str("pool"), + "connections": s.Object{ + "accepted": c.Int("accepted conn"), + "queued": c.Int("listen queue"), + }, + "processes": s.Object{ + "idle": c.Int("idle processes"), + "active": c.Int("active processes"), + }, + "slow_requests": c.Int("slow requests"), + } +) diff --git a/metricbeat/module/php_fpm/pool/pool.go b/metricbeat/module/php_fpm/pool/pool.go index 982fe1e3899b..bb1ede66fe65 100644 --- a/metricbeat/module/php_fpm/pool/pool.go +++ b/metricbeat/module/php_fpm/pool/pool.go @@ -6,69 +6,58 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/beats/metricbeat/helper" "github.com/elastic/beats/metricbeat/mb" - - "github.com/elastic/beats/metricbeat/module/php_fpm" + "github.com/elastic/beats/metricbeat/mb/parse" ) // init registers the MetricSet with the central registry. -// The New method will be called after the setup of the module and before starting to fetch data func init() { - if err := mb.Registry.AddMetricSet("php_fpm", "pool", New, php_fpm.HostParser); err != nil { + if err := mb.Registry.AddMetricSet("php_fpm", "pool", New, HostParser); err != nil { panic(err) } } +const ( + defaultScheme = "http" + defaultPath = "/status" +) + +// HostParser is used for parsing the configured php-fpm hosts. +var HostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + QueryParams: "json", + PathConfigKey: "status_path", +}.Build() + // MetricSet type defines all fields of the MetricSet -// As a minimum it must inherit the mb.BaseMetricSet fields, but can be extended with -// additional entries. These variables can be used to persist data or configuration between -// multiple fetch calls. type MetricSet struct { mb.BaseMetricSet - client *php_fpm.StatsClient // StatsClient that is reused across requests. + *helper.HTTP } // New create a new instance of the MetricSet -// Part of new is also setting up the configuration by processing additional -// configuration entries if needed. func New(base mb.BaseMetricSet) (mb.MetricSet, error) { logp.Warn("EXPERIMENTAL: The php-fpm pool metricset is experimental") return &MetricSet{ - BaseMetricSet: base, - client: php_fpm.NewStatsClient(base), + base, + helper.NewHTTP(base), }, nil } -// Fetch methods implements the data gathering and data conversion to the right format -// It returns the event which is then forward to the output. In case of an error, a -// descriptive error must be returned. +// Fetch gathers data for the pool metricset func (m *MetricSet) Fetch() (common.MapStr, error) { - body, err := m.client.Fetch() - + content, err := m.HTTP.FetchContent() if err != nil { return nil, err } - defer body.Close() - - stats := &poolStats{} - err = json.NewDecoder(body).Decode(stats) + var stats map[string]interface{} + err = json.Unmarshal(content, &stats) if err != nil { return nil, fmt.Errorf("error parsing json: %v", err) } - return common.MapStr{ - "hostname": m.Host(), - - "pool": stats.Pool, - "connections": common.MapStr{ - "queue": stats.ListenQueue, - "accepted": stats.AcceptedConn, - }, - "processes": common.MapStr{ - "idle": stats.IdleProcesses, - "active": stats.ActiveProcesses, - }, - "slow_requests": stats.SlowRequests, - }, nil + return schema.Apply(stats), nil } diff --git a/metricbeat/module/php_fpm/pool/pool_integration_test.go b/metricbeat/module/php_fpm/pool/pool_integration_test.go index b5026cfa7d20..f914d316d4c7 100644 --- a/metricbeat/module/php_fpm/pool/pool_integration_test.go +++ b/metricbeat/module/php_fpm/pool/pool_integration_test.go @@ -3,6 +3,7 @@ package pool import ( + "os" "testing" mbtest "github.com/elastic/beats/metricbeat/mb/testing" @@ -20,6 +21,24 @@ func getConfig() map[string]interface{} { return map[string]interface{}{ "module": "php_fpm", "metricsets": []string{"pool"}, - "hosts": []string{"127.0.0.1:81"}, + "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, } } + +func GetEnvHost() string { + host := os.Getenv("PHPFPM_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +func GetEnvPort() string { + port := os.Getenv("PHPFPM_PORT") + + if len(port) == 0 { + port = "81" + } + return port +} diff --git a/metricbeat/tests/system/test_phpfpm.py b/metricbeat/tests/system/test_phpfpm.py new file mode 100644 index 000000000000..80a81c30ca80 --- /dev/null +++ b/metricbeat/tests/system/test_phpfpm.py @@ -0,0 +1,38 @@ +import os +import metricbeat +import unittest +from nose.plugins.attrib import attr + +PHPFPM_FIELDS = metricbeat.COMMON_FIELDS + ["php_fpm"] + +class Test(metricbeat.BaseTest): + @unittest.skipUnless(metricbeat.INTEGRATION_TESTS, "integration test") + def test_info(self): + """ + php_fpm pool metricset test + """ + self.render_config_template(modules=[{ + "name": "php_fpm", + "metricsets": ["pool"], + "hosts": self.get_hosts(), + "period": "5s" + }]) + proc = self.start_beat() + self.wait_until(lambda: self.output_lines() > 0) + proc.check_kill_and_wait() + + # Ensure no errors or warnings exist in the log. + log = self.get_log() + self.assertNotRegexpMatches(log.replace("WARN EXPERIMENTAL", ""), "ERR|WARN") + + output = self.read_output_json() + self.assertEqual(len(output), 1) + evt = output[0] + + self.assertItemsEqual(self.de_dot(PHPFPM_FIELDS), evt.keys(), evt) + + self.assert_fields_are_documented(evt) + + def get_hosts(self): + return [os.getenv('PHPFPM_HOST', 'localhost') + ':' + + os.getenv('PHPFPM_PORT', '81')]