From 41763754804a36700cbf720e6e6321390fcc26d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20de=20Metz?= Date: Tue, 28 Jun 2016 19:26:41 +0200 Subject: [PATCH 1/8] Add basic filestack webhook. --- .../webhooks/filestack/filestack_webhooks.go | 44 ++++++++++++++++++ .../filestack/filestack_webhooks_events.go | 21 +++++++++ .../filestack_webhooks_events_json_test.go | 41 +++++++++++++++++ .../filestack/filestack_webhooks_test.go | 46 +++++++++++++++++++ plugins/inputs/webhooks/webhooks.go | 8 ++-- 5 files changed, 157 insertions(+), 3 deletions(-) create mode 100644 plugins/inputs/webhooks/filestack/filestack_webhooks.go create mode 100644 plugins/inputs/webhooks/filestack/filestack_webhooks_events.go create mode 100644 plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go create mode 100644 plugins/inputs/webhooks/filestack/filestack_webhooks_test.go diff --git a/plugins/inputs/webhooks/filestack/filestack_webhooks.go b/plugins/inputs/webhooks/filestack/filestack_webhooks.go new file mode 100644 index 0000000000000..38f05eab066fa --- /dev/null +++ b/plugins/inputs/webhooks/filestack/filestack_webhooks.go @@ -0,0 +1,44 @@ +package filestack + +import ( + "encoding/json" + "io/ioutil" + "log" + "net/http" + "time" + + "github.com/gorilla/mux" + "github.com/influxdata/telegraf" +) + +type FilestackWebhook struct { + Path string + acc telegraf.Accumulator +} + +func (fs *FilestackWebhook) Register(router *mux.Router, acc telegraf.Accumulator) { + router.HandleFunc(fs.Path, fs.eventHandler).Methods("POST") + + log.Printf("Started the webhooks_filestack on %s\n", fs.Path) + fs.acc = acc +} + +func (fs *FilestackWebhook) eventHandler(w http.ResponseWriter, r *http.Request) { + defer r.Body.Close() + body, err := ioutil.ReadAll(r.Body) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + + event := &DialogEvent{} + err = json.Unmarshal(body, event) + if err != nil { + w.WriteHeader(http.StatusBadRequest) + return + } + + fs.acc.AddFields("filestack_webhooks", event.Fields(), event.Tags(), time.Unix(event.TimeStamp, 0)) + + w.WriteHeader(http.StatusOK) +} diff --git a/plugins/inputs/webhooks/filestack/filestack_webhooks_events.go b/plugins/inputs/webhooks/filestack/filestack_webhooks_events.go new file mode 100644 index 0000000000000..06b7db9583082 --- /dev/null +++ b/plugins/inputs/webhooks/filestack/filestack_webhooks_events.go @@ -0,0 +1,21 @@ +package filestack + +import "strconv" + +type DialogEvent struct { + Action string `json:"action"` + TimeStamp int64 `json:"timestamp"` + Id int `json:"id"` +} + +func (de *DialogEvent) Tags() map[string]string { + return map[string]string{ + "action": de.Action, + } +} + +func (de *DialogEvent) Fields() map[string]interface{} { + return map[string]interface{}{ + "id": strconv.Itoa(de.Id), + } +} diff --git a/plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go b/plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go new file mode 100644 index 0000000000000..1af79c2055c54 --- /dev/null +++ b/plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go @@ -0,0 +1,41 @@ +package filestack + +func DialogOpenJSON() string { + return `{ + "action": "fp.dialog", + "timestamp": 1435584646, + "id": 102, + "text": { + "mimetypes": ["*/*"], + "iframe": false, + "language": "en", + "id": "1435584650723", + "mobile": false, + "app":{ + "upsell": "false", + "apikey": "YOUR_API_KEY", + "customization":{ + "saveas_subheader": "Save it down to your local device or onto the Cloud", + "folder_subheader": "Choose a folder to share with this application", + "open_subheader": "Choose from the files on your local device or the ones you have online", + "folder_header": "Select a folder", + "help_text": "", + "saveas_header": "Save your file", + "open_header": "Upload a file" + } + }, + "dialogType": "open", + "auth": false, + "welcome_header": "Upload a file", + "welcome_subheader": "Choose from the files on your local device or the ones you have online", + "help_text": "", + "recent_path": "/", + "extensions": null, + "maxSize": 0, + "signature": null, + "policy": null, + "custom_providers": "imgur,cloudapp", + "intra": false + } + }` +} diff --git a/plugins/inputs/webhooks/filestack/filestack_webhooks_test.go b/plugins/inputs/webhooks/filestack/filestack_webhooks_test.go new file mode 100644 index 0000000000000..47782b572c0fa --- /dev/null +++ b/plugins/inputs/webhooks/filestack/filestack_webhooks_test.go @@ -0,0 +1,46 @@ +package filestack + +import ( + "github.com/influxdata/telegraf/testutil" + "net/http" + "net/http/httptest" + + "strings" + "testing" +) + +func postWebhooks(md *FilestackWebhook, eventBody string) *httptest.ResponseRecorder { + req, _ := http.NewRequest("POST", "/filestack", strings.NewReader(eventBody)) + w := httptest.NewRecorder() + + md.eventHandler(w, req) + + return w +} + +func TestDialogEvent(t *testing.T) { + var acc testutil.Accumulator + fs := &FilestackWebhook{Path: "/filestack", acc: &acc} + resp := postWebhooks(fs, DialogOpenJSON()) + if resp.Code != http.StatusOK { + t.Errorf("POST returned HTTP status code %v.\nExpected %v", resp.Code, http.StatusOK) + } + + fields := map[string]interface{}{ + "id": "102", + } + + tags := map[string]string{ + "action": "fp.dialog", + } + + acc.AssertContainsTaggedFields(t, "filestack_webhooks", fields, tags) +} + +func TestParseError(t *testing.T) { + fs := &FilestackWebhook{Path: "/filestack"} + resp := postWebhooks(fs, "") + if resp.Code != http.StatusBadRequest { + t.Errorf("POST returned HTTP status code %v.\nExpected %v", resp.Code, http.StatusBadRequest) + } +} diff --git a/plugins/inputs/webhooks/webhooks.go b/plugins/inputs/webhooks/webhooks.go index 884435c36c0a6..1bbd9f4f8fe6e 100644 --- a/plugins/inputs/webhooks/webhooks.go +++ b/plugins/inputs/webhooks/webhooks.go @@ -10,6 +10,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" + "github.com/influxdata/telegraf/plugins/inputs/webhooks/filestack" "github.com/influxdata/telegraf/plugins/inputs/webhooks/github" "github.com/influxdata/telegraf/plugins/inputs/webhooks/mandrill" "github.com/influxdata/telegraf/plugins/inputs/webhooks/rollbar" @@ -26,9 +27,10 @@ func init() { type Webhooks struct { ServiceAddress string - Github *github.GithubWebhook - Mandrill *mandrill.MandrillWebhook - Rollbar *rollbar.RollbarWebhook + Github *github.GithubWebhook + Filestack *filestack.FilestackWebhook + Mandrill *mandrill.MandrillWebhook + Rollbar *rollbar.RollbarWebhook } func NewWebhooks() *Webhooks { From 75be330318b420bf3a4f4994af5891f5095f0df3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20de=20Metz?= Date: Tue, 28 Jun 2016 19:32:41 +0200 Subject: [PATCH 2/8] Generalize event. --- .../webhooks/filestack/filestack_webhooks.go | 2 +- .../filestack/filestack_webhooks_events.go | 10 +++++----- .../filestack_webhooks_events_json_test.go | 15 +++++++++++++++ .../filestack/filestack_webhooks_test.go | 19 +++++++++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/plugins/inputs/webhooks/filestack/filestack_webhooks.go b/plugins/inputs/webhooks/filestack/filestack_webhooks.go index 38f05eab066fa..623737670c698 100644 --- a/plugins/inputs/webhooks/filestack/filestack_webhooks.go +++ b/plugins/inputs/webhooks/filestack/filestack_webhooks.go @@ -31,7 +31,7 @@ func (fs *FilestackWebhook) eventHandler(w http.ResponseWriter, r *http.Request) return } - event := &DialogEvent{} + event := &FilestackEvent{} err = json.Unmarshal(body, event) if err != nil { w.WriteHeader(http.StatusBadRequest) diff --git a/plugins/inputs/webhooks/filestack/filestack_webhooks_events.go b/plugins/inputs/webhooks/filestack/filestack_webhooks_events.go index 06b7db9583082..93f976f6074be 100644 --- a/plugins/inputs/webhooks/filestack/filestack_webhooks_events.go +++ b/plugins/inputs/webhooks/filestack/filestack_webhooks_events.go @@ -2,20 +2,20 @@ package filestack import "strconv" -type DialogEvent struct { +type FilestackEvent struct { Action string `json:"action"` TimeStamp int64 `json:"timestamp"` Id int `json:"id"` } -func (de *DialogEvent) Tags() map[string]string { +func (fe *FilestackEvent) Tags() map[string]string { return map[string]string{ - "action": de.Action, + "action": fe.Action, } } -func (de *DialogEvent) Fields() map[string]interface{} { +func (fe *FilestackEvent) Fields() map[string]interface{} { return map[string]interface{}{ - "id": strconv.Itoa(de.Id), + "id": strconv.Itoa(fe.Id), } } diff --git a/plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go b/plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go index 1af79c2055c54..8243fe04a70ea 100644 --- a/plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go +++ b/plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go @@ -39,3 +39,18 @@ func DialogOpenJSON() string { } }` } + +func UploadJSON() string { + return `{ + "action":"fp.upload", + "timestamp":1443444905, + "id":100946, + "text":{ + "url":"https://www.filestackapi.com/api/file/WAunDTTqQfCNWwUUyf6n", + "client":"Facebook", + "type":"image/jpeg", + "filename":"1579337399020824.jpg", + "size":139154 + } + }` +} diff --git a/plugins/inputs/webhooks/filestack/filestack_webhooks_test.go b/plugins/inputs/webhooks/filestack/filestack_webhooks_test.go index 47782b572c0fa..339f63a67d2cb 100644 --- a/plugins/inputs/webhooks/filestack/filestack_webhooks_test.go +++ b/plugins/inputs/webhooks/filestack/filestack_webhooks_test.go @@ -44,3 +44,22 @@ func TestParseError(t *testing.T) { t.Errorf("POST returned HTTP status code %v.\nExpected %v", resp.Code, http.StatusBadRequest) } } + +func TestUploadEvent(t *testing.T) { + var acc testutil.Accumulator + fs := &FilestackWebhook{Path: "/filestack", acc: &acc} + resp := postWebhooks(fs, UploadJSON()) + if resp.Code != http.StatusOK { + t.Errorf("POST returned HTTP status code %v.\nExpected %v", resp.Code, http.StatusOK) + } + + fields := map[string]interface{}{ + "id": "100946", + } + + tags := map[string]string{ + "action": "fp.upload", + } + + acc.AssertContainsTaggedFields(t, "filestack_webhooks", fields, tags) +} From 932e7d16a44f05a59d636eb3b8497338a461676f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20de=20Metz?= Date: Tue, 28 Jun 2016 19:36:48 +0200 Subject: [PATCH 3/8] Add doc. --- plugins/inputs/webhooks/README.md | 1 + plugins/inputs/webhooks/filestack/README.md | 15 +++++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 plugins/inputs/webhooks/filestack/README.md diff --git a/plugins/inputs/webhooks/README.md b/plugins/inputs/webhooks/README.md index 86e6685b88c58..bc7714e9e5b38 100644 --- a/plugins/inputs/webhooks/README.md +++ b/plugins/inputs/webhooks/README.md @@ -15,6 +15,7 @@ $ sudo service telegraf start ## Available webhooks +- [Filestack](filestack/) - [Github](github/) - [Mandrill](mandrill/) - [Rollbar](rollbar/) diff --git a/plugins/inputs/webhooks/filestack/README.md b/plugins/inputs/webhooks/filestack/README.md new file mode 100644 index 0000000000000..56bf5bb77764f --- /dev/null +++ b/plugins/inputs/webhooks/filestack/README.md @@ -0,0 +1,15 @@ +# Filestack webhook + +You should configure your Filestack's Webhooks to point at the `webhooks` service. To do this go to `filestack.com/`, select your app and click `Credentials > Webhooks`. In the resulting page, set the `URL` to `http://:1619/filestack`, and click on `Add`. + +## Events + +See the [webhook doc](https://www.filestack.com/docs/webhooks). + +All events for logs the original timestamp, the event name and the id. + +**Tags:** +* 'action' = `event.action` string + +**Fields:** +* 'id' = `event.id` string From 8dd75f537771301e3ba6a9505dfb0b564aacc353 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20de=20Metz?= Date: Tue, 28 Jun 2016 19:38:12 +0200 Subject: [PATCH 4/8] Update default config. --- plugins/inputs/webhooks/webhooks.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plugins/inputs/webhooks/webhooks.go b/plugins/inputs/webhooks/webhooks.go index 1bbd9f4f8fe6e..592656a141140 100644 --- a/plugins/inputs/webhooks/webhooks.go +++ b/plugins/inputs/webhooks/webhooks.go @@ -42,6 +42,9 @@ func (wb *Webhooks) SampleConfig() string { ## Address and port to host Webhook listener on service_address = ":1619" + [inputs.webhooks.filestack] + path = "/filestack" + [inputs.webhooks.github] path = "/github" From 8376c2d9ea2619c55ef789a101c4d9cb10dc3ee5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20de=20Metz?= Date: Tue, 28 Jun 2016 19:39:31 +0200 Subject: [PATCH 5/8] Add filestack to the list of plugins. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d0b1b870d4279..8b7b9d594cf84 100644 --- a/README.md +++ b/README.md @@ -220,6 +220,7 @@ Telegraf can also collect metrics via the following service plugins: * [kafka_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/kafka_consumer) * [nats_consumer](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/nats_consumer) * [webhooks](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks) + * [filestack](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/filestack) * [github](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/github) * [mandrill](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/mandrill) * [rollbar](https://github.com/influxdata/telegraf/tree/master/plugins/inputs/webhooks/rollbar) From 47dfdd10583203f28018e57761b48b6f5cbe037a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20de=20Metz?= Date: Wed, 29 Jun 2016 11:19:46 +0200 Subject: [PATCH 6/8] Check that video conversion event returns 400. --- .../filestack_webhooks_events_json_test.go | 54 +++++++++++++++++++ .../filestack/filestack_webhooks_test.go | 9 ++++ 2 files changed, 63 insertions(+) diff --git a/plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go b/plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go index 8243fe04a70ea..351de01b4d153 100644 --- a/plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go +++ b/plugins/inputs/webhooks/filestack/filestack_webhooks_events_json_test.go @@ -54,3 +54,57 @@ func UploadJSON() string { } }` } + +func VideoConversionJSON() string { + return `{ + "status":"completed", + "message":"Done", + "data":{ + "thumb":"https://cdn.filestackcontent.com/f1e8V88QDuxzOvtOAq1W", + "thumb100x100":"https://process.filestackapi.com/AhTgLagciQByzXpFGRI0Az/resize=w:100,h:100,f:crop/output=f:jpg,q:66/https://cdn.filestackcontent.com/f1e8V88QDuxzOvtOAq1W", + "thumb200x200":"https://process.filestackapi.com/AhTgLagciQByzXpFGRI0Az/resize=w:200,h:200,f:crop/output=f:jpg,q:66/https://cdn.filestackcontent.com/f1e8V88QDuxzOvtOAq1W", + "thumb300x300":"https://process.filestackapi.com/AhTgLagciQByzXpFGRI0Az/resize=w:300,h:300,f:crop/output=f:jpg,q:66/https://cdn.filestackcontent.com/f1e8V88QDuxzOvtOAq1W", + "url":"https://cdn.filestackcontent.com/VgvFVdvvTkml0WXPIoGn" + }, + "metadata":{ + "result":{ + "audio_channels":2, + "audio_codec":"vorbis", + "audio_sample_rate":44100, + "created_at":"2015/12/21 20:45:19 +0000", + "duration":10587, + "encoding_progress":100, + "encoding_time":8, + "extname":".webm", + "file_size":293459, + "fps":24, + "height":260, + "mime_type":"video/webm", + "started_encoding_at":"2015/12/21 20:45:22 +0000", + "updated_at":"2015/12/21 20:45:32 +0000", + "video_bitrate":221, + "video_codec":"vp8", + "width":300 + }, + "source":{ + "audio_bitrate":125, + "audio_channels":2, + "audio_codec":"aac", + "audio_sample_rate":44100, + "created_at":"2015/12/21 20:45:19 +0000", + "duration":10564, + "extname":".mp4", + "file_size":875797, + "fps":24, + "height":360, + "mime_type":"video/mp4", + "updated_at":"2015/12/21 20:45:32 +0000", + "video_bitrate":196, + "video_codec":"h264", + "width":480 + } + }, + "timestamp":"1453850583", + "uuid":"638311d89d2bc849563a674a45809b7c" + }` +} diff --git a/plugins/inputs/webhooks/filestack/filestack_webhooks_test.go b/plugins/inputs/webhooks/filestack/filestack_webhooks_test.go index 339f63a67d2cb..13f69e5cbb13d 100644 --- a/plugins/inputs/webhooks/filestack/filestack_webhooks_test.go +++ b/plugins/inputs/webhooks/filestack/filestack_webhooks_test.go @@ -63,3 +63,12 @@ func TestUploadEvent(t *testing.T) { acc.AssertContainsTaggedFields(t, "filestack_webhooks", fields, tags) } + +func TestVideoConversionEvent(t *testing.T) { + var acc testutil.Accumulator + fs := &FilestackWebhook{Path: "/filestack", acc: &acc} + resp := postWebhooks(fs, VideoConversionJSON()) + if resp.Code != http.StatusBadRequest { + t.Errorf("POST returned HTTP status code %v.\nExpected %v", resp.Code, http.StatusBadRequest) + } +} From d106b184541510248879415ce62fc90035485730 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20de=20Metz?= Date: Wed, 29 Jun 2016 11:20:50 +0200 Subject: [PATCH 7/8] Update the readme. --- plugins/inputs/webhooks/filestack/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plugins/inputs/webhooks/filestack/README.md b/plugins/inputs/webhooks/filestack/README.md index 56bf5bb77764f..585e6f202467b 100644 --- a/plugins/inputs/webhooks/filestack/README.md +++ b/plugins/inputs/webhooks/filestack/README.md @@ -6,7 +6,9 @@ You should configure your Filestack's Webhooks to point at the `webhooks` servic See the [webhook doc](https://www.filestack.com/docs/webhooks). -All events for logs the original timestamp, the event name and the id. +*Limitations*: It stores all events except video conversions events. + +All events for logs the original timestamp, the action and the id. **Tags:** * 'action' = `event.action` string From 1630d5c694eb20957d5cb14db963bbe1ac5294d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20de=20Metz?= Date: Sun, 24 Jul 2016 18:37:58 +0200 Subject: [PATCH 8/8] Update the changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 665add6ca0339..249933c28f2d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ - [#1513](https://github.com/influxdata/telegraf/issues/1513): Add Ceph Cluster Performance Statistics - [#1650](https://github.com/influxdata/telegraf/issues/1650): Ability to configure response_timeout in httpjson input. - [#1685](https://github.com/influxdata/telegraf/issues/1685): Add additional redis metrics. +- [#1542](https://github.com/influxdata/telegraf/pull/1542): Add filestack webhook plugin. ### Bugfixes