From 0a5a2c933debb3e4207d24fd7ed35f485d9cbbda Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 8 Oct 2021 18:10:52 +0200 Subject: [PATCH 001/146] Throttle courier queues when the channel has rate limit redis key not expired --- handlers/whatsapp/whatsapp.go | 18 ++++++++++-- handlers/whatsapp/whatsapp_test.go | 6 ++++ queue/queue.go | 14 +++++++++ queue/queue_test.go | 46 ++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+), 2 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index e1fd4116d..651b960cd 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -12,6 +12,7 @@ import ( "time" "github.com/buger/jsonparser" + "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" @@ -495,6 +496,9 @@ const maxMsgLength = 4096 // SendMsg sends the passed in message, returning any error func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { start := time.Now() + conn := h.Backend().RedisPool().Get() + defer conn.Close() + // get our token token := msg.Channel().StringConfigForKey(courier.ConfigAuthToken, "") if token == "" { @@ -525,7 +529,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat for i, payload := range payloads { externalID := "" - wppID, externalID, logs, err = sendWhatsAppMsg(msg, sendPath, payload) + wppID, externalID, logs, err = sendWhatsAppMsg(conn, msg, sendPath, payload) // add logs to our status for _, log := range logs { status.AddLog(log) @@ -827,7 +831,7 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri return mediaID, logs, nil } -func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, payload interface{}) (string, string, []*courier.ChannelLog, error) { +func sendWhatsAppMsg(rc redis.Conn, msg courier.Msg, sendPath *url.URL, payload interface{}) (string, string, []*courier.ChannelLog, error) { start := time.Now() jsonBody, err := json.Marshal(payload) @@ -839,6 +843,16 @@ func sendWhatsAppMsg(msg courier.Msg, sendPath *url.URL, payload interface{}) (s req, _ := http.NewRequest(http.MethodPost, sendPath.String(), bytes.NewReader(jsonBody)) req.Header = buildWhatsAppHeaders(msg.Channel()) rr, err := utils.MakeHTTPRequest(req) + + if rr.StatusCode == 429 || rr.StatusCode == 503 { + rateLimitKey := fmt.Sprintf("rate_limit:%s", msg.Channel().UUID().String()) + rc.Do("set", rateLimitKey, "engaged") + rc.Do("expire", rateLimitKey, 2) + + log := courier.NewChannelLogFromRR("rate limit engaged", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) + return "", "", []*courier.ChannelLog{log}, err + } + log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) errPayload := &mtErrorPayload{} err = json.Unmarshal(rr.Body, errPayload) diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 40d940445..653a62051 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -388,6 +388,12 @@ var defaultSendTestCases = []ChannelSendTestCase{ ResponseBody: `{ "errors": [{ "title": "Error Sending" }] }`, ResponseStatus: 403, RequestBody: `{"to":"250788123123","type":"text","text":{"body":"Error"}}`, SendPrep: setSendURL}, + {Label: "Rate Limit Engaged", + Text: "Error", URN: "whatsapp:250788123123", + Status: "E", + ResponseBody: `{ "errors": [{ "title": "Too many requests" }] }`, ResponseStatus: 429, + RequestBody: `{"to":"250788123123","type":"text","text":{"body":"Error"}}`, + SendPrep: setSendURL}, {Label: "No Message ID", Text: "Error", URN: "whatsapp:250788123123", Status: "E", diff --git a/queue/queue.go b/queue/queue.go index 7aa4bebaa..da6b96e83 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -83,10 +83,24 @@ var luaPop = redis.NewScript(2, `-- KEYS: [EpochMS QueueType] local delim = string.find(queue, "|") local tps = 0 local tpsKey = "" + + local queueName = "" + if delim then + queueName = string.sub(queue, string.len(KEYS[2])+2, delim-1) tps = tonumber(string.sub(queue, delim+1)) end + if queueName then + local rateLimitKey = "rate_limit:" .. queueName + local rateLimitEngaged = redis.call("get", rateLimitKey) + if rateLimitEngaged then + redis.call("zincrby", KEYS[2] .. ":throttled", workers, queue) + redis.call("zrem", KEYS[2] .. ":active", queue) + return {"retry", ""} + end + end + -- if we have a tps, then check whether we exceed it if tps > 0 then tpsKey = queue .. ":tps:" .. math.floor(KEYS[1]) diff --git a/queue/queue_test.go b/queue/queue_test.go index 157793a0e..81cd415a3 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -50,6 +50,7 @@ func TestLua(t *testing.T) { defer close(quitter) rate := 10 + for i := 0; i < 20; i++ { err := PushOntoQueue(conn, "msgs", "chan1", rate, fmt.Sprintf(`[{"id":%d}]`, i), LowPriority) assert.NoError(err) @@ -166,6 +167,51 @@ func TestLua(t *testing.T) { assert.NoError(err) assert.Equal(EmptyQueue, queue) assert.Empty(value) + + err = PushOntoQueue(conn, "msgs", "chan1", rate, `[{"id":34}]`, HighPriority) + assert.NoError(err) + + conn.Do("set", "rate_limit:chan1", "engaged") + conn.Do("EXPIRE", "rate_limit:chan1", 5) + + // we have the rate limit set, + queue, value, err = PopFromQueue(conn, "msgs") + if value != "" && queue != EmptyQueue { + t.Fatal("Should be throttled") + } + + time.Sleep(2 * time.Second) + queue, value, err = PopFromQueue(conn, "msgs") + if value != "" && queue != EmptyQueue { + t.Fatal("Should be throttled") + } + + count, err = redis.Int(conn.Do("zcard", "msgs:throttled")) + assert.NoError(err) + assert.Equal(1, count, "Expected chan1 to be throttled") + + count, err = redis.Int(conn.Do("zcard", "msgs:active")) + assert.NoError(err) + assert.Equal(0, count, "Expected chan1 to not be active") + + // but if we wait for the rate limit to expire + time.Sleep(3 * time.Second) + + // next should be 34 + queue, value, err = PopFromQueue(conn, "msgs") + assert.NotEqual(queue, EmptyQueue) + assert.Equal(`{"id":34}`, value) + assert.NoError(err) + + // nothing should be left + queue = Retry + for queue == Retry { + queue, value, err = PopFromQueue(conn, "msgs") + } + assert.NoError(err) + assert.Equal(EmptyQueue, queue) + assert.Empty(value) + } func nTestThrottle(t *testing.T) { From d63ccf354fa1b5ffd92ce9d53df1d76ad695ff84 Mon Sep 17 00:00:00 2001 From: Rafael Soares Date: Thu, 28 Oct 2021 18:42:42 -0300 Subject: [PATCH 002/146] refactor external channel handler to use headers config on send --- channel.go | 3 +++ handlers/external/external.go | 9 +++++--- handlers/external/external_test.go | 34 +++++++++++++++--------------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/channel.go b/channel.go index d64394cca..7a71a40d1 100644 --- a/channel.go +++ b/channel.go @@ -52,6 +52,9 @@ const ( // ConfigUseNational is a constant key for channel configs ConfigUseNational = "use_national" + + // ConfigSendHeaders is a constant key for channel configs + ConfigSendHeaders = "headers" ) // ChannelType is our typing of the two char channel types diff --git a/handlers/external/external.go b/handlers/external/external.go index c447228d2..97f3f6c2b 100644 --- a/handlers/external/external.go +++ b/handlers/external/external.go @@ -353,9 +353,12 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } req.Header.Set("Content-Type", contentTypeHeader) - authorization := msg.Channel().StringConfigForKey(courier.ConfigSendAuthorization, "") - if authorization != "" { - req.Header.Set("Authorization", authorization) + headers := msg.Channel().ConfigForKey(courier.ConfigSendHeaders, map[string]interface{}{}).(map[string]interface{}) + + if len(headers) > 0 { + for hKey, hValue := range headers { + req.Header.Set(hKey, fmt.Sprint(hValue)) + } } rr, err := utils.MakeHTTPRequest(req) diff --git a/handlers/external/external_test.go b/handlers/external/external_test.go index 1c4b84685..bbed3d214 100644 --- a/handlers/external/external_test.go +++ b/handlers/external/external_test.go @@ -423,11 +423,11 @@ func TestSending(t *testing.T) { var jsonChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", map[string]interface{}{ - "send_path": "", - courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, - courier.ConfigContentType: contentJSON, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", + "send_path": "", + courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, + courier.ConfigContentType: contentJSON, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendHeaders: map[string]interface{}{"Authorization": "Token ABCDEF", "foo": "bar"}, }) var xmlChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", @@ -472,22 +472,22 @@ func TestSending(t *testing.T) { var jsonChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", map[string]interface{}{ - "send_path": "", - "max_length": 30, - courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, - courier.ConfigContentType: contentJSON, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", + "send_path": "", + "max_length": 30, + courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, + courier.ConfigContentType: contentJSON, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendHeaders: map[string]interface{}{"Authorization": "Token ABCDEF", "foo": "bar"}, }) var xmlChannel30IntLength = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", map[string]interface{}{ - "send_path": "", - "max_length": 30, - courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, - courier.ConfigContentType: contentXML, - courier.ConfigSendMethod: http.MethodPost, - courier.ConfigSendAuthorization: "Token ABCDEF", + "send_path": "", + "max_length": 30, + courier.ConfigSendBody: `{{to}}{{text}}{{from}}{{quick_replies}}`, + courier.ConfigContentType: contentXML, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendHeaders: map[string]interface{}{"Authorization": "Token ABCDEF", "foo": "bar"}, }) RunChannelSendTestCases(t, getChannel30IntLength, newHandler(), longSendTestCases, nil) From 71f1763092fa3c468d499a753ebbc110e6153e73 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 23 Nov 2021 08:10:47 -0500 Subject: [PATCH 003/146] Pin to go 1.17.2 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b6f2486b..d43f9d7f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: CI on: [push, pull_request] env: - go-version: '1.17.x' + go-version: '1.17.2' # https://github.com/golang/go/issues/49366 redis-version: '3.2.4' jobs: test: From 7466834b6755d72f67ac39d4e8cf6f922dbf578d Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 23 Nov 2021 09:32:36 -0500 Subject: [PATCH 004/146] Update CHANGELOG.md for v7.1.1 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index bf834618c..57302032d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.1 +---------- + * Pin to go 1.17.2 + v7.1.0 ---------- * Remove chatbase support From a1f5a5d2387e06c4f3ef9f17b383655ae3f17303 Mon Sep 17 00:00:00 2001 From: Rafael Soares Date: Tue, 23 Nov 2021 11:49:28 -0300 Subject: [PATCH 005/146] add support to old way to set authorization token header --- handlers/external/external.go | 5 +++++ handlers/external/external_test.go | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/handlers/external/external.go b/handlers/external/external.go index 97f3f6c2b..62ee86876 100644 --- a/handlers/external/external.go +++ b/handlers/external/external.go @@ -353,6 +353,11 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } req.Header.Set("Content-Type", contentTypeHeader) + authorization := msg.Channel().StringConfigForKey(courier.ConfigSendAuthorization, "") + if authorization != "" { + req.Header.Set("Authorization", authorization) + } + headers := msg.Channel().ConfigForKey(courier.ConfigSendHeaders, map[string]interface{}{}).(map[string]interface{}) if len(headers) > 0 { diff --git a/handlers/external/external_test.go b/handlers/external/external_test.go index bbed3d214..8a9343c68 100644 --- a/handlers/external/external_test.go +++ b/handlers/external/external_test.go @@ -503,4 +503,14 @@ func TestSending(t *testing.T) { RunChannelSendTestCases(t, nationalChannel, newHandler(), nationalGetSendTestCases, nil) + var jsonChannelWithSendAuthorization = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "EX", "2020", "US", + map[string]interface{}{ + "send_path": "", + courier.ConfigSendBody: `{ "to":{{to}}, "text":{{text}}, "from":{{from}}, "quick_replies":{{quick_replies}} }`, + courier.ConfigContentType: contentJSON, + courier.ConfigSendMethod: http.MethodPost, + courier.ConfigSendAuthorization: "Token ABCDEF", + }) + RunChannelSendTestCases(t, jsonChannelWithSendAuthorization, newHandler(), jsonSendTestCases, nil) + } From 538fdcf7f8f7ddd79e04c3027648b77fd5834180 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 23 Nov 2021 10:36:24 -0500 Subject: [PATCH 006/146] Add comment --- handlers/external/external.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/handlers/external/external.go b/handlers/external/external.go index 62ee86876..da01ecce2 100644 --- a/handlers/external/external.go +++ b/handlers/external/external.go @@ -353,17 +353,15 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } req.Header.Set("Content-Type", contentTypeHeader) + // TODO can drop this when channels have been migrated to use ConfigSendHeaders authorization := msg.Channel().StringConfigForKey(courier.ConfigSendAuthorization, "") if authorization != "" { req.Header.Set("Authorization", authorization) } headers := msg.Channel().ConfigForKey(courier.ConfigSendHeaders, map[string]interface{}{}).(map[string]interface{}) - - if len(headers) > 0 { - for hKey, hValue := range headers { - req.Header.Set(hKey, fmt.Sprint(hValue)) - } + for hKey, hValue := range headers { + req.Header.Set(hKey, fmt.Sprint(hValue)) } rr, err := utils.MakeHTTPRequest(req) From 34471015ea0db7aab0e30a490059c1e5154db5bf Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 23 Nov 2021 10:37:22 -0500 Subject: [PATCH 007/146] Update CHANGELOG.md for v7.1.2 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 57302032d..d7838a92a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.2 +---------- + * External channel handler should use headers config setting if provided + v7.1.1 ---------- * Pin to go 1.17.2 From 42f1110764d9403c5ff9d54ee305c89c046944f8 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 30 Nov 2021 16:38:14 -0500 Subject: [PATCH 008/146] Use response_to_external_id instead of response_to_id --- backends/rapidpro/backend.go | 2 +- backends/rapidpro/backend_test.go | 8 ++------ backends/rapidpro/msg.go | 2 -- handlers/chikka/chikka_test.go | 2 -- handlers/facebook/facebook.go | 2 +- handlers/facebook/facebook_test.go | 2 +- handlers/facebookapp/facebookapp.go | 2 +- handlers/facebookapp/facebookapp_test.go | 2 +- handlers/test.go | 3 +-- msg.go | 1 - test.go | 11 ++--------- 11 files changed, 10 insertions(+), 27 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index da6df33c6..0d362e471 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -238,7 +238,7 @@ func (b *backend) IsMsgLoop(ctx context.Context, msg courier.Msg) (bool, error) m := msg.(*DBMsg) // things that aren't replies can't be loops, neither do we count retries - if m.ResponseToID_ == courier.NilMsgID || m.ErrorCount_ > 0 { + if m.ResponseToExternalID_ == "" || m.ErrorCount_ > 0 { return false, nil } diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index 14982c8b1..b803f8e7b 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -120,7 +120,6 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { "sent_on": null, "high_priority": true, "channel_id": 11, - "response_to_id": 15, "response_to_external_id": "external-id", "external_id": null, "is_resend": true, @@ -137,7 +136,6 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { ts.Equal(msg.ExternalID(), "") ts.Equal([]string{"Yes", "No"}, msg.QuickReplies()) ts.Equal("event", msg.Topic()) - ts.Equal(courier.NewMsgID(15), msg.ResponseToID()) ts.Equal("external-id", msg.ResponseToExternalID()) ts.True(msg.HighPriority()) ts.True(msg.IsResend()) @@ -162,8 +160,7 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { "sent_on": null, "high_priority": true, "channel_id": 11, - "response_to_id": null, - "response_to_external_id": "", + "response_to_external_id": null, "external_id": null, "metadata": null }` @@ -173,7 +170,6 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { ts.NoError(err) ts.Equal([]string{}, msg.QuickReplies()) ts.Equal("", msg.Topic()) - ts.Equal(courier.NilMsgID, msg.ResponseToID()) ts.Equal("", msg.ResponseToExternalID()) ts.False(msg.IsResend()) } @@ -756,7 +752,7 @@ func (ts *BackendTestSuite) TestLoop() { ctx := context.Background() dbMsg := readMsgFromDB(ts.b, courier.NewMsgID(10000)) - dbMsg.ResponseToID_ = courier.MsgID(5) + dbMsg.ResponseToExternalID_ = "65474" loop, err := ts.b.IsMsgLoop(ctx, dbMsg) ts.NoError(err) diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index 6f7ef5dfd..020071f84 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -489,7 +489,6 @@ type DBMsg struct { Text_ string `json:"text" db:"text"` Attachments_ pq.StringArray `json:"attachments" db:"attachments"` ExternalID_ null.String `json:"external_id" db:"external_id"` - ResponseToID_ courier.MsgID `json:"response_to_id" db:"response_to_id"` ResponseToExternalID_ string `json:"response_to_external_id"` IsResend_ bool `json:"is_resend,omitempty"` Metadata_ json.RawMessage `json:"metadata" db:"metadata"` @@ -534,7 +533,6 @@ func (m *DBMsg) ContactName() string { return m.ContactName_ } func (m *DBMsg) HighPriority() bool { return m.HighPriority_ } func (m *DBMsg) ReceivedOn() *time.Time { return m.SentOn_ } func (m *DBMsg) SentOn() *time.Time { return m.SentOn_ } -func (m *DBMsg) ResponseToID() courier.MsgID { return m.ResponseToID_ } func (m *DBMsg) ResponseToExternalID() string { return m.ResponseToExternalID_ } func (m *DBMsg) IsResend() bool { return m.IsResend_ } diff --git a/handlers/chikka/chikka_test.go b/handlers/chikka/chikka_test.go index dd6fbc38a..b31a9533e 100644 --- a/handlers/chikka/chikka_test.go +++ b/handlers/chikka/chikka_test.go @@ -71,7 +71,6 @@ var defaultSendTestCases = []ChannelSendTestCase{ {Label: "Plain Reply", Text: "Simple Message", URN: "tel:+63911231234", Status: "W", - ResponseToID: 5, ResponseToExternalID: "external-id", ResponseBody: "Success", ResponseStatus: 200, PostParams: map[string]string{ @@ -88,7 +87,6 @@ var defaultSendTestCases = []ChannelSendTestCase{ SendPrep: setSendURL}, {Label: "Failed Reply use Send", Text: "Simple Message", URN: "tel:+63911231234", - ResponseToID: 5, ResponseToExternalID: "external-id", ResponseBody: `{"status":400,"message":"BAD REQUEST","description":"Invalid\\/Used Request ID"}`, ResponseStatus: 400, diff --git a/handlers/facebook/facebook.go b/handlers/facebook/facebook.go index 62fd688bc..75aff3de4 100644 --- a/handlers/facebook/facebook.go +++ b/handlers/facebook/facebook.go @@ -481,7 +481,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload := mtPayload{} // set our message type - if msg.ResponseToID() != courier.NilMsgID { + if msg.ResponseToExternalID() != "" { payload.MessagingType = "RESPONSE" } else if topic != "" { payload.MessagingType = "MESSAGE_TAG" diff --git a/handlers/facebook/facebook_test.go b/handlers/facebook/facebook_test.go index 6a22ebbc9..42f2ad648 100644 --- a/handlers/facebook/facebook_test.go +++ b/handlers/facebook/facebook_test.go @@ -569,7 +569,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ SendPrep: setSendURL}, {Label: "Plain Response", Text: "Simple Message", URN: "facebook:12345", - Status: "W", ExternalID: "mid.133", ResponseToID: 23526, + Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 09686c217..77982f4cb 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -488,7 +488,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload := mtPayload{} // set our message type - if msg.ResponseToID() != courier.NilMsgID { + if msg.ResponseToExternalID() != "" { payload.MessagingType = "RESPONSE" } else if topic != "" { payload.MessagingType = "MESSAGE_TAG" diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 79e27f7e8..5514c7114 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -570,7 +570,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ SendPrep: setSendURL}, {Label: "Plain Response", Text: "Simple Message", URN: "facebook:12345", - Status: "W", ExternalID: "mid.133", ResponseToID: 23526, + Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, diff --git a/handlers/test.go b/handlers/test.go index 6f139d194..9f5a88e3a 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -87,7 +87,6 @@ type ChannelSendTestCase struct { QuickReplies []string Topic string HighPriority bool - ResponseToID int64 ResponseToExternalID string Metadata json.RawMessage @@ -221,7 +220,7 @@ func RunChannelSendTestCases(t *testing.T, channel courier.Channel, handler cour t.Run(testCase.Label, func(t *testing.T) { require := require.New(t) - msg := mb.NewOutgoingMsg(channel, courier.NewMsgID(10), urns.URN(testCase.URN), testCase.Text, testCase.HighPriority, testCase.QuickReplies, testCase.Topic, testCase.ResponseToID, testCase.ResponseToExternalID) + msg := mb.NewOutgoingMsg(channel, courier.NewMsgID(10), urns.URN(testCase.URN), testCase.Text, testCase.HighPriority, testCase.QuickReplies, testCase.Topic, testCase.ResponseToExternalID) for _, a := range testCase.Attachments { msg.WithAttachment(a) diff --git a/msg.go b/msg.go index 7ee4044e9..809370f20 100644 --- a/msg.go +++ b/msg.go @@ -95,7 +95,6 @@ type Msg interface { QuickReplies() []string Topic() string Metadata() json.RawMessage - ResponseToID() MsgID ResponseToExternalID() string IsResend() bool diff --git a/test.go b/test.go index 527853648..09762d438 100644 --- a/test.go +++ b/test.go @@ -119,13 +119,8 @@ func (mb *MockBackend) NewIncomingMsg(channel Channel, urn urns.URN, text string } // NewOutgoingMsg creates a new outgoing message from the given params -func (mb *MockBackend) NewOutgoingMsg(channel Channel, id MsgID, urn urns.URN, text string, highPriority bool, quickReplies []string, topic string, responseToID int64, responseToExternalID string) Msg { - msgResponseToID := NilMsgID - if responseToID != 0 { - msgResponseToID = NewMsgID(responseToID) - } - - return &mockMsg{channel: channel, id: id, urn: urn, text: text, highPriority: highPriority, quickReplies: quickReplies, topic: topic, responseToID: msgResponseToID, responseToExternalID: responseToExternalID} +func (mb *MockBackend) NewOutgoingMsg(channel Channel, id MsgID, urn urns.URN, text string, highPriority bool, quickReplies []string, topic string, responseToExternalID string) Msg { + return &mockMsg{channel: channel, id: id, urn: urn, text: text, highPriority: highPriority, quickReplies: quickReplies, topic: topic, responseToExternalID: responseToExternalID} } // PushOutgoingMsg is a test method to add a message to our queue of messages to send @@ -568,7 +563,6 @@ type mockMsg struct { highPriority bool quickReplies []string topic string - responseToID MsgID responseToExternalID string metadata json.RawMessage alreadyWritten bool @@ -594,7 +588,6 @@ func (m *mockMsg) ContactName() string { return m.contactName } func (m *mockMsg) HighPriority() bool { return m.highPriority } func (m *mockMsg) QuickReplies() []string { return m.quickReplies } func (m *mockMsg) Topic() string { return m.topic } -func (m *mockMsg) ResponseToID() MsgID { return m.responseToID } func (m *mockMsg) ResponseToExternalID() string { return m.responseToExternalID } func (m *mockMsg) Metadata() json.RawMessage { return m.metadata } func (m *mockMsg) IsResend() bool { return m.isResend } From b609d29ab99b70079e2626dbe5122f1930a37cca Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 1 Dec 2021 11:52:45 -0500 Subject: [PATCH 009/146] Update CHANGELOG.md for v7.1.3 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d7838a92a..a94dd65fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.3 +---------- + * Use response_to_external_id instead of response_to_id + v7.1.2 ---------- * External channel handler should use headers config setting if provided From 30f9a91d87b4b50cf6fc950cc167d97489209192 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 1 Dec 2021 13:41:49 -0500 Subject: [PATCH 010/146] Remove loop detection now that mailroom does this --- backend.go | 4 --- backends/rapidpro/backend.go | 60 ------------------------------- backends/rapidpro/backend_test.go | 30 ---------------- sender.go | 13 ------- test.go | 5 --- 5 files changed, 112 deletions(-) diff --git a/backend.go b/backend.go index 196d02d4f..9516a0cdc 100644 --- a/backend.go +++ b/backend.go @@ -74,10 +74,6 @@ type Backend interface { // a message is being forced in being resent by a user ClearMsgSent(context.Context, MsgID) error - // IsMsgLoop returns whether the passed in message is part of a message loop, possibly with another bot. Backends should - // implement their own logic to implement this. - IsMsgLoop(ctx context.Context, msg Msg) (bool, error) - // MarkOutgoingMsgComplete marks the passed in message as having been processed. Note this should be called even in the case // of errors during sending as it will manage the number of active workers per channel. The optional status parameter can be // used to determine any sort of deduping of msg sends diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 0d362e471..364d0d3fe 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -35,9 +35,6 @@ const sentSetName = "msgs_sent_%s" // our timeout for backend operations const backendTimeout = time.Second * 20 -// number of messages for loop detection -const msgLoopThreshold = 20 - func init() { courier.RegisterBackend("rapidpro", newBackend) } @@ -202,63 +199,6 @@ func (b *backend) ClearMsgSent(ctx context.Context, id courier.MsgID) error { return err } -var luaMsgLoop = redis.NewScript(3, `-- KEYS: [key, contact_id, text] - local key = KEYS[1] - local contact_id = KEYS[2] - local text = KEYS[3] - local count = 1 - - -- try to look up in window - local record = redis.call("hget", key, contact_id) - if record then - local record_count = tonumber(string.sub(record, 1, 2)) - local record_text = string.sub(record, 4, -1) - - if record_text == text then - count = math.min(record_count + 1, 99) - else - count = 1 - end - end - - -- create our new record with our updated count - record = string.format("%02d:%s", count, text) - - -- write our new record with updated count - redis.call("hset", key, contact_id, record) - - -- sets its expiration - redis.call("expire", key, 300) - - return count -`) - -// IsMsgLoop checks whether the passed in message is part of a loop -func (b *backend) IsMsgLoop(ctx context.Context, msg courier.Msg) (bool, error) { - m := msg.(*DBMsg) - - // things that aren't replies can't be loops, neither do we count retries - if m.ResponseToExternalID_ == "" || m.ErrorCount_ > 0 { - return false, nil - } - - // otherwise run our script to check whether this is a loop in the past 5 minutes - rc := b.redisPool.Get() - defer rc.Close() - - keyTime := time.Now().UTC().Round(time.Minute * 5) - key := fmt.Sprintf(sentSetName, fmt.Sprintf("loop_msgs:%s", keyTime.Format("2006-01-02-15:04"))) - count, err := redis.Int(luaMsgLoop.Do(rc, key, m.ContactID_, m.Text_)) - if err != nil { - return false, errors.Wrapf(err, "error while checking for msg loop") - } - - if count >= msgLoopThreshold { - return true, nil - } - return false, nil -} - // MarkOutgoingMsgComplete marks the passed in message as having completed processing, freeing up a worker for that channel func (b *backend) MarkOutgoingMsgComplete(ctx context.Context, msg courier.Msg, status courier.MsgStatus) { rc := b.redisPool.Get() diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index b803f8e7b..f58bb3186 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -748,36 +748,6 @@ func (ts *BackendTestSuite) TestExternalIDDupes() { ts.True(m2.alreadyWritten) } -func (ts *BackendTestSuite) TestLoop() { - ctx := context.Background() - dbMsg := readMsgFromDB(ts.b, courier.NewMsgID(10000)) - - dbMsg.ResponseToExternalID_ = "65474" - - loop, err := ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.False(loop) - - // call it 18 times more, no loop still - for i := 0; i < 18; i++ { - loop, err = ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.False(loop) - } - - // last one should make us a loop - loop, err = ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.True(loop) - - // make sure this keeps working even in hundreds of loops - for i := 0; i < 100; i++ { - loop, err = ts.b.IsMsgLoop(ctx, dbMsg) - ts.NoError(err) - ts.True(loop) - } -} - func (ts *BackendTestSuite) TestStatus() { // our health should just contain the header ts.True(strings.Contains(ts.b.Status(), "Channel"), ts.b.Status()) diff --git a/sender.go b/sender.go index 63d966822..dbed21aab 100644 --- a/sender.go +++ b/sender.go @@ -189,23 +189,10 @@ func (w *Sender) sendMessage(msg Msg) { log.WithError(err).Error("error looking up msg was sent") } - // is this msg in a loop? - loop, err := backend.IsMsgLoop(sendCTX, msg) - - // failing on loop lookup isn't permanent, but log - if err != nil { - log.WithError(err).Error("error looking up msg loop") - } - if sent { // if this message was already sent, create a wired status for it status = backend.NewMsgStatusForID(msg.Channel(), msg.ID(), MsgWired) log.Warning("duplicate send, marking as wired") - } else if loop { - // if this contact is in a loop, fail the message immediately without sending - status = backend.NewMsgStatusForID(msg.Channel(), msg.ID(), MsgFailed) - status.AddLog(NewChannelLogFromError("Message Loop", msg.Channel(), msg.ID(), 0, fmt.Errorf("message loop detected, failing message without send"))) - log.Error("message loop detected, failing message") } else { // send our message status, err = server.SendMsg(sendCTX, msg) diff --git a/test.go b/test.go index 09762d438..012c67a1b 100644 --- a/test.go +++ b/test.go @@ -161,11 +161,6 @@ func (mb *MockBackend) ClearMsgSent(ctx context.Context, id MsgID) error { return nil } -// IsMsgLoop returns whether the passed in msg is a loop -func (mb *MockBackend) IsMsgLoop(ctx context.Context, msg Msg) (bool, error) { - return false, nil -} - // MarkOutgoingMsgComplete marks the passed msg as having been dealt with func (mb *MockBackend) MarkOutgoingMsgComplete(ctx context.Context, msg Msg, s MsgStatus) { mb.mutex.Lock() From 99c9b6aaa2cb43171f07ee07e5cfaf0f02b3783d Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 2 Dec 2021 15:20:03 -0500 Subject: [PATCH 011/146] Update CHANGELOG.md for v7.1.4 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a94dd65fc..ec33438ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.1.4 +---------- + * Remove loop detection now that mailroom does this + * Smarter organization of quick replies for viber keyboards + v7.1.3 ---------- * Use response_to_external_id instead of response_to_id From 756b32f6f22382e320c1ac462fa43b4ef8afcdaf Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 6 Dec 2021 13:45:36 -0500 Subject: [PATCH 012/146] Add Msg.failed_reason --- backends/rapidpro/backend_test.go | 4 ++++ backends/rapidpro/msg.go | 6 ++++-- backends/rapidpro/schema.sql | 4 +++- backends/rapidpro/status.go | 16 ++++++++++++++++ 4 files changed, 27 insertions(+), 3 deletions(-) diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index f58bb3186..74de96d55 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -481,6 +481,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { ts.Equal(null.String("ext0"), m.ExternalID_) ts.True(m.ModifiedOn_.After(now)) ts.True(m.SentOn_.After(now)) + ts.Equal(null.NullString, m.FailedReason_) sentOn := *m.SentOn_ @@ -584,6 +585,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { ts.Equal(m.ErrorCount_, 1) ts.True(m.ModifiedOn_.After(now)) ts.True(m.NextAttempt_.After(now)) + ts.Equal(null.NullString, m.FailedReason_) // second go status = ts.b.NewMsgStatusForExternalID(channel, "ext1", courier.MsgErrored) @@ -594,6 +596,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { m = readMsgFromDB(ts.b, courier.NewMsgID(10000)) ts.Equal(m.Status_, courier.MsgErrored) ts.Equal(m.ErrorCount_, 2) + ts.Equal(null.NullString, m.FailedReason_) // third go status = ts.b.NewMsgStatusForExternalID(channel, "ext1", courier.MsgErrored) @@ -604,6 +607,7 @@ func (ts *BackendTestSuite) TestMsgStatus() { m = readMsgFromDB(ts.b, courier.NewMsgID(10000)) ts.Equal(m.Status_, courier.MsgFailed) ts.Equal(m.ErrorCount_, 3) + ts.Equal(null.String("E"), m.FailedReason_) // update URN when the new doesn't exist tx, _ := ts.b.db.BeginTxx(ctx, nil) diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index 020071f84..a194a9363 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -175,6 +175,7 @@ SELECT attachments, msg_count, error_count, + failed_reason, high_priority, status, visibility, @@ -497,8 +498,9 @@ type DBMsg struct { ContactID_ ContactID `json:"contact_id" db:"contact_id"` ContactURNID_ ContactURNID `json:"contact_urn_id" db:"contact_urn_id"` - MessageCount_ int `json:"msg_count" db:"msg_count"` - ErrorCount_ int `json:"error_count" db:"error_count"` + MessageCount_ int `json:"msg_count" db:"msg_count"` + ErrorCount_ int `json:"error_count" db:"error_count"` + FailedReason_ null.String `json:"failed_reason" db:"failed_reason"` ChannelUUID_ courier.ChannelUUID `json:"channel_uuid"` ContactName_ string `json:"contact_name"` diff --git a/backends/rapidpro/schema.sql b/backends/rapidpro/schema.sql index 191169480..e21cbb7c0 100644 --- a/backends/rapidpro/schema.sql +++ b/backends/rapidpro/schema.sql @@ -72,6 +72,7 @@ CREATE TABLE msgs_msg ( msg_count integer NOT NULL, error_count integer NOT NULL, next_attempt timestamp with time zone NOT NULL, + failed_reason character varying(1), external_id character varying(255), attachments character varying(255)[], channel_id integer references channels_channel(id) on delete cascade, @@ -79,7 +80,8 @@ CREATE TABLE msgs_msg ( contact_urn_id integer NOT NULL references contacts_contacturn(id) on delete cascade, org_id integer NOT NULL references orgs_org(id) on delete cascade, metadata text, - topup_id integer + topup_id integer, + delete_from_counts boolean ); DROP TABLE IF EXISTS channels_channellog CASCADE; diff --git a/backends/rapidpro/status.go b/backends/rapidpro/status.go index 6a81d6335..a707bb7a4 100644 --- a/backends/rapidpro/status.go +++ b/backends/rapidpro/status.go @@ -107,6 +107,14 @@ UPDATE msgs_msg SET ELSE next_attempt END, + failed_reason = CASE + WHEN + error_count >= 2 + THEN + 'E' + ELSE + failed_reason + END, sent_on = CASE WHEN :status = 'W' @@ -164,6 +172,14 @@ UPDATE msgs_msg SET ELSE next_attempt END, + failed_reason = CASE + WHEN + error_count >= 2 + THEN + 'E' + ELSE + failed_reason + END, sent_on = CASE WHEN :status IN ('W', 'S', 'D') From ca40349d6822883ce64fc54067ddcf2bca439303 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 6 Dec 2021 14:12:40 -0500 Subject: [PATCH 013/146] Update CHANGELOG.md for v7.1.5 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ec33438ce..5a14e5798 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.5 +---------- + * Add Msg.failed_reason and set when msg fails due to reaching error limit + v7.1.4 ---------- * Remove loop detection now that mailroom does this From 9560dc96c3d6feb0c89ebe2d87101cb728c6bc7f Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 8 Dec 2021 16:11:43 +0200 Subject: [PATCH 014/146] Add comment about the 2 seconds pause choice --- handlers/whatsapp/whatsapp.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 651b960cd..ee9f30022 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -847,6 +847,10 @@ func sendWhatsAppMsg(rc redis.Conn, msg courier.Msg, sendPath *url.URL, payload if rr.StatusCode == 429 || rr.StatusCode == 503 { rateLimitKey := fmt.Sprintf("rate_limit:%s", msg.Channel().UUID().String()) rc.Do("set", rateLimitKey, "engaged") + + // The rate limit is 50 requests per second + // We pause sending 2 seconds so the limit count is reset + // TODO: In the future we should the header value when available rc.Do("expire", rateLimitKey, 2) log := courier.NewChannelLogFromRR("rate limit engaged", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) From cb453c1451772e3c1e153291398625d46c73233b Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 8 Dec 2021 09:27:06 -0500 Subject: [PATCH 015/146] Update CHANGELOG.md for v7.1.6 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a14e5798..a481824da 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.6 +---------- + * Throttle WA queues when we get 429 responses + v7.1.5 ---------- * Add Msg.failed_reason and set when msg fails due to reaching error limit From 718aaa137c2885b8d951ef810272b66763d5cf32 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 10 Dec 2021 15:34:46 -0500 Subject: [PATCH 016/146] Update .gitignore to include deploy/ --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 6b5579d0b..86bba308f 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.a *.so *~ +deploy fabric fabfile.py fabfile.pyc @@ -34,4 +35,4 @@ _testmain.go dist/ .envrc courier -_storage \ No newline at end of file +_storage From 3233f3d7b0b229e8ae7ec700d4f7d64a1b3b9bbe Mon Sep 17 00:00:00 2001 From: Robi9 Date: Mon, 29 Nov 2021 10:51:22 -0300 Subject: [PATCH 017/146] Add instagram handler --- handlers/instagram/instagram.go | 600 +++++++++++++++++++++++++++ handlers/instagram/instagram_test.go | 461 ++++++++++++++++++++ 2 files changed, 1061 insertions(+) create mode 100644 handlers/instagram/instagram.go create mode 100644 handlers/instagram/instagram_test.go diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go new file mode 100644 index 000000000..6c4c9f35f --- /dev/null +++ b/handlers/instagram/instagram.go @@ -0,0 +1,600 @@ +package instagram + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha1" + "encoding/hex" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/buger/jsonparser" + "github.com/nyaruka/courier" + "github.com/nyaruka/courier/handlers" + "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/urns" + "github.com/pkg/errors" +) + +// Endpoints we hit +var ( + sendURL = "https://graph.facebook.com/v12.0/me/messages" + graphURL = "https://graph.facebook.com/v12.0/" + + signatureHeader = "X-Hub-Signature" + + // max for the body + maxMsgLength = 1000 + + //Only Human_Agent tag available for instagram + tagByTopic = map[string]string{ + "agent": "HUMAN_AGENT", + } +) + +// keys for extra in channel events +const ( + titleKey = "title" + payloadKey = "payload" +) + +func init() { + courier.RegisterHandler(newHandler()) +} + +type handler struct { + handlers.BaseHandler +} + +func newHandler() courier.ChannelHandler { + return &handler{handlers.NewBaseHandlerWithParams(courier.ChannelType("IG"), "Instagram", false)} +} + +// Initialize is called by the engine once everything is loaded +func (h *handler) Initialize(s courier.Server) error { + h.SetServer(s) + s.AddHandlerRoute(h, http.MethodGet, "receive", h.receiveVerify) + s.AddHandlerRoute(h, http.MethodPost, "receive", h.receiveEvent) + return nil +} + +type igSender struct { + ID string `json:"id"` +} + +type igUser struct { + ID string `json:"id"` +} + +// { +// "object":"instagram", +// "entry":[{ +// "id":"180005062406476", +// "time":1514924367082, +// "messaging":[{ +// "sender": {"id":"1630934236957797"}, +// "recipient":{"id":"180005062406476"}, +// "timestamp":1514924366807, +// "message":{ +// "mid":"mid.$cAAD5QiNHkz1m6cyj11guxokwkhi2", +// "text":"65863634" +// } +// }] +// }] +// } + +type moPayload struct { + Object string `json:"object"` + Entry []struct { + ID string `json:"id"` + Time int64 `json:"time"` + Messaging []struct { + Sender igSender `json:"sender"` + Recipient igUser `json:"recipient"` + Timestamp int64 `json:"timestamp"` + + Postback *struct { + MID string `json:"mid"` + Title string `json:"title"` + Payload string `json:"payload"` + } `json:"postback,omitempty"` + + Message *struct { + IsEcho bool `json:"is_echo,omitempty"` + MID string `json:"mid"` + Text string `json:"text,omitempty"` + QuickReply struct { + Payload string `json:"payload"` + } `json:"quick_replies,omitempty"` + Attachments []struct { + Type string `json:"type"` + Payload *struct { + URL string `json:"url"` + } `json:"payload"` + } `json:"attachments,omitempty"` + } `json:"message,omitempty"` + } `json:"messaging"` + } `json:"entry"` +} + +/*type moPayload struct { + Object string `json:"object"` + Entry []struct { + ID string `json:"id"` + Time int64 `json:"time"` + Changes []struct { + Field string `json:"field"` + Value struct { + Sender struct { + ID string `json:"id"` + } `json:"sender"` + + Recipient struct { + ID string `json:"id"` + } `json:"recipient"` + Timestamp int64 `json:"timestamp"` + + Postback *struct { + MID string `json:"mid"` + Title string `json:"title"` + Payload string `json:"payload"` + } `json:"postback,omitempty"` + + Message *struct { + IsEcho bool `json:"is_echo,omitempty"` + MID string `json:"mid"` + Text string `json:"text,omitempty"` + QuickReply struct { + Payload string `json:"payload"` + } `json:"quick_replies,omitempty"` + Attachments []struct { + Type string `json:"type"` + Payload *struct { + URL string `json:"url"` + } `json:"payload"` + } `json:"attachments,omitempty"` + } `json:"message,omitempty"` + } `json:"value"` + } `json:"changes"` + } `json:"entry"` +}*/ + +// GetChannel returns the channel +func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Channel, error) { + + if r.Method == http.MethodGet { + + return nil, nil + } + + payload := &moPayload{} + + err := handlers.DecodeAndValidateJSON(payload, r) + + if err != nil { + + return nil, err + } + + // not a instagram object? ignore + if payload.Object != "instagram" { + + return nil, fmt.Errorf("object expected 'instagram', found %s", payload.Object) + } + + // no entries? ignore this request + if len(payload.Entry) == 0 { + + return nil, fmt.Errorf("no entries found") + } + + igID := payload.Entry[0].ID + + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(igID)) +} + +// receiveVerify handles Instagram's webhook verification callback +func (h *handler) receiveVerify(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { + mode := r.URL.Query().Get("hub.mode") + + // this isn't a subscribe verification, that's an error + if mode != "subscribe" { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("unknown request")) + } + + // verify the token against our server facebook webhook secret, if the same return the challenge IG sent us + secret := r.URL.Query().Get("hub.verify_token") + + if secret != h.Server().Config().FacebookWebhookSecret { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("token does not match secret")) + } + // and respond with the challenge token + _, err := fmt.Fprint(w, r.URL.Query().Get("hub.challenge")) + return nil, err +} + +// receiveEvent is our HTTP handler function for incoming messages and status updates +func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { + err := h.validateSignature(r) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + payload := &moPayload{} + err = handlers.DecodeAndValidateJSON(payload, r) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + // not a instagram object? ignore + if payload.Object != "instagram" { + return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request") + } + + // no entries? ignore this request + if len(payload.Entry) == 0 { + return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request, no entries") + } + + // the list of events we deal with + events := make([]courier.Event, 0, 2) + + // the list of data we will return in our response + data := make([]interface{}, 0, 2) + + // for each entry + for _, entry := range payload.Entry { + // no entry, ignore + if len(entry.Messaging) == 0 { + continue + } + + // grab our message, there is always a single one + msg := entry.Messaging[0] + + //msg.Value.Recipient.ID = "218041941572367" + + // ignore this entry if it is to another page + if channel.Address() != msg.Recipient.ID { + continue + } + + // create our date from the timestamp (they give us millis, arg is nanos) + date := time.Unix(0, msg.Timestamp*1000000).UTC() + + sender := msg.Sender.ID + if sender == "" { + sender = msg.Sender.ID + } + + // create our URN + urn, err := urns.NewInstagramURN(sender) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + if msg.Postback != nil { + // by default postbacks are treated as new conversations + eventType := courier.NewConversation + event := h.Backend().NewChannelEvent(channel, eventType, urn).WithOccurredOn(date) + + // build our extra + extra := map[string]interface{}{ + titleKey: msg.Postback.Title, + payloadKey: msg.Postback.Payload, + } + + event = event.WithExtra(extra) + + err := h.Backend().WriteChannelEvent(ctx, event) + if err != nil { + return nil, err + } + + events = append(events, event) + data = append(data, courier.NewEventReceiveData(event)) + } else if msg.Message != nil { + // this is an incoming message + // ignore echos + if msg.Message.IsEcho { + data = append(data, courier.NewInfoData("ignoring echo")) + continue + } + + text := msg.Message.Text + + attachmentURLs := make([]string, 0, 2) + + for _, att := range msg.Message.Attachments { + if att.Payload != nil && att.Payload.URL != "" { + attachmentURLs = append(attachmentURLs, att.Payload.URL) + } + } + + // create our message + ev := h.Backend().NewIncomingMsg(channel, urn, text).WithExternalID(msg.Message.MID).WithReceivedOn(date) + event := h.Backend().CheckExternalIDSeen(ev) + + // add any attachment URL found + for _, attURL := range attachmentURLs { + event.WithAttachment(attURL) + } + + err := h.Backend().WriteMsg(ctx, event) + if err != nil { + return nil, err + } + + h.Backend().WriteExternalIDSeen(event) + + events = append(events, event) + data = append(data, courier.NewMsgReceiveData(event)) + + } else { + data = append(data, courier.NewInfoData("ignoring unknown entry type")) + } + } + return events, courier.WriteDataResponse(ctx, w, http.StatusOK, "Events Handled", data) +} + +// { +// "messaging_type": "" +// "recipient":{ +// "id":"" +// }, +// "message":{ +// "text":"hello, world!" +// "attachment":{ +// "type":"image", +// "payload":{ +// "url":"http://www.messenger-rocks.com/image.jpg", +// "is_reusable":true +// } +// } +// } +// } +type mtPayload struct { + MessagingType string `json:"messaging_type"` + Tag string `json:"tag,omitempty"` + Recipient struct { + //UserRef string `json:"user_ref,omitempty"` + ID string `json:"id,omitempty"` + } `json:"recipient"` + Message struct { + Text string `json:"text,omitempty"` + QuickReplies []mtQuickReply `json:"quick_replies,omitempty"` + Attachment *mtAttachment `json:"attachment,omitempty"` + } `json:"message"` +} + +type mtAttachment struct { + Type string `json:"type"` + Payload struct { + URL string `json:"url,omitempty"` + IsReusable bool `json:"is_reusable,omitempty"` + } `json:"payload"` +} +type mtQuickReply struct { + Title string `json:"title"` + Payload string `json:"payload"` + ContentType string `json:"content_type"` +} + +func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { + // can't do anything without an access token + accessToken := msg.Channel().StringConfigForKey(courier.ConfigAuthToken, "") + if accessToken == "" { + return nil, fmt.Errorf("missing access token") + } + + topic := msg.Topic() + payload := mtPayload{} + + // set our message type + if msg.ResponseToID() != courier.NilMsgID { + payload.MessagingType = "RESPONSE" + } else if topic != "" { + payload.MessagingType = "MESSAGE_TAG" + payload.Tag = tagByTopic[topic] + } else { + payload.MessagingType = "UPDATE" + } + + payload.Recipient.ID = msg.URN().Path() + + msgURL, _ := url.Parse(sendURL) + query := url.Values{} + query.Set("access_token", accessToken) + msgURL.RawQuery = query.Encode() + + status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) + + msgParts := make([]string, 0) + if msg.Text() != "" { + msgParts = handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) + } + + // send each part and each attachment separately. we send attachments first as otherwise quick replies + // attached to text messages get hidden when images get delivered + for i := 0; i < len(msgParts)+len(msg.Attachments()); i++ { + if i < len(msg.Attachments()) { + // this is an attachment + payload.Message.Attachment = &mtAttachment{} + attType, attURL := handlers.SplitAttachment(msg.Attachments()[i]) + attType = strings.Split(attType, "/")[0] + payload.Message.Attachment.Type = attType + payload.Message.Attachment.Payload.URL = attURL + payload.Message.Attachment.Payload.IsReusable = true + payload.Message.Text = "" + } else { + // this is still a msg part + payload.Message.Text = msgParts[i-len(msg.Attachments())] + payload.Message.Attachment = nil + } + + // include any quick replies on the last piece we send + if i == (len(msgParts)+len(msg.Attachments()))-1 { + for _, qr := range msg.QuickReplies() { + payload.Message.QuickReplies = append(payload.Message.QuickReplies, mtQuickReply{qr, qr, "text"}) + } + } else { + payload.Message.QuickReplies = nil + } + + jsonBody, err := json.Marshal(payload) + if err != nil { + return status, err + } + + req, err := http.NewRequest(http.MethodPost, msgURL.String(), bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + rr, err := utils.MakeHTTPRequest(req) + + // record our status and log + log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) + status.AddLog(log) + if err != nil { + return status, nil + } + + externalID, err := jsonparser.GetString(rr.Body, "message_id") + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to get message_id from body")) + return status, nil + } + + // if this is our first message, record the external id + if i == 0 { + status.SetExternalID(externalID) + if msg.URN().IsInstagramRef() { + recipientID, err := jsonparser.GetString(rr.Body, "recipient_id") + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to get recipient_id from body")) + return status, nil + } + + referralID := msg.URN().InstagramRef() + + realIDURN, err := urns.NewInstagramURN(recipientID) + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to make Instagram urn from %s", recipientID)) + } + + contact, err := h.Backend().GetContact(ctx, msg.Channel(), msg.URN(), "", "") + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to get contact for %s", msg.URN().String())) + } + realURN, err := h.Backend().AddURNtoContact(ctx, msg.Channel(), contact, realIDURN) + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to add real Instagram URN %s to contact with uuid %s", realURN.String(), contact.UUID())) + } + referralIDExtURN, err := urns.NewURNFromParts(urns.ExternalScheme, referralID, "", "") + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to make ext urn from %s", referralID)) + } + extURN, err := h.Backend().AddURNtoContact(ctx, msg.Channel(), contact, referralIDExtURN) + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to add URN %s to contact with uuid %s", extURN.String(), contact.UUID())) + } + + referralInstagramURN, err := h.Backend().RemoveURNfromContact(ctx, msg.Channel(), contact, msg.URN()) + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to remove referral Instagram URN %s from contact with uuid %s", referralInstagramURN.String(), contact.UUID())) + } + + } + + } + + // this was wired successfully + status.SetStatus(courier.MsgWired) + } + + return status, nil +} + +// DescribeURN looks up URN metadata for new contacts +func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn urns.URN) (map[string]string, error) { + // can't do anything with Instagram refs, ignore them + if urn.IsInstagramRef() { + return map[string]string{}, nil + } + + accessToken := channel.StringConfigForKey(courier.ConfigAuthToken, "") + if accessToken == "" { + return nil, fmt.Errorf("missing access token") + } + + // build a request to lookup the stats for this contact + base, _ := url.Parse(graphURL) + path, _ := url.Parse(fmt.Sprintf("/%s", urn.Path())) + u := base.ResolveReference(path) + + query := url.Values{} + query.Set("fields", "first_name,last_name") + query.Set("access_token", accessToken) + u.RawQuery = query.Encode() + req, _ := http.NewRequest(http.MethodGet, u.String(), nil) + rr, err := utils.MakeHTTPRequest(req) + if err != nil { + return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) + } + + // read our first and last name + firstName, _ := jsonparser.GetString(rr.Body, "first_name") + lastName, _ := jsonparser.GetString(rr.Body, "last_name") + + return map[string]string{"name": utils.JoinNonEmpty(" ", firstName, lastName)}, nil +} + +// see https://developers.facebook.com/docs/messenger-platform/webhook#security +func (h *handler) validateSignature(r *http.Request) error { + headerSignature := r.Header.Get(signatureHeader) + if headerSignature == "" { + return fmt.Errorf("missing request signature") + } + appSecret := h.Server().Config().FacebookApplicationSecret + + body, err := handlers.ReadBody(r, 100000) + if err != nil { + return fmt.Errorf("unable to read request body: %s", err) + } + + expectedSignature, err := fbCalculateSignature(appSecret, body) + if err != nil { + return err + } + + signature := "" + if len(headerSignature) == 45 && strings.HasPrefix(headerSignature, "sha1=") { + signature = strings.TrimPrefix(headerSignature, "sha1=") + } + + // compare signatures in way that isn't sensitive to a timing attack + if !hmac.Equal([]byte(expectedSignature), []byte(signature)) { + return fmt.Errorf("invalid request signature, expected: %s got: %s for body: '%s'", expectedSignature, signature, string(body)) + } + + return nil +} + +func fbCalculateSignature(appSecret string, body []byte) (string, error) { + var buffer bytes.Buffer + buffer.Write(body) + + // hash with SHA1 + mac := hmac.New(sha1.New, []byte(appSecret)) + mac.Write(buffer.Bytes()) + + return hex.EncodeToString(mac.Sum(nil)), nil +} diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go new file mode 100644 index 000000000..b18ab10f3 --- /dev/null +++ b/handlers/instagram/instagram_test.go @@ -0,0 +1,461 @@ +package instagram + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/nyaruka/courier" + "github.com/nyaruka/courier/handlers" + . "github.com/nyaruka/courier/handlers" + "github.com/nyaruka/gocommon/urns" + "github.com/stretchr/testify/assert" +) + +var testChannels = []courier.Channel{ + courier.NewMockChannel("8ab23e93-5ecb-45ba-b726-3b064e0c568c", "IG", "1234", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), +} + +var helloMsg = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var duplicateMsg = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }, + { + "id": "1234", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var invalidURN = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "abc5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var attachment = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "message": { + "mid": "external_id", + "attachments":[{ + "type":"image", + "payload":{ + "url":"https://image-url/foo.png" + } + }] + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var like_heart = `{ + "object":"instagram", + "entry":[{ + "id":"1234", + "messaging":[{ + "sender":{"id":"5678"}, + "recipient":{"id":"1234"}, + "timestamp":1459991487970, + "message":{ + "mid":"external_id", + "attachments":[{ + "type":"like_heart" + }] + } + }], + "time":1459991487970 + }] +}` + +var differentPage = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1235" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var echo = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970, + "message": { + "is_echo": true, + "mid": "qT7ywaK" + } + }] + }] +}` + +var icebreakerGetStarted = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "postback": { + "title": "icebreaker question", + "payload": "get_started" + }, + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var notInstagram = `{ + "object":"notinstagram", + "entry": [{}] +}` + +var noEntries = `{ + "object":"instagram", + "entry": [] +}` + +var noMessagingEntries = `{ + "object":"instagram", + "entry": [{ + "id": "1234" + }] +}` + +var unkownMessagingEntry = `{ + "object":"instagram", + "entry": [{ + "id": "1234", + "messaging": [{ + "recipient": { + "id": "1234" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }] + }] +}` + +var notJSON = `blargh` + +var testCases = []ChannelHandleTestCase{ + {Label: "Receive Message", URL: "/c/ig/receive", Data: helloMsg, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Invalid Signature", URL: "/c/ig/receive", Data: helloMsg, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, + + {Label: "No Duplicate Receive Message", URL: "/c/ig/receive", Data: duplicateMsg, Status: 200, Response: "Handled", + Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Attachment", URL: "/c/ig/receive", Data: attachment, Status: 200, Response: "Handled", + Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Like Heart", URL: "/c/ig/receive", Data: like_heart, Status: 200, Response: "Handled", + Text: Sp(""), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Icebreaker Get Started", URL: "/c/ig/receive", Data: icebreakerGetStarted, Status: 200, Response: "Handled", + URN: Sp("instagram:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), + ChannelEventExtra: map[string]interface{}{"title": "icebreaker question", "payload": "get_started"}, + PrepRequest: addValidSignature}, + + {Label: "Different Page", URL: "/c/ig/receive", Data: differentPage, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, + {Label: "Echo", URL: "/c/ig/receive", Data: echo, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, + {Label: "Not Instagram", URL: "/c/ig/receive", Data: notInstagram, Status: 400, Response: "expected 'instagram', found notinstagram", PrepRequest: addValidSignature}, + {Label: "No Entries", URL: "/c/ig/receive", Data: noEntries, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, + {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntries, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntry, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURN, Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, +} + +func addValidSignature(r *http.Request) { + body, _ := handlers.ReadBody(r, 100000) + sig, _ := fbCalculateSignature("fb_app_secret", body) + r.Header.Set(signatureHeader, fmt.Sprintf("sha1=%s", string(sig))) +} + +func addInvalidSignature(r *http.Request) { + r.Header.Set(signatureHeader, "invalidsig") +} + +// mocks the call to the Facebook graph API +func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + accessToken := r.URL.Query().Get("access_token") + defer r.Body.Close() + + // invalid auth token + if accessToken != "a123" { + http.Error(w, "invalid auth token", 403) + } + + // user has a name + if strings.HasSuffix(r.URL.Path, "1337") { + w.Write([]byte(`{ "first_name": "John", "last_name": "Doe"}`)) + return + } + + // no name + w.Write([]byte(`{ "first_name": "", "last_name": ""}`)) + })) + graphURL = server.URL + + return server +} + +func TestDescribe(t *testing.T) { + fbGraph := buildMockFBGraph(testCases) + defer fbGraph.Close() + + handler := newHandler().(courier.URNDescriber) + tcs := []struct { + urn urns.URN + metadata map[string]string + }{{"instagram:1337", map[string]string{"name": "John Doe"}}, + {"instagram:4567", map[string]string{"name": ""}}, + {"instagram:ref:1337", map[string]string{}}} + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) + } +} + +func TestHandler(t *testing.T) { + RunChannelTestCases(t, testChannels, newHandler(), testCases) +} + +func BenchmarkHandler(b *testing.B) { + fbService := buildMockFBGraph(testCases) + defer fbService.Close() + + RunChannelBenchmarks(b, testChannels, newHandler(), testCases) +} + +func TestVerify(t *testing.T) { + + RunChannelTestCases(t, testChannels, newHandler(), []ChannelHandleTestCase{ + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, + Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, + {Label: "Verify No Mode", URL: "/c/ig/receive", Status: 400, Response: "unknown request"}, + {Label: "Verify No Secret", URL: "/c/ig/receive?hub.mode=subscribe", Status: 400, Response: "token does not match secret"}, + {Label: "Invalid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=blah", Status: 400, Response: "token does not match secret"}, + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, + }) + +} + +// setSendURL takes care of setting the send_url to our test server host +func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, m courier.Msg) { + sendURL = s.URL +} + +var defaultSendTestCases = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "instagram:12345", + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + + {Label: "Plain Response", + Text: "Simple Message", URN: "instagram:12345", + Status: "W", ExternalID: "mid.133", ResponseToID: 23526, + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + + {Label: "Tag Human Agent", + Text: "Simple Message", URN: "instagram:12345", + Status: "W", ExternalID: "mid.133", Topic: "agent", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + + {Label: "Plain Send using ref URN", + Text: "Simple Message", URN: "instagram:ref:67890", + ContactURNs: map[string]bool{"instagram:12345": true, "ext:67890": true, "instagram:ref:67890": false}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133", "recipient_id": "12345"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"user_ref":"67890"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Long Message", + Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", + URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"we exceed the max length?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + + {Label: "Send caption and photo with Quick Reply", + Text: "This is some text.", + URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + QuickReplies: []string{"Yes", "No"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"This is some text.","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + + {Label: "ID Error", + Text: "ID Error", URN: "instagram12345", + Status: "E", + ResponseBody: `{ "is_error": true }`, ResponseStatus: 200, + SendPrep: setSendURL}, + + {Label: "Error", + Text: "Error", URN: "instagram12345", + Status: "E", + ResponseBody: `{ "is_error": true }`, ResponseStatus: 403, + SendPrep: setSendURL}, + + {Label: "Quick Reply", + URN: "instagram:12345", Text: "Are you happy?", QuickReplies: []string{"Yes", "No"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + + {Label: "Send Photo", + URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, + SendPrep: setSendURL}, +} + +func TestSending(t *testing.T) { + // shorter max msg length for testing + maxMsgLength = 100 + var defaultChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "IG", "2020", "US", map[string]interface{}{courier.ConfigAuthToken: "access_token"}) + RunChannelSendTestCases(t, defaultChannel, newHandler(), defaultSendTestCases, nil) +} + +func TestSigning(t *testing.T) { + tcs := []struct { + Body string + Signature string + }{ + { + "hello world", + "308de7627fe19e92294c4572a7f831bc1002809d", + }, + { + "hello world2", + "ab6f902b58b9944032d4a960f470d7a8ebfd12b7", + }, + } + + for i, tc := range tcs { + sig, err := fbCalculateSignature("sesame", []byte(tc.Body)) + assert.NoError(t, err) + assert.Equal(t, tc.Signature, sig, "%d: mismatched signature", i) + } +} From ba9732855b18521bafc4847defcd2a55d09226e8 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Mon, 29 Nov 2021 11:05:06 -0300 Subject: [PATCH 018/146] refactor instagram.go --- handlers/instagram/instagram.go | 44 --------------------------------- 1 file changed, 44 deletions(-) diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go index 6c4c9f35f..bb803f818 100644 --- a/handlers/instagram/instagram.go +++ b/handlers/instagram/instagram.go @@ -122,48 +122,6 @@ type moPayload struct { } `json:"entry"` } -/*type moPayload struct { - Object string `json:"object"` - Entry []struct { - ID string `json:"id"` - Time int64 `json:"time"` - Changes []struct { - Field string `json:"field"` - Value struct { - Sender struct { - ID string `json:"id"` - } `json:"sender"` - - Recipient struct { - ID string `json:"id"` - } `json:"recipient"` - Timestamp int64 `json:"timestamp"` - - Postback *struct { - MID string `json:"mid"` - Title string `json:"title"` - Payload string `json:"payload"` - } `json:"postback,omitempty"` - - Message *struct { - IsEcho bool `json:"is_echo,omitempty"` - MID string `json:"mid"` - Text string `json:"text,omitempty"` - QuickReply struct { - Payload string `json:"payload"` - } `json:"quick_replies,omitempty"` - Attachments []struct { - Type string `json:"type"` - Payload *struct { - URL string `json:"url"` - } `json:"payload"` - } `json:"attachments,omitempty"` - } `json:"message,omitempty"` - } `json:"value"` - } `json:"changes"` - } `json:"entry"` -}*/ - // GetChannel returns the channel func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Channel, error) { @@ -257,8 +215,6 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h // grab our message, there is always a single one msg := entry.Messaging[0] - //msg.Value.Recipient.ID = "218041941572367" - // ignore this entry if it is to another page if channel.Address() != msg.Recipient.ID { continue From 91885b23c94da08b796ef6c5708eaf73c5f2e0ff Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 30 Nov 2021 15:34:26 -0300 Subject: [PATCH 019/146] Refactor instagram handler --- handlers/instagram/instagram.go | 46 ---------------------------- handlers/instagram/instagram_test.go | 12 ++------ 2 files changed, 2 insertions(+), 56 deletions(-) diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go index bb803f818..9c7da6198 100644 --- a/handlers/instagram/instagram.go +++ b/handlers/instagram/instagram.go @@ -318,7 +318,6 @@ type mtPayload struct { MessagingType string `json:"messaging_type"` Tag string `json:"tag,omitempty"` Recipient struct { - //UserRef string `json:"user_ref,omitempty"` ID string `json:"id,omitempty"` } `json:"recipient"` Message struct { @@ -422,54 +421,13 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat if err != nil { return status, nil } - externalID, err := jsonparser.GetString(rr.Body, "message_id") if err != nil { log.WithError("Message Send Error", errors.Errorf("unable to get message_id from body")) return status, nil } - - // if this is our first message, record the external id if i == 0 { status.SetExternalID(externalID) - if msg.URN().IsInstagramRef() { - recipientID, err := jsonparser.GetString(rr.Body, "recipient_id") - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to get recipient_id from body")) - return status, nil - } - - referralID := msg.URN().InstagramRef() - - realIDURN, err := urns.NewInstagramURN(recipientID) - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to make Instagram urn from %s", recipientID)) - } - - contact, err := h.Backend().GetContact(ctx, msg.Channel(), msg.URN(), "", "") - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to get contact for %s", msg.URN().String())) - } - realURN, err := h.Backend().AddURNtoContact(ctx, msg.Channel(), contact, realIDURN) - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to add real Instagram URN %s to contact with uuid %s", realURN.String(), contact.UUID())) - } - referralIDExtURN, err := urns.NewURNFromParts(urns.ExternalScheme, referralID, "", "") - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to make ext urn from %s", referralID)) - } - extURN, err := h.Backend().AddURNtoContact(ctx, msg.Channel(), contact, referralIDExtURN) - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to add URN %s to contact with uuid %s", extURN.String(), contact.UUID())) - } - - referralInstagramURN, err := h.Backend().RemoveURNfromContact(ctx, msg.Channel(), contact, msg.URN()) - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to remove referral Instagram URN %s from contact with uuid %s", referralInstagramURN.String(), contact.UUID())) - } - - } - } // this was wired successfully @@ -481,10 +439,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat // DescribeURN looks up URN metadata for new contacts func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn urns.URN) (map[string]string, error) { - // can't do anything with Instagram refs, ignore them - if urn.IsInstagramRef() { - return map[string]string{}, nil - } accessToken := channel.StringConfigForKey(courier.ConfigAuthToken, "") if accessToken == "" { diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go index b18ab10f3..217ccb2ac 100644 --- a/handlers/instagram/instagram_test.go +++ b/handlers/instagram/instagram_test.go @@ -110,7 +110,7 @@ var attachment = `{ "attachments":[{ "type":"image", "payload":{ - "url":"https://image-url/foo.png" + "url":"https://image-url/foo.png" } }] }, @@ -320,8 +320,7 @@ func TestDescribe(t *testing.T) { urn urns.URN metadata map[string]string }{{"instagram:1337", map[string]string{"name": "John Doe"}}, - {"instagram:4567", map[string]string{"name": ""}}, - {"instagram:ref:1337", map[string]string{}}} + {"instagram:4567", map[string]string{"name": ""}}} for _, tc := range tcs { metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) @@ -380,13 +379,6 @@ var defaultSendTestCases = []ChannelSendTestCase{ RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, - {Label: "Plain Send using ref URN", - Text: "Simple Message", URN: "instagram:ref:67890", - ContactURNs: map[string]bool{"instagram:12345": true, "ext:67890": true, "instagram:ref:67890": false}, - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133", "recipient_id": "12345"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"UPDATE","recipient":{"user_ref":"67890"},"message":{"text":"Simple Message"}}`, - SendPrep: setSendURL}, {Label: "Long Message", Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", From 86f48fef11117f44abfb10d7e8751b39c79ab0eb Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 3 Dec 2021 16:55:04 -0300 Subject: [PATCH 020/146] Add environment variables to instagram --- config.go | 94 +++++++++++++++------------- handlers/instagram/instagram.go | 6 +- handlers/instagram/instagram_test.go | 6 +- handlers/test.go | 2 + 4 files changed, 57 insertions(+), 51 deletions(-) diff --git a/config.go b/config.go index 5e6447690..336664e96 100644 --- a/config.go +++ b/config.go @@ -4,31 +4,33 @@ import "github.com/nyaruka/ezconf" // Config is our top level configuration object type Config struct { - Backend string `help:"the backend that will be used by courier (currently only rapidpro is supported)"` - SentryDSN string `help:"the DSN used for logging errors to Sentry"` - Domain string `help:"the domain courier is exposed on"` - Address string `help:"the network interface address courier will bind to"` - Port int `help:"the port courier will listen on"` - DB string `help:"URL describing how to connect to the RapidPro database"` - Redis string `help:"URL describing how to connect to Redis"` - SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"` - S3Endpoint string `help:"the S3 endpoint we will write attachments to"` - S3Region string `help:"the S3 region we will write attachments to"` - S3MediaBucket string `help:"the S3 bucket we will write attachments to"` - S3MediaPrefix string `help:"the prefix that will be added to attachment filenames"` - S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"` - S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"` - AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` - AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` - FacebookApplicationSecret string `help:"the Facebook app secret"` - FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` - MaxWorkers int `help:"the maximum number of go routines that will be used for sending (set to 0 to disable sending)"` - LibratoUsername string `help:"the username that will be used to authenticate to Librato"` - LibratoToken string `help:"the token that will be used to authenticate to Librato"` - StatusUsername string `help:"the username that is needed to authenticate against the /status endpoint"` - StatusPassword string `help:"the password that is needed to authenticate against the /status endpoint"` - LogLevel string `help:"the logging level courier should use"` - Version string `help:"the version that will be used in request and response headers"` + Backend string `help:"the backend that will be used by courier (currently only rapidpro is supported)"` + SentryDSN string `help:"the DSN used for logging errors to Sentry"` + Domain string `help:"the domain courier is exposed on"` + Address string `help:"the network interface address courier will bind to"` + Port int `help:"the port courier will listen on"` + DB string `help:"URL describing how to connect to the RapidPro database"` + Redis string `help:"URL describing how to connect to Redis"` + SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"` + S3Endpoint string `help:"the S3 endpoint we will write attachments to"` + S3Region string `help:"the S3 region we will write attachments to"` + S3MediaBucket string `help:"the S3 bucket we will write attachments to"` + S3MediaPrefix string `help:"the prefix that will be added to attachment filenames"` + S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"` + S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"` + AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` + AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` + FacebookApplicationSecret string `help:"the Facebook app secret"` + FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` + InstagramApplicationSecret string `help:"the Instagram app secret"` + InstagramWebhookSecret string `help:"the secret for Instagram webhook URL verification"` + MaxWorkers int `help:"the maximum number of go routines that will be used for sending (set to 0 to disable sending)"` + LibratoUsername string `help:"the username that will be used to authenticate to Librato"` + LibratoToken string `help:"the token that will be used to authenticate to Librato"` + StatusUsername string `help:"the username that is needed to authenticate against the /status endpoint"` + StatusPassword string `help:"the password that is needed to authenticate against the /status endpoint"` + LogLevel string `help:"the logging level courier should use"` + Version string `help:"the version that will be used in request and response headers"` // IncludeChannels is the list of channels to enable, empty means include all IncludeChannels []string @@ -40,26 +42,28 @@ type Config struct { // NewConfig returns a new default configuration object func NewConfig() *Config { return &Config{ - Backend: "rapidpro", - Domain: "localhost", - Address: "", - Port: 8080, - DB: "postgres://temba:temba@localhost/temba?sslmode=disable", - Redis: "redis://localhost:6379/15", - SpoolDir: "/var/spool/courier", - S3Endpoint: "https://s3.amazonaws.com", - S3Region: "us-east-1", - S3MediaBucket: "courier-media", - S3MediaPrefix: "/media/", - S3DisableSSL: false, - S3ForcePathStyle: false, - AWSAccessKeyID: "", - AWSSecretAccessKey: "", - FacebookApplicationSecret: "missing_facebook_app_secret", - FacebookWebhookSecret: "missing_facebook_webhook_secret", - MaxWorkers: 32, - LogLevel: "error", - Version: "Dev", + Backend: "rapidpro", + Domain: "localhost", + Address: "", + Port: 8080, + DB: "postgres://temba:temba@localhost/temba?sslmode=disable", + Redis: "redis://localhost:6379/15", + SpoolDir: "/var/spool/courier", + S3Endpoint: "https://s3.amazonaws.com", + S3Region: "us-east-1", + S3MediaBucket: "courier-media", + S3MediaPrefix: "/media/", + S3DisableSSL: false, + S3ForcePathStyle: false, + AWSAccessKeyID: "", + AWSSecretAccessKey: "", + FacebookApplicationSecret: "missing_facebook_app_secret", + FacebookWebhookSecret: "missing_facebook_webhook_secret", + InstagramApplicationSecret: "missing_instagram_app_secret", + InstagramWebhookSecret: "missing_instagram_webhook_secret", + MaxWorkers: 32, + LogLevel: "error", + Version: "Dev", } } diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go index 9c7da6198..f51ec1cef 100644 --- a/handlers/instagram/instagram.go +++ b/handlers/instagram/instagram.go @@ -165,10 +165,10 @@ func (h *handler) receiveVerify(ctx context.Context, channel courier.Channel, w return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("unknown request")) } - // verify the token against our server facebook webhook secret, if the same return the challenge IG sent us + // verify the token against our server instagram webhook secret, if the same return the challenge IG sent us secret := r.URL.Query().Get("hub.verify_token") - if secret != h.Server().Config().FacebookWebhookSecret { + if secret != h.Server().Config().InstagramWebhookSecret { return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("token does not match secret")) } // and respond with the challenge token @@ -473,7 +473,7 @@ func (h *handler) validateSignature(r *http.Request) error { if headerSignature == "" { return fmt.Errorf("missing request signature") } - appSecret := h.Server().Config().FacebookApplicationSecret + appSecret := h.Server().Config().InstagramApplicationSecret body, err := handlers.ReadBody(r, 100000) if err != nil { diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go index 217ccb2ac..249f0e0b3 100644 --- a/handlers/instagram/instagram_test.go +++ b/handlers/instagram/instagram_test.go @@ -278,7 +278,7 @@ var testCases = []ChannelHandleTestCase{ func addValidSignature(r *http.Request) { body, _ := handlers.ReadBody(r, 100000) - sig, _ := fbCalculateSignature("fb_app_secret", body) + sig, _ := fbCalculateSignature("ig_app_secret", body) r.Header.Set(signatureHeader, fmt.Sprintf("sha1=%s", string(sig))) } @@ -342,12 +342,12 @@ func BenchmarkHandler(b *testing.B) { func TestVerify(t *testing.T) { RunChannelTestCases(t, testChannels, newHandler(), []ChannelHandleTestCase{ - {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=ig_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, {Label: "Verify No Mode", URL: "/c/ig/receive", Status: 400, Response: "unknown request"}, {Label: "Verify No Secret", URL: "/c/ig/receive?hub.mode=subscribe", Status: 400, Response: "token does not match secret"}, {Label: "Invalid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=blah", Status: 400, Response: "token does not match secret"}, - {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=ig_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, }) } diff --git a/handlers/test.go b/handlers/test.go index 9f5a88e3a..bd1354628 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -200,6 +200,8 @@ func newServer(backend courier.Backend) courier.Server { config := courier.NewConfig() config.FacebookWebhookSecret = "fb_webhook_secret" config.FacebookApplicationSecret = "fb_app_secret" + config.InstagramWebhookSecret = "ig_webhook_secret" + config.InstagramApplicationSecret = "ig_app_secret" return courier.NewServerWithLogger(config, backend, logger) From 2cf769958fc55a8868ba36e256a4de1f5a55fd68 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 16 Dec 2021 16:39:10 -0300 Subject: [PATCH 021/146] fix: Metadata search for a new contact --- handlers/instagram/instagram.go | 8 +++----- handlers/instagram/instagram_test.go | 10 ++++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go index f51ec1cef..fab5ab6be 100644 --- a/handlers/instagram/instagram.go +++ b/handlers/instagram/instagram.go @@ -451,7 +451,6 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn u := base.ResolveReference(path) query := url.Values{} - query.Set("fields", "first_name,last_name") query.Set("access_token", accessToken) u.RawQuery = query.Encode() req, _ := http.NewRequest(http.MethodGet, u.String(), nil) @@ -460,11 +459,10 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) } - // read our first and last name - firstName, _ := jsonparser.GetString(rr.Body, "first_name") - lastName, _ := jsonparser.GetString(rr.Body, "last_name") + // read our name + name, _ := jsonparser.GetString(rr.Body, "name") - return map[string]string{"name": utils.JoinNonEmpty(" ", firstName, lastName)}, nil + return map[string]string{"name": name}, nil } // see https://developers.facebook.com/docs/messenger-platform/webhook#security diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go index 249f0e0b3..0d891554d 100644 --- a/handlers/instagram/instagram_test.go +++ b/handlers/instagram/instagram_test.go @@ -299,12 +299,12 @@ func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { // user has a name if strings.HasSuffix(r.URL.Path, "1337") { - w.Write([]byte(`{ "first_name": "John", "last_name": "Doe"}`)) + w.Write([]byte(`{ "name": "John Doe"}`)) return } // no name - w.Write([]byte(`{ "first_name": "", "last_name": ""}`)) + w.Write([]byte(`{ "name": ""}`)) })) graphURL = server.URL @@ -319,8 +319,10 @@ func TestDescribe(t *testing.T) { tcs := []struct { urn urns.URN metadata map[string]string - }{{"instagram:1337", map[string]string{"name": "John Doe"}}, - {"instagram:4567", map[string]string{"name": ""}}} + }{ + {"instagram:1337", map[string]string{"name": "John Doe"}}, + {"instagram:4567", map[string]string{"name": ""}}, + } for _, tc := range tcs { metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) From 3cfcb5a3238756f5d2c50d7d541efe29391c694f Mon Sep 17 00:00:00 2001 From: Matheus Soares Date: Fri, 17 Dec 2021 11:23:50 -0300 Subject: [PATCH 022/146] add import for instagram handler --- cmd/courier/main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/courier/main.go b/cmd/courier/main.go index 5c44ef92f..62f0afee5 100644 --- a/cmd/courier/main.go +++ b/cmd/courier/main.go @@ -34,6 +34,7 @@ import ( _ "github.com/nyaruka/courier/handlers/hub9" _ "github.com/nyaruka/courier/handlers/i2sms" _ "github.com/nyaruka/courier/handlers/infobip" + _ "github.com/nyaruka/courier/handlers/instagram" _ "github.com/nyaruka/courier/handlers/jasmin" _ "github.com/nyaruka/courier/handlers/jiochat" _ "github.com/nyaruka/courier/handlers/junebug" From 7965bd8f85a645777e432a9cb5b65f417f506794 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 17 Dec 2021 12:40:55 -0300 Subject: [PATCH 023/146] Refactor response field to external ID --- handlers/instagram/instagram.go | 35 +--------------------------- handlers/instagram/instagram_test.go | 2 +- 2 files changed, 2 insertions(+), 35 deletions(-) diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go index fab5ab6be..1cdea4a2e 100644 --- a/handlers/instagram/instagram.go +++ b/handlers/instagram/instagram.go @@ -71,23 +71,6 @@ type igUser struct { ID string `json:"id"` } -// { -// "object":"instagram", -// "entry":[{ -// "id":"180005062406476", -// "time":1514924367082, -// "messaging":[{ -// "sender": {"id":"1630934236957797"}, -// "recipient":{"id":"180005062406476"}, -// "timestamp":1514924366807, -// "message":{ -// "mid":"mid.$cAAD5QiNHkz1m6cyj11guxokwkhi2", -// "text":"65863634" -// } -// }] -// }] -// } - type moPayload struct { Object string `json:"object"` Entry []struct { @@ -298,22 +281,6 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h return events, courier.WriteDataResponse(ctx, w, http.StatusOK, "Events Handled", data) } -// { -// "messaging_type": "" -// "recipient":{ -// "id":"" -// }, -// "message":{ -// "text":"hello, world!" -// "attachment":{ -// "type":"image", -// "payload":{ -// "url":"http://www.messenger-rocks.com/image.jpg", -// "is_reusable":true -// } -// } -// } -// } type mtPayload struct { MessagingType string `json:"messaging_type"` Tag string `json:"tag,omitempty"` @@ -351,7 +318,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload := mtPayload{} // set our message type - if msg.ResponseToID() != courier.NilMsgID { + if msg.ResponseToExternalID() != "" { payload.MessagingType = "RESPONSE" } else if topic != "" { payload.MessagingType = "MESSAGE_TAG" diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go index 0d891554d..648a3ed68 100644 --- a/handlers/instagram/instagram_test.go +++ b/handlers/instagram/instagram_test.go @@ -369,7 +369,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ {Label: "Plain Response", Text: "Simple Message", URN: "instagram:12345", - Status: "W", ExternalID: "mid.133", ResponseToID: 23526, + Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, From d27ee1aa6638e005e026288da81e2207a9194977 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 23 Dec 2021 15:09:24 -0300 Subject: [PATCH 024/146] feat: Add Instagram channel support to Facebook handler --- cmd/courier/main.go | 1 - handlers/facebookapp/facebookapp.go | 64 +-- handlers/facebookapp/facebookapp_test.go | 533 +++++++++++++++++++---- 3 files changed, 483 insertions(+), 115 deletions(-) diff --git a/cmd/courier/main.go b/cmd/courier/main.go index 62f0afee5..5c44ef92f 100644 --- a/cmd/courier/main.go +++ b/cmd/courier/main.go @@ -34,7 +34,6 @@ import ( _ "github.com/nyaruka/courier/handlers/hub9" _ "github.com/nyaruka/courier/handlers/i2sms" _ "github.com/nyaruka/courier/handlers/infobip" - _ "github.com/nyaruka/courier/handlers/instagram" _ "github.com/nyaruka/courier/handlers/jasmin" _ "github.com/nyaruka/courier/handlers/jiochat" _ "github.com/nyaruka/courier/handlers/junebug" diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 77982f4cb..3781adce8 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -23,13 +23,13 @@ import ( // Endpoints we hit var ( - sendURL = "https://graph.facebook.com/v7.0/me/messages" - graphURL = "https://graph.facebook.com/v7.0/" + sendURL = "https://graph.facebook.com/v12.0/me/messages" + graphURL = "https://graph.facebook.com/v12.0/" signatureHeader = "X-Hub-Signature" - // Facebook API says 640 is max for the body - maxMsgLength = 640 + // max for the body + maxMsgLength = 1000 // Sticker ID substitutions stickerIDToEmoji = map[int64]string{ @@ -56,18 +56,20 @@ const ( payloadKey = "payload" ) +func newHandler(channelType courier.ChannelType, name string, validateSignatures bool) courier.ChannelHandler { + return &handler{handlers.NewBaseHandlerWithParams(channelType, name, validateSignatures)} +} + func init() { - courier.RegisterHandler(newHandler()) + courier.RegisterHandler(newHandler("IG", "Instagram", false)) + courier.RegisterHandler(newHandler("FBA", "Facebook", false)) + } type handler struct { handlers.BaseHandler } -func newHandler() courier.ChannelHandler { - return &handler{handlers.NewBaseHandlerWithParams(courier.ChannelType("FBA"), "Facebook", false)} -} - // Initialize is called by the engine once everything is loaded func (h *handler) Initialize(s courier.Server) error { h.SetServer(s) @@ -76,12 +78,12 @@ func (h *handler) Initialize(s courier.Server) error { return nil } -type fbSender struct { +type Sender struct { ID string `json:"id"` - UserRef string `json:"user_ref"` + UserRef string `json:"user_ref,omitempty"` } -type fbUser struct { +type User struct { ID string `json:"id"` } @@ -108,9 +110,9 @@ type moPayload struct { ID string `json:"id"` Time int64 `json:"time"` Messaging []struct { - Sender fbSender `json:"sender"` - Recipient fbUser `json:"recipient"` - Timestamp int64 `json:"timestamp"` + Sender Sender `json:"sender"` + Recipient User `json:"recipient"` + Timestamp int64 `json:"timestamp"` OptIn *struct { Ref string `json:"ref"` @@ -125,6 +127,7 @@ type moPayload struct { } `json:"referral"` Postback *struct { + MID string `json:"mid"` Title string `json:"title"` Payload string `json:"payload"` Referral struct { @@ -172,9 +175,9 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan return nil, err } - // not a page object? ignore - if payload.Object != "page" { - return nil, fmt.Errorf("object expected 'page', found %s", payload.Object) + // is not a 'page' and 'instagram' object? ignore it + if payload.Object != "page" && payload.Object != "instagram" { + return nil, fmt.Errorf("object expected 'page' or 'instagram', found %s", payload.Object) } // no entries? ignore this request @@ -182,9 +185,14 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan return nil, fmt.Errorf("no entries found") } - pageID := payload.Entry[0].ID + EntryID := payload.Entry[0].ID - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(pageID)) + //if object is 'page' returns type FBA, if object is 'instagram' returns type IG + if payload.Object == "page" { + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(EntryID)) + } else { + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(EntryID)) + } } // receiveVerify handles Facebook's webhook verification callback @@ -219,9 +227,9 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } - // not a page object? ignore - if payload.Object != "page" { - return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring non-page request") + // // is not a 'page' and 'instagram' object? ignore it + if payload.Object != "page" && payload.Object != "instagram" { + return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request") } // no entries? ignore this request @@ -494,7 +502,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload.MessagingType = "MESSAGE_TAG" payload.Tag = tagByTopic[topic] } else { - payload.MessagingType = "NON_PROMOTIONAL_SUBSCRIPTION" // only allowed until Jan 15, 2020 + payload.MessagingType = "UPDATE" } // build our recipient @@ -641,7 +649,6 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn u := base.ResolveReference(path) query := url.Values{} - query.Set("fields", "first_name,last_name") query.Set("access_token", accessToken) u.RawQuery = query.Encode() req, _ := http.NewRequest(http.MethodGet, u.String(), nil) @@ -650,11 +657,10 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) } - // read our first and last name - firstName, _ := jsonparser.GetString(rr.Body, "first_name") - lastName, _ := jsonparser.GetString(rr.Body, "last_name") + // read our name + name, _ := jsonparser.GetString(rr.Body, "name") - return map[string]string{"name": utils.JoinNonEmpty(" ", firstName, lastName)}, nil + return map[string]string{"name": name}, nil } // see https://developers.facebook.com/docs/messenger-platform/webhook#security diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 5514c7114..008a2d840 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -17,20 +17,45 @@ import ( ) var testChannels = []courier.Channel{ - courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "FBA", "1234", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), } -var helloMsg = `{ +var testChannelsIG = []courier.Channel{ + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), +} + +var helloMsgFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var helloMsgIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -41,17 +66,55 @@ var helloMsg = `{ }] }` -var duplicateMsg = `{ +var duplicateMsgFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }, + { + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var duplicateMsgIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -61,14 +124,14 @@ var duplicateMsg = `{ "time": 1459991487970 }, { - "id": "1234", + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -79,17 +142,38 @@ var duplicateMsg = `{ }] }` -var invalidURN = `{ +var invalidURNFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "abc5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var invalidURNIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "message": { "text": "Hello World", "mid": "external_id" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "abc5678" @@ -100,10 +184,10 @@ var invalidURN = `{ }] }` -var attachment = `{ +var attachmentFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "message": { "mid": "external_id", @@ -115,7 +199,33 @@ var attachment = `{ }] }, "recipient": { - "id": "1234" + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var attachmentIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "message": { + "mid": "external_id", + "attachments":[{ + "type":"image", + "payload":{ + "url":"https://image-url/foo.png" + } + }] + }, + "recipient": { + "id": "12345" }, "sender": { "id": "5678" @@ -129,7 +239,7 @@ var attachment = `{ var locationAttachment = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "message": { "mid": "external_id", @@ -144,7 +254,7 @@ var locationAttachment = `{ }] }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -158,11 +268,11 @@ var locationAttachment = `{ var thumbsUp = `{ "object":"page", "entry":[{ - "id":"1234", + "id":"12345", "time":1459991487970, "messaging":[{ "sender":{"id":"5678"}, - "recipient":{"id":"1234"}, + "recipient":{"id":"12345"}, "timestamp":1459991487970, "message":{ "mid":"external_id", @@ -178,10 +288,50 @@ var thumbsUp = `{ }] }` -var differentPage = `{ +var like_heart = `{ + "object":"instagram", + "entry":[{ + "id":"12345", + "messaging":[{ + "sender":{"id":"5678"}, + "recipient":{"id":"12345"}, + "timestamp":1459991487970, + "message":{ + "mid":"external_id", + "attachments":[{ + "type":"like_heart" + }] + } + }], + "time":1459991487970 + }] +}` + +var differentPageIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1235" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + +var differentPageFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "message": { "text": "Hello World", @@ -199,13 +349,33 @@ var differentPage = `{ }] }` -var echo = `{ +var echoFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970, + "message": { + "is_echo": true, + "mid": "qT7ywaK" + } + }] + }] +}` + +var echoIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -219,17 +389,38 @@ var echo = `{ }] }` +var icebreakerGetStarted = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "postback": { + "title": "icebreaker question", + "payload": "get_started" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + var optInUserRef = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "optin": { "ref": "optin_ref", "user_ref": "optin_user_ref" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -243,13 +434,13 @@ var optInUserRef = `{ var optIn = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "optin": { "ref": "optin_ref" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -263,7 +454,7 @@ var optIn = `{ var postback = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "postback": { "title": "postback title", @@ -275,7 +466,7 @@ var postback = `{ } }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -289,7 +480,7 @@ var postback = `{ var postbackReferral = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "postback": { "title": "postback title", @@ -302,7 +493,7 @@ var postbackReferral = `{ } }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -316,14 +507,14 @@ var postbackReferral = `{ var postbackGetStarted = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "postback": { "title": "postback title", "payload": "get_started" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -337,7 +528,7 @@ var postbackGetStarted = `{ var referral = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "referral": { "ref": "referral id", @@ -346,7 +537,7 @@ var referral = `{ "type": "referral type" }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678", @@ -361,7 +552,7 @@ var referral = `{ var dlr = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", "messaging": [{ "delivery":{ "mids":[ @@ -371,7 +562,7 @@ var dlr = `{ "seq":37 }, "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -387,25 +578,58 @@ var notPage = `{ "entry": [{}] }` -var noEntries = `{ +var notInstagram = `{ + "object":"notinstagram", + "entry": [{}] +}` + +var noEntriesFBA = `{ "object":"page", "entry": [] }` -var noMessagingEntries = `{ +var noEntriesIG = `{ + "object":"instagram", + "entry": [] +}` + +var noMessagingEntriesFBA = `{ "object":"page", "entry": [{ - "id": "1234" + "id": "12345" + }] +}` + +var noMessagingEntriesIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345" }] }` -var unkownMessagingEntry = `{ +var unkownMessagingEntryFBA = `{ "object":"page", "entry": [{ - "id": "1234", + "id": "12345", + "messaging": [{ + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }] + }] +}` + +var unkownMessagingEntryIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", "messaging": [{ "recipient": { - "id": "1234" + "id": "12345" }, "sender": { "id": "5678" @@ -417,17 +641,16 @@ var unkownMessagingEntry = `{ var notJSON = `blargh` -var testCases = []ChannelHandleTestCase{ - {Label: "Receive Message", URL: "/c/fba/receive", Data: helloMsg, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, +var testCasesFBA = []ChannelHandleTestCase{ + {Label: "Receive Message FBA", URL: "/c/fba/receive", Data: helloMsgFBA, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, + {Label: "Receive Invalid Signature", URL: "/c/fba/receive", Data: helloMsgFBA, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, - {Label: "Receive Invalid Signature", URL: "/c/fba/receive", Data: helloMsg, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, - - {Label: "No Duplicate Receive Message", URL: "/c/fba/receive", Data: duplicateMsg, Status: 200, Response: "Handled", + {Label: "No Duplicate Receive Message", URL: "/c/fba/receive", Data: duplicateMsgFBA, Status: 200, Response: "Handled", Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Attachment", URL: "/c/fba/receive", Data: attachment, Status: 200, Response: "Handled", + {Label: "Receive Attachment", URL: "/c/fba/receive", Data: attachmentFBA, Status: 200, Response: "Handled", Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, @@ -469,14 +692,47 @@ var testCases = []ChannelHandleTestCase{ Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), MsgStatus: Sp(courier.MsgDelivered), ExternalID: Sp("mid.1458668856218:ed81099e15d3f4f233"), PrepRequest: addValidSignature}, - {Label: "Different Page", URL: "/c/fba/receive", Data: differentPage, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, - {Label: "Echo", URL: "/c/fba/receive", Data: echo, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, - {Label: "Not Page", URL: "/c/fba/receive", Data: notPage, Status: 400, Response: "expected 'page', found notpage", PrepRequest: addValidSignature}, - {Label: "No Entries", URL: "/c/fba/receive", Data: noEntries, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, - {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: noMessagingEntries, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unkownMessagingEntry, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Different Page", URL: "/c/fba/receive", Data: differentPageFBA, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, + {Label: "Echo", URL: "/c/fba/receive", Data: echoFBA, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, + {Label: "Not Page", URL: "/c/fba/receive", Data: notPage, Status: 400, Response: "object expected 'page' or 'instagram', found notpage", PrepRequest: addValidSignature}, + {Label: "No Entries", URL: "/c/fba/receive", Data: noEntriesFBA, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, + {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: noMessagingEntriesFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unkownMessagingEntryFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/fba/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, - {Label: "Invalid URN", URL: "/c/fba/receive", Data: invalidURN, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/fba/receive", Data: invalidURNFBA, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, +} +var testCasesIG = []ChannelHandleTestCase{ + {Label: "Receive Message", URL: "/c/ig/receive", Data: helloMsgIG, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Invalid Signature", URL: "/c/ig/receive", Data: helloMsgIG, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, + + {Label: "No Duplicate Receive Message", URL: "/c/ig/receive", Data: duplicateMsgIG, Status: 200, Response: "Handled", + Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Attachment", URL: "/c/ig/receive", Data: attachmentIG, Status: 200, Response: "Handled", + Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Like Heart", URL: "/c/ig/receive", Data: like_heart, Status: 200, Response: "Handled", + Text: Sp(""), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Icebreaker Get Started", URL: "/c/ig/receive", Data: icebreakerGetStarted, Status: 200, Response: "Handled", + URN: Sp("facebook:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), + ChannelEventExtra: map[string]interface{}{"title": "icebreaker question", "payload": "get_started"}, + PrepRequest: addValidSignature}, + + {Label: "Different Page", URL: "/c/ig/receive", Data: differentPageIG, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, + {Label: "Echo", URL: "/c/ig/receive", Data: echoIG, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, + {Label: "No Entries", URL: "/c/ig/receive", Data: noEntriesIG, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, + {Label: "Not Instagram", URL: "/c/ig/receive", Data: notInstagram, Status: 400, Response: "object expected 'page' or 'instagram', found notinstagram", PrepRequest: addValidSignature}, + {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntriesIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, } func addValidSignature(r *http.Request) { @@ -502,12 +758,12 @@ func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { // user has a name if strings.HasSuffix(r.URL.Path, "1337") { - w.Write([]byte(`{ "first_name": "John", "last_name": "Doe"}`)) + w.Write([]byte(`{ "name": "John Doe"}`)) return } // no name - w.Write([]byte(`{ "first_name": "", "last_name": ""}`)) + w.Write([]byte(`{ "name": ""}`)) })) graphURL = server.URL @@ -515,37 +771,69 @@ func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { } func TestDescribe(t *testing.T) { - fbGraph := buildMockFBGraph(testCases) - defer fbGraph.Close() + var testCases [][]ChannelHandleTestCase + testCases = append(testCases, testCasesFBA) + testCases = append(testCases, testCasesIG) + + for i, tc := range testCases { + fbGraph := buildMockFBGraph(tc) + defer fbGraph.Close() + + if i == 0 { + handler := newHandler("FBA", "Facebook", false).(courier.URNDescriber) + tcs := []struct { + urn urns.URN + metadata map[string]string + }{ + {"facebook:1337", map[string]string{"name": "John Doe"}}, + {"facebook:4567", map[string]string{"name": ""}}, + } + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) + } + } else { + handler := newHandler("IG", "Instagram", false).(courier.URNDescriber) + tcs := []struct { + urn urns.URN + metadata map[string]string + }{ + {"facebook:1337", map[string]string{"name": "John Doe"}}, + {"facebook:4567", map[string]string{"name": ""}}, + } + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannelsIG[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) + } + } - handler := newHandler().(courier.URNDescriber) - tcs := []struct { - urn urns.URN - metadata map[string]string - }{{"facebook:1337", map[string]string{"name": "John Doe"}}, - {"facebook:4567", map[string]string{"name": ""}}, - {"facebook:ref:1337", map[string]string{}}} - - for _, tc := range tcs { - metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) - assert.Equal(t, metadata, tc.metadata) } + } func TestHandler(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler(), testCases) + RunChannelTestCases(t, testChannels, newHandler("FBA", "Facebook", false), testCasesFBA) + RunChannelTestCases(t, testChannelsIG, newHandler("IG", "Instagram", false), testCasesIG) + } func BenchmarkHandler(b *testing.B) { - fbService := buildMockFBGraph(testCases) - defer fbService.Close() + fbService := buildMockFBGraph(testCasesFBA) - RunChannelBenchmarks(b, testChannels, newHandler(), testCases) + RunChannelBenchmarks(b, testChannels, newHandler("FBA", "Facebook", false), testCasesFBA) + fbService.Close() + + fbServiceIG := buildMockFBGraph(testCasesIG) + + RunChannelBenchmarks(b, testChannelsIG, newHandler("IG", "Instagram", false), testCasesIG) + fbServiceIG.Close() } func TestVerify(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler(), []ChannelHandleTestCase{ + RunChannelTestCases(t, testChannels, newHandler("FBA", "Facebook", false), []ChannelHandleTestCase{ {Label: "Valid Secret", URL: "/c/fba/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, {Label: "Verify No Mode", URL: "/c/fba/receive", Status: 400, Response: "unknown request"}, @@ -554,6 +842,15 @@ func TestVerify(t *testing.T) { {Label: "Valid Secret", URL: "/c/fba/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, }) + RunChannelTestCases(t, testChannelsIG, newHandler("IG", "Instagram", false), []ChannelHandleTestCase{ + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, + Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, + {Label: "Verify No Mode", URL: "/c/ig/receive", Status: 400, Response: "unknown request"}, + {Label: "Verify No Secret", URL: "/c/ig/receive?hub.mode=subscribe", Status: 400, Response: "token does not match secret"}, + {Label: "Invalid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=blah", Status: 400, Response: "token does not match secret"}, + {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, + }) + } // setSendURL takes care of setting the send_url to our test server host @@ -561,12 +858,12 @@ func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, sendURL = s.URL } -var defaultSendTestCases = []ChannelSendTestCase{ +var SendTestCasesFBA = []ChannelSendTestCase{ {Label: "Plain Send", Text: "Simple Message", URN: "facebook:12345", Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Plain Response", Text: "Simple Message", URN: "facebook:12345", @@ -579,13 +876,13 @@ var defaultSendTestCases = []ChannelSendTestCase{ ContactURNs: map[string]bool{"facebook:12345": true, "ext:67890": true, "facebook:ref:67890": false}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133", "recipient_id": "12345"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"user_ref":"67890"},"message":{"text":"Simple Message"}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"user_ref":"67890"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Quick Reply", Text: "Are you happy?", URN: "facebook:12345", QuickReplies: []string{"Yes", "No"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, SendPrep: setSendURL}, {Label: "Long Message", Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", @@ -598,7 +895,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "facebook:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, SendPrep: setSendURL}, {Label: "Send caption and photo with Quick Reply", Text: "This is some text.", @@ -612,7 +909,71 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "facebook:12345", Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"NON_PROMOTIONAL_SUBSCRIPTION","recipient":{"id":"12345"},"message":{"attachment":{"type":"file","payload":{"url":"https://foo.bar/document.pdf","is_reusable":true}}}}`, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"file","payload":{"url":"https://foo.bar/document.pdf","is_reusable":true}}}}`, + SendPrep: setSendURL}, + {Label: "ID Error", + Text: "ID Error", URN: "facebook:12345", + Status: "E", + ResponseBody: `{ "is_error": true }`, ResponseStatus: 200, + SendPrep: setSendURL}, + {Label: "Error", + Text: "Error", URN: "facebook:12345", + Status: "E", + ResponseBody: `{ "is_error": true }`, ResponseStatus: 403, + SendPrep: setSendURL}, +} + +var SendTestCasesIG = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "facebook:12345", + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Plain Response", + Text: "Simple Message", URN: "facebook:12345", + Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Quick Reply", + Text: "Are you happy?", URN: "facebook:12345", QuickReplies: []string{"Yes", "No"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + {Label: "Long Message", + Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", + URN: "facebook:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"we exceed the max length?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + {Label: "Send Photo", + URN: "facebook:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, + SendPrep: setSendURL}, + {Label: "Send caption and photo with Quick Reply", + Text: "This is some text.", + URN: "facebook:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + QuickReplies: []string{"Yes", "No"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"This is some text.","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, + SendPrep: setSendURL}, + {Label: "Tag Human Agent", + Text: "Simple Message", URN: "facebook:12345", + Status: "W", ExternalID: "mid.133", Topic: "agent", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Send Document", + URN: "facebook:12345", Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + Status: "W", ExternalID: "mid.133", + ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, + RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"file","payload":{"url":"https://foo.bar/document.pdf","is_reusable":true}}}}`, SendPrep: setSendURL}, {Label: "ID Error", Text: "ID Error", URN: "facebook:12345", @@ -629,8 +990,10 @@ var defaultSendTestCases = []ChannelSendTestCase{ func TestSending(t *testing.T) { // shorter max msg length for testing maxMsgLength = 100 - var defaultChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "FBA", "2020", "US", map[string]interface{}{courier.ConfigAuthToken: "access_token"}) - RunChannelSendTestCases(t, defaultChannel, newHandler(), defaultSendTestCases, nil) + var ChannelFBA = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) + var ChannelIG = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) + RunChannelSendTestCases(t, ChannelFBA, newHandler("FBA", "Facebook", false), SendTestCasesFBA, nil) + RunChannelSendTestCases(t, ChannelIG, newHandler("IG", "Instagram", false), SendTestCasesIG, nil) } func TestSigning(t *testing.T) { From b914672dd4e227d3000899be332849987bdd04d1 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 23 Dec 2021 17:56:29 -0300 Subject: [PATCH 025/146] Remove unused instagram handler files --- handlers/instagram/instagram.go | 475 --------------------------- handlers/instagram/instagram_test.go | 455 ------------------------- 2 files changed, 930 deletions(-) delete mode 100644 handlers/instagram/instagram.go delete mode 100644 handlers/instagram/instagram_test.go diff --git a/handlers/instagram/instagram.go b/handlers/instagram/instagram.go deleted file mode 100644 index 1cdea4a2e..000000000 --- a/handlers/instagram/instagram.go +++ /dev/null @@ -1,475 +0,0 @@ -package instagram - -import ( - "bytes" - "context" - "crypto/hmac" - "crypto/sha1" - "encoding/hex" - "encoding/json" - "fmt" - "net/http" - "net/url" - "strings" - "time" - - "github.com/buger/jsonparser" - "github.com/nyaruka/courier" - "github.com/nyaruka/courier/handlers" - "github.com/nyaruka/courier/utils" - "github.com/nyaruka/gocommon/urns" - "github.com/pkg/errors" -) - -// Endpoints we hit -var ( - sendURL = "https://graph.facebook.com/v12.0/me/messages" - graphURL = "https://graph.facebook.com/v12.0/" - - signatureHeader = "X-Hub-Signature" - - // max for the body - maxMsgLength = 1000 - - //Only Human_Agent tag available for instagram - tagByTopic = map[string]string{ - "agent": "HUMAN_AGENT", - } -) - -// keys for extra in channel events -const ( - titleKey = "title" - payloadKey = "payload" -) - -func init() { - courier.RegisterHandler(newHandler()) -} - -type handler struct { - handlers.BaseHandler -} - -func newHandler() courier.ChannelHandler { - return &handler{handlers.NewBaseHandlerWithParams(courier.ChannelType("IG"), "Instagram", false)} -} - -// Initialize is called by the engine once everything is loaded -func (h *handler) Initialize(s courier.Server) error { - h.SetServer(s) - s.AddHandlerRoute(h, http.MethodGet, "receive", h.receiveVerify) - s.AddHandlerRoute(h, http.MethodPost, "receive", h.receiveEvent) - return nil -} - -type igSender struct { - ID string `json:"id"` -} - -type igUser struct { - ID string `json:"id"` -} - -type moPayload struct { - Object string `json:"object"` - Entry []struct { - ID string `json:"id"` - Time int64 `json:"time"` - Messaging []struct { - Sender igSender `json:"sender"` - Recipient igUser `json:"recipient"` - Timestamp int64 `json:"timestamp"` - - Postback *struct { - MID string `json:"mid"` - Title string `json:"title"` - Payload string `json:"payload"` - } `json:"postback,omitempty"` - - Message *struct { - IsEcho bool `json:"is_echo,omitempty"` - MID string `json:"mid"` - Text string `json:"text,omitempty"` - QuickReply struct { - Payload string `json:"payload"` - } `json:"quick_replies,omitempty"` - Attachments []struct { - Type string `json:"type"` - Payload *struct { - URL string `json:"url"` - } `json:"payload"` - } `json:"attachments,omitempty"` - } `json:"message,omitempty"` - } `json:"messaging"` - } `json:"entry"` -} - -// GetChannel returns the channel -func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Channel, error) { - - if r.Method == http.MethodGet { - - return nil, nil - } - - payload := &moPayload{} - - err := handlers.DecodeAndValidateJSON(payload, r) - - if err != nil { - - return nil, err - } - - // not a instagram object? ignore - if payload.Object != "instagram" { - - return nil, fmt.Errorf("object expected 'instagram', found %s", payload.Object) - } - - // no entries? ignore this request - if len(payload.Entry) == 0 { - - return nil, fmt.Errorf("no entries found") - } - - igID := payload.Entry[0].ID - - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(igID)) -} - -// receiveVerify handles Instagram's webhook verification callback -func (h *handler) receiveVerify(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { - mode := r.URL.Query().Get("hub.mode") - - // this isn't a subscribe verification, that's an error - if mode != "subscribe" { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("unknown request")) - } - - // verify the token against our server instagram webhook secret, if the same return the challenge IG sent us - secret := r.URL.Query().Get("hub.verify_token") - - if secret != h.Server().Config().InstagramWebhookSecret { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("token does not match secret")) - } - // and respond with the challenge token - _, err := fmt.Fprint(w, r.URL.Query().Get("hub.challenge")) - return nil, err -} - -// receiveEvent is our HTTP handler function for incoming messages and status updates -func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { - err := h.validateSignature(r) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) - } - - payload := &moPayload{} - err = handlers.DecodeAndValidateJSON(payload, r) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) - } - - // not a instagram object? ignore - if payload.Object != "instagram" { - return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request") - } - - // no entries? ignore this request - if len(payload.Entry) == 0 { - return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request, no entries") - } - - // the list of events we deal with - events := make([]courier.Event, 0, 2) - - // the list of data we will return in our response - data := make([]interface{}, 0, 2) - - // for each entry - for _, entry := range payload.Entry { - // no entry, ignore - if len(entry.Messaging) == 0 { - continue - } - - // grab our message, there is always a single one - msg := entry.Messaging[0] - - // ignore this entry if it is to another page - if channel.Address() != msg.Recipient.ID { - continue - } - - // create our date from the timestamp (they give us millis, arg is nanos) - date := time.Unix(0, msg.Timestamp*1000000).UTC() - - sender := msg.Sender.ID - if sender == "" { - sender = msg.Sender.ID - } - - // create our URN - urn, err := urns.NewInstagramURN(sender) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) - } - - if msg.Postback != nil { - // by default postbacks are treated as new conversations - eventType := courier.NewConversation - event := h.Backend().NewChannelEvent(channel, eventType, urn).WithOccurredOn(date) - - // build our extra - extra := map[string]interface{}{ - titleKey: msg.Postback.Title, - payloadKey: msg.Postback.Payload, - } - - event = event.WithExtra(extra) - - err := h.Backend().WriteChannelEvent(ctx, event) - if err != nil { - return nil, err - } - - events = append(events, event) - data = append(data, courier.NewEventReceiveData(event)) - } else if msg.Message != nil { - // this is an incoming message - // ignore echos - if msg.Message.IsEcho { - data = append(data, courier.NewInfoData("ignoring echo")) - continue - } - - text := msg.Message.Text - - attachmentURLs := make([]string, 0, 2) - - for _, att := range msg.Message.Attachments { - if att.Payload != nil && att.Payload.URL != "" { - attachmentURLs = append(attachmentURLs, att.Payload.URL) - } - } - - // create our message - ev := h.Backend().NewIncomingMsg(channel, urn, text).WithExternalID(msg.Message.MID).WithReceivedOn(date) - event := h.Backend().CheckExternalIDSeen(ev) - - // add any attachment URL found - for _, attURL := range attachmentURLs { - event.WithAttachment(attURL) - } - - err := h.Backend().WriteMsg(ctx, event) - if err != nil { - return nil, err - } - - h.Backend().WriteExternalIDSeen(event) - - events = append(events, event) - data = append(data, courier.NewMsgReceiveData(event)) - - } else { - data = append(data, courier.NewInfoData("ignoring unknown entry type")) - } - } - return events, courier.WriteDataResponse(ctx, w, http.StatusOK, "Events Handled", data) -} - -type mtPayload struct { - MessagingType string `json:"messaging_type"` - Tag string `json:"tag,omitempty"` - Recipient struct { - ID string `json:"id,omitempty"` - } `json:"recipient"` - Message struct { - Text string `json:"text,omitempty"` - QuickReplies []mtQuickReply `json:"quick_replies,omitempty"` - Attachment *mtAttachment `json:"attachment,omitempty"` - } `json:"message"` -} - -type mtAttachment struct { - Type string `json:"type"` - Payload struct { - URL string `json:"url,omitempty"` - IsReusable bool `json:"is_reusable,omitempty"` - } `json:"payload"` -} -type mtQuickReply struct { - Title string `json:"title"` - Payload string `json:"payload"` - ContentType string `json:"content_type"` -} - -func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { - // can't do anything without an access token - accessToken := msg.Channel().StringConfigForKey(courier.ConfigAuthToken, "") - if accessToken == "" { - return nil, fmt.Errorf("missing access token") - } - - topic := msg.Topic() - payload := mtPayload{} - - // set our message type - if msg.ResponseToExternalID() != "" { - payload.MessagingType = "RESPONSE" - } else if topic != "" { - payload.MessagingType = "MESSAGE_TAG" - payload.Tag = tagByTopic[topic] - } else { - payload.MessagingType = "UPDATE" - } - - payload.Recipient.ID = msg.URN().Path() - - msgURL, _ := url.Parse(sendURL) - query := url.Values{} - query.Set("access_token", accessToken) - msgURL.RawQuery = query.Encode() - - status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) - - msgParts := make([]string, 0) - if msg.Text() != "" { - msgParts = handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) - } - - // send each part and each attachment separately. we send attachments first as otherwise quick replies - // attached to text messages get hidden when images get delivered - for i := 0; i < len(msgParts)+len(msg.Attachments()); i++ { - if i < len(msg.Attachments()) { - // this is an attachment - payload.Message.Attachment = &mtAttachment{} - attType, attURL := handlers.SplitAttachment(msg.Attachments()[i]) - attType = strings.Split(attType, "/")[0] - payload.Message.Attachment.Type = attType - payload.Message.Attachment.Payload.URL = attURL - payload.Message.Attachment.Payload.IsReusable = true - payload.Message.Text = "" - } else { - // this is still a msg part - payload.Message.Text = msgParts[i-len(msg.Attachments())] - payload.Message.Attachment = nil - } - - // include any quick replies on the last piece we send - if i == (len(msgParts)+len(msg.Attachments()))-1 { - for _, qr := range msg.QuickReplies() { - payload.Message.QuickReplies = append(payload.Message.QuickReplies, mtQuickReply{qr, qr, "text"}) - } - } else { - payload.Message.QuickReplies = nil - } - - jsonBody, err := json.Marshal(payload) - if err != nil { - return status, err - } - - req, err := http.NewRequest(http.MethodPost, msgURL.String(), bytes.NewReader(jsonBody)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "application/json") - - rr, err := utils.MakeHTTPRequest(req) - - // record our status and log - log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) - status.AddLog(log) - if err != nil { - return status, nil - } - externalID, err := jsonparser.GetString(rr.Body, "message_id") - if err != nil { - log.WithError("Message Send Error", errors.Errorf("unable to get message_id from body")) - return status, nil - } - if i == 0 { - status.SetExternalID(externalID) - } - - // this was wired successfully - status.SetStatus(courier.MsgWired) - } - - return status, nil -} - -// DescribeURN looks up URN metadata for new contacts -func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn urns.URN) (map[string]string, error) { - - accessToken := channel.StringConfigForKey(courier.ConfigAuthToken, "") - if accessToken == "" { - return nil, fmt.Errorf("missing access token") - } - - // build a request to lookup the stats for this contact - base, _ := url.Parse(graphURL) - path, _ := url.Parse(fmt.Sprintf("/%s", urn.Path())) - u := base.ResolveReference(path) - - query := url.Values{} - query.Set("access_token", accessToken) - u.RawQuery = query.Encode() - req, _ := http.NewRequest(http.MethodGet, u.String(), nil) - rr, err := utils.MakeHTTPRequest(req) - if err != nil { - return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) - } - - // read our name - name, _ := jsonparser.GetString(rr.Body, "name") - - return map[string]string{"name": name}, nil -} - -// see https://developers.facebook.com/docs/messenger-platform/webhook#security -func (h *handler) validateSignature(r *http.Request) error { - headerSignature := r.Header.Get(signatureHeader) - if headerSignature == "" { - return fmt.Errorf("missing request signature") - } - appSecret := h.Server().Config().InstagramApplicationSecret - - body, err := handlers.ReadBody(r, 100000) - if err != nil { - return fmt.Errorf("unable to read request body: %s", err) - } - - expectedSignature, err := fbCalculateSignature(appSecret, body) - if err != nil { - return err - } - - signature := "" - if len(headerSignature) == 45 && strings.HasPrefix(headerSignature, "sha1=") { - signature = strings.TrimPrefix(headerSignature, "sha1=") - } - - // compare signatures in way that isn't sensitive to a timing attack - if !hmac.Equal([]byte(expectedSignature), []byte(signature)) { - return fmt.Errorf("invalid request signature, expected: %s got: %s for body: '%s'", expectedSignature, signature, string(body)) - } - - return nil -} - -func fbCalculateSignature(appSecret string, body []byte) (string, error) { - var buffer bytes.Buffer - buffer.Write(body) - - // hash with SHA1 - mac := hmac.New(sha1.New, []byte(appSecret)) - mac.Write(buffer.Bytes()) - - return hex.EncodeToString(mac.Sum(nil)), nil -} diff --git a/handlers/instagram/instagram_test.go b/handlers/instagram/instagram_test.go deleted file mode 100644 index 648a3ed68..000000000 --- a/handlers/instagram/instagram_test.go +++ /dev/null @@ -1,455 +0,0 @@ -package instagram - -import ( - "context" - "fmt" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" - - "github.com/nyaruka/courier" - "github.com/nyaruka/courier/handlers" - . "github.com/nyaruka/courier/handlers" - "github.com/nyaruka/gocommon/urns" - "github.com/stretchr/testify/assert" -) - -var testChannels = []courier.Channel{ - courier.NewMockChannel("8ab23e93-5ecb-45ba-b726-3b064e0c568c", "IG", "1234", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), -} - -var helloMsg = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var duplicateMsg = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }, - { - "id": "1234", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var invalidURN = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "abc5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var attachment = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "message": { - "mid": "external_id", - "attachments":[{ - "type":"image", - "payload":{ - "url":"https://image-url/foo.png" - } - }] - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var like_heart = `{ - "object":"instagram", - "entry":[{ - "id":"1234", - "messaging":[{ - "sender":{"id":"5678"}, - "recipient":{"id":"1234"}, - "timestamp":1459991487970, - "message":{ - "mid":"external_id", - "attachments":[{ - "type":"like_heart" - }] - } - }], - "time":1459991487970 - }] -}` - -var differentPage = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1235" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var echo = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970, - "message": { - "is_echo": true, - "mid": "qT7ywaK" - } - }] - }] -}` - -var icebreakerGetStarted = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "postback": { - "title": "icebreaker question", - "payload": "get_started" - }, - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var notInstagram = `{ - "object":"notinstagram", - "entry": [{}] -}` - -var noEntries = `{ - "object":"instagram", - "entry": [] -}` - -var noMessagingEntries = `{ - "object":"instagram", - "entry": [{ - "id": "1234" - }] -}` - -var unkownMessagingEntry = `{ - "object":"instagram", - "entry": [{ - "id": "1234", - "messaging": [{ - "recipient": { - "id": "1234" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }] - }] -}` - -var notJSON = `blargh` - -var testCases = []ChannelHandleTestCase{ - {Label: "Receive Message", URL: "/c/ig/receive", Data: helloMsg, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, - Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), - PrepRequest: addValidSignature}, - - {Label: "Receive Invalid Signature", URL: "/c/ig/receive", Data: helloMsg, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, - - {Label: "No Duplicate Receive Message", URL: "/c/ig/receive", Data: duplicateMsg, Status: 200, Response: "Handled", - Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), - PrepRequest: addValidSignature}, - - {Label: "Receive Attachment", URL: "/c/ig/receive", Data: attachment, Status: 200, Response: "Handled", - Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), - PrepRequest: addValidSignature}, - - {Label: "Receive Like Heart", URL: "/c/ig/receive", Data: like_heart, Status: 200, Response: "Handled", - Text: Sp(""), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), - PrepRequest: addValidSignature}, - - {Label: "Receive Icebreaker Get Started", URL: "/c/ig/receive", Data: icebreakerGetStarted, Status: 200, Response: "Handled", - URN: Sp("instagram:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), - ChannelEventExtra: map[string]interface{}{"title": "icebreaker question", "payload": "get_started"}, - PrepRequest: addValidSignature}, - - {Label: "Different Page", URL: "/c/ig/receive", Data: differentPage, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, - {Label: "Echo", URL: "/c/ig/receive", Data: echo, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, - {Label: "Not Instagram", URL: "/c/ig/receive", Data: notInstagram, Status: 400, Response: "expected 'instagram', found notinstagram", PrepRequest: addValidSignature}, - {Label: "No Entries", URL: "/c/ig/receive", Data: noEntries, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, - {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntries, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntry, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, - {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURN, Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, -} - -func addValidSignature(r *http.Request) { - body, _ := handlers.ReadBody(r, 100000) - sig, _ := fbCalculateSignature("ig_app_secret", body) - r.Header.Set(signatureHeader, fmt.Sprintf("sha1=%s", string(sig))) -} - -func addInvalidSignature(r *http.Request) { - r.Header.Set(signatureHeader, "invalidsig") -} - -// mocks the call to the Facebook graph API -func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - accessToken := r.URL.Query().Get("access_token") - defer r.Body.Close() - - // invalid auth token - if accessToken != "a123" { - http.Error(w, "invalid auth token", 403) - } - - // user has a name - if strings.HasSuffix(r.URL.Path, "1337") { - w.Write([]byte(`{ "name": "John Doe"}`)) - return - } - - // no name - w.Write([]byte(`{ "name": ""}`)) - })) - graphURL = server.URL - - return server -} - -func TestDescribe(t *testing.T) { - fbGraph := buildMockFBGraph(testCases) - defer fbGraph.Close() - - handler := newHandler().(courier.URNDescriber) - tcs := []struct { - urn urns.URN - metadata map[string]string - }{ - {"instagram:1337", map[string]string{"name": "John Doe"}}, - {"instagram:4567", map[string]string{"name": ""}}, - } - - for _, tc := range tcs { - metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) - assert.Equal(t, metadata, tc.metadata) - } -} - -func TestHandler(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler(), testCases) -} - -func BenchmarkHandler(b *testing.B) { - fbService := buildMockFBGraph(testCases) - defer fbService.Close() - - RunChannelBenchmarks(b, testChannels, newHandler(), testCases) -} - -func TestVerify(t *testing.T) { - - RunChannelTestCases(t, testChannels, newHandler(), []ChannelHandleTestCase{ - {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=ig_webhook_secret&hub.challenge=yarchallenge", Status: 200, - Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, - {Label: "Verify No Mode", URL: "/c/ig/receive", Status: 400, Response: "unknown request"}, - {Label: "Verify No Secret", URL: "/c/ig/receive?hub.mode=subscribe", Status: 400, Response: "token does not match secret"}, - {Label: "Invalid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=blah", Status: 400, Response: "token does not match secret"}, - {Label: "Valid Secret", URL: "/c/ig/receive?hub.mode=subscribe&hub.verify_token=ig_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge"}, - }) - -} - -// setSendURL takes care of setting the send_url to our test server host -func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, m courier.Msg) { - sendURL = s.URL -} - -var defaultSendTestCases = []ChannelSendTestCase{ - {Label: "Plain Send", - Text: "Simple Message", URN: "instagram:12345", - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, - SendPrep: setSendURL}, - - {Label: "Plain Response", - Text: "Simple Message", URN: "instagram:12345", - Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, - SendPrep: setSendURL}, - - {Label: "Tag Human Agent", - Text: "Simple Message", URN: "instagram:12345", - Status: "W", ExternalID: "mid.133", Topic: "agent", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, - SendPrep: setSendURL}, - - {Label: "Long Message", - Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", - URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"we exceed the max length?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, - SendPrep: setSendURL}, - - {Label: "Send caption and photo with Quick Reply", - Text: "This is some text.", - URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, - QuickReplies: []string{"Yes", "No"}, - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"This is some text.","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, - SendPrep: setSendURL}, - - {Label: "ID Error", - Text: "ID Error", URN: "instagram12345", - Status: "E", - ResponseBody: `{ "is_error": true }`, ResponseStatus: 200, - SendPrep: setSendURL}, - - {Label: "Error", - Text: "Error", URN: "instagram12345", - Status: "E", - ResponseBody: `{ "is_error": true }`, ResponseStatus: 403, - SendPrep: setSendURL}, - - {Label: "Quick Reply", - URN: "instagram:12345", Text: "Are you happy?", QuickReplies: []string{"Yes", "No"}, - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, - SendPrep: setSendURL}, - - {Label: "Send Photo", - URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, - Status: "W", ExternalID: "mid.133", - ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, - RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, - SendPrep: setSendURL}, -} - -func TestSending(t *testing.T) { - // shorter max msg length for testing - maxMsgLength = 100 - var defaultChannel = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "IG", "2020", "US", map[string]interface{}{courier.ConfigAuthToken: "access_token"}) - RunChannelSendTestCases(t, defaultChannel, newHandler(), defaultSendTestCases, nil) -} - -func TestSigning(t *testing.T) { - tcs := []struct { - Body string - Signature string - }{ - { - "hello world", - "308de7627fe19e92294c4572a7f831bc1002809d", - }, - { - "hello world2", - "ab6f902b58b9944032d4a960f470d7a8ebfd12b7", - }, - } - - for i, tc := range tcs { - sig, err := fbCalculateSignature("sesame", []byte(tc.Body)) - assert.NoError(t, err) - assert.Equal(t, tc.Signature, sig, "%d: mismatched signature", i) - } -} From fe834f7c1ac4a790b1d5498629bac0a842aea20b Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 28 Dec 2021 16:35:34 -0300 Subject: [PATCH 026/146] Ignore story mention callback --- handlers/facebookapp/facebookapp.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 3781adce8..aa4c5b4eb 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -142,6 +142,7 @@ type moPayload struct { IsEcho bool `json:"is_echo"` MID string `json:"mid"` Text string `json:"text"` + IsDeleted bool `json:"is_deleted"` Attachments []struct { Type string `json:"type"` Payload *struct { @@ -388,6 +389,11 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h attachmentURLs = append(attachmentURLs, fmt.Sprintf("geo:%f,%f", att.Payload.Coordinates.Lat, att.Payload.Coordinates.Long)) } + if att.Type == "story_mention" { + data = append(data, courier.NewInfoData("ignoring story_mention")) + continue + } + if att.Payload != nil && att.Payload.URL != "" { attachmentURLs = append(attachmentURLs, att.Payload.URL) } From 20553306b419251a97c4d6cfce9cc7b04b525c44 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 30 Dec 2021 12:17:16 -0300 Subject: [PATCH 027/146] Add quick replies for vk --- handlers/vk/vk.go | 88 ++++++++++++++++++++++++++++++++++++++++-- handlers/vk/vk_test.go | 50 +++++++++++++++++++++++- 2 files changed, 132 insertions(+), 6 deletions(-) diff --git a/handlers/vk/vk.go b/handlers/vk/vk.go index e82ceeaa3..8fc2bbc43 100644 --- a/handlers/vk/vk.go +++ b/handlers/vk/vk.go @@ -56,6 +56,7 @@ var ( paramMessage = "message" paramAttachments = "attachment" paramRandomId = "random_id" + paramKeyboard = "keyboard" // base upload media values paramServerId = "server" @@ -113,6 +114,7 @@ type moNewMessagePayload struct { Lng float64 `json:"longitude"` } `json:"coordinates"` } `json:"geo"` + Payload string `json:"payload"` } `json:"message" validate:"required"` } `json:"object" validate:"required"` } @@ -189,6 +191,23 @@ type mediaUploadInfoPayload struct { OwnerId int64 `json:"owner_id"` } +type Keyboard struct { + One_Time bool `json:"one_time"` + Buttons [][]buttonPayload `json:"buttons"` + Inline bool `json:"inline"` +} + +type buttonPayload struct { + Action buttonAction `json:"action"` + Color string `json:"color"` +} + +type buttonAction struct { + Type string `json:"type"` + Label string `json:"label"` + Payload string `json:"payload"` +} + // receiveEvent handles request event type func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { // read request body @@ -384,11 +403,7 @@ func takeFirstAttachmentUrl(payload moNewMessagePayload) string { func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) - req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionSendMessage, nil) - if err != nil { - return status, errors.New("Cannot create send message request") - } params := buildApiBaseParams(msg.Channel()) params.Set(paramUserId, msg.URN().Path()) params.Set(paramRandomId, msg.ID().String()) @@ -397,6 +412,19 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat params.Set(paramMessage, text) params.Set(paramAttachments, attachments) + if len(msg.QuickReplies()) != 0 { + keyboard := buildQuickRepliesParams(msg) + keyboardString := ToJSON(keyboard) + + params.Set(paramKeyboard, keyboardString) + } + + req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionSendMessage, nil) + + if err != nil { + return status, errors.New("Cannot create send message request") + } + req.URL.RawQuery = params.Encode() res, err := utils.MakeHTTPRequest(req) @@ -417,6 +445,58 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat return status, nil } +// ToJSON returns the JSON encoding of Keyboard. +func ToJSON(keyboard Keyboard) string { + b, _ := json.Marshal(keyboard) + return string(b) +} + +//keyboard +func buildQuickRepliesParams(msg courier.Msg) Keyboard { + + var keyboard Keyboard + keyboard = AddRowKeyboard(keyboard) + + for _, qr := range msg.QuickReplies() { + keyboard = AddTextButton(qr, qr, "secondary", keyboard) + } + return keyboard +} + +func AddRowKeyboard(keyboard Keyboard) Keyboard { + if len(keyboard.Buttons) == 0 { + keyboard.Buttons = make([][]buttonPayload, 1) + } else { + row := make([]buttonPayload, 0) + keyboard.Buttons = append(keyboard.Buttons, row) + + } + keyboard.One_Time = true + + return keyboard +} + +func AddTextButton(label string, payload interface{}, color string, keyboard Keyboard) Keyboard { + b, err := json.Marshal(payload) + if err != nil { + panic(err) + } + + button := buttonPayload{ + Action: buttonAction{ + Type: "text", + Label: label, + Payload: string(b), + }, + Color: color, + } + + lastRow := len(keyboard.Buttons) - 1 + keyboard.Buttons[lastRow] = append(keyboard.Buttons[lastRow], button) + + return keyboard +} + // buildTextAndAttachmentParams builds msg text with attachment links (if needed) and attachments list param, also returns the errors that occurred func buildTextAndAttachmentParams(msg courier.Msg, status courier.MsgStatus) (string, string) { var msgAttachments []string diff --git a/handlers/vk/vk_test.go b/handlers/vk/vk_test.go index 01cef3e58..dbca85cfb 100644 --- a/handlers/vk/vk_test.go +++ b/handlers/vk/vk_test.go @@ -2,8 +2,6 @@ package vk import ( "context" - "github.com/nyaruka/gocommon/urns" - "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" "net/url" @@ -11,6 +9,9 @@ import ( "testing" "time" + "github.com/nyaruka/gocommon/urns" + "github.com/stretchr/testify/assert" + "github.com/nyaruka/courier" . "github.com/nyaruka/courier/handlers" ) @@ -210,6 +211,22 @@ const eventServerVerification = `{ "secret": "abc123xyz" }` +const msgKeyboard = `{ + "type": "message_new", + "object": { + "message": { + "id": 1, + "date": 1580125800, + "from_id": 123456, + "text": "Yes", + "payload": "\"Yes\"" + } + }, + "secret": "abc123xyz" + }` + +const keyboardJson = `{"one_time":true,"buttons":[[{"action":{"type":"text","label":"Yes","payload":"\"Yes\""},"color":"secondary"}]],"inline":false}` + var testCases = []ChannelHandleTestCase{ { Label: "Receive Message", @@ -281,6 +298,16 @@ var testCases = []ChannelHandleTestCase{ ExternalID: Sp("1"), Date: Tp(time.Date(2020, 1, 27, 11, 50, 0, 0, time.UTC)), Attachments: []string{"https://foo.bar/doc.pdf"}, }, + { + Label: "Receive Message Keyboard", + URL: receiveURL, + Data: msgKeyboard, + Status: 200, + Response: "ok", + URN: Sp("vk:123456"), + ExternalID: Sp("1"), + Date: Tp(time.Date(2020, 1, 27, 11, 50, 0, 0, time.UTC)), + }, { Label: "Receive Geolocation Attachment", URL: receiveURL, @@ -445,6 +472,25 @@ var sendTestCases = []ChannelSendTestCase{ }, }, }, + { + Label: "Send keyboard", + Text: "Send keyboard", + URN: "vk:123456789", + QuickReplies: []string{"Yes"}, + Status: "S", + SendPrep: setSendURL, + ExternalID: "1", + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: actionSendMessage, + RawQuery: "access_token=token123xyz&attachment=&keyboard=" + url.QueryEscape(keyboardJson) + "&message=Send+keyboard&random_id=10&user_id=123456789&v=5.103", + }: { + Status: 200, + Body: `{"response": 1}`, + }, + }, + }, } func mockAttachmentURLs(mediaServer *httptest.Server, testCases []ChannelSendTestCase) []ChannelSendTestCase { From 220295b235129966e9d8cdc10e4e6a14da5f564e Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 4 Jan 2022 13:04:56 -0300 Subject: [PATCH 028/146] Add support for new keyboard rows --- handlers/vk/keyboard.go | 41 +++++++++++++++ handlers/vk/keyboard_test.go | 98 ++++++++++++++++++++++++++++++++++++ handlers/vk/vk.go | 76 ++-------------------------- handlers/vk/vk_test.go | 4 +- 4 files changed, 145 insertions(+), 74 deletions(-) create mode 100644 handlers/vk/keyboard.go create mode 100644 handlers/vk/keyboard_test.go diff --git a/handlers/vk/keyboard.go b/handlers/vk/keyboard.go new file mode 100644 index 000000000..a8b435254 --- /dev/null +++ b/handlers/vk/keyboard.go @@ -0,0 +1,41 @@ +package vk + +import ( + "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/jsonx" +) + +type Keyboard struct { + One_Time bool `json:"one_time"` + Buttons [][]ButtonPayload `json:"buttons"` + Inline bool `json:"inline"` +} + +type ButtonPayload struct { + Action ButtonAction `json:"action"` + Color string `json:"color"` +} + +type ButtonAction struct { + Type string `json:"type"` + Label string `json:"label"` + Payload string `json:"payload"` +} + +// NewKeyboardFromReplies creates a keyboard from the given quick replies +func NewKeyboardFromReplies(replies []string) *Keyboard { + rows := utils.StringsToRows(replies, 10, 30, 2) + buttons := make([][]ButtonPayload, len(rows)) + + for i := range rows { + buttons[i] = make([]ButtonPayload, len(rows[i])) + for j := range rows[i] { + buttons[i][j].Action.Label = rows[i][j] + buttons[i][j].Action.Type = "text" + buttons[i][j].Action.Payload = string(jsonx.MustMarshal(rows[i][j])) + buttons[i][j].Color = "primary" + } + } + + return &Keyboard{One_Time: true, Buttons: buttons, Inline: false} +} diff --git a/handlers/vk/keyboard_test.go b/handlers/vk/keyboard_test.go new file mode 100644 index 000000000..6a2491b68 --- /dev/null +++ b/handlers/vk/keyboard_test.go @@ -0,0 +1,98 @@ +package vk_test + +import ( + "testing" + + "github.com/nyaruka/courier/handlers/vk" + "github.com/stretchr/testify/assert" +) + +func TestKeyboardFromReplies(t *testing.T) { + tcs := []struct { + replies []string + expected *vk.Keyboard + }{ + { + + []string{"OK"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + { + {vk.ButtonAction{Type: "text", Label: "OK", Payload: "\"OK\""}, "primary"}, + }, + }, + false, + }, + }, + { + []string{"Yes", "No", "Maybe"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + { + {vk.ButtonAction{Type: "text", Label: "Yes", Payload: "\"Yes\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "No", Payload: "\"No\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "Maybe", Payload: "\"Maybe\""}, "primary"}, + }, + }, + false, + }, + }, + { + []string{"Vanilla", "Chocolate", "Mint", "Lemon Sorbet", "Papaya", "Strawberry"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + + {{vk.ButtonAction{Type: "text", Label: "Vanilla", Payload: "\"Vanilla\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Chocolate", Payload: "\"Chocolate\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Mint", Payload: "\"Mint\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Lemon Sorbet", Payload: "\"Lemon Sorbet\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Papaya", Payload: "\"Papaya\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Strawberry", Payload: "\"Strawberry\""}, "primary"}}, + }, + false, + }, + }, + { + []string{"A", "B", "C", "D", "Chicken", "Fish", "Peanut Butter Pickle"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + + {{vk.ButtonAction{Type: "text", Label: "A", Payload: "\"A\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "B", Payload: "\"B\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "C", Payload: "\"C\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "D", Payload: "\"D\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Chicken", Payload: "\"Chicken\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Fish", Payload: "\"Fish\""}, "primary"}}, + {{vk.ButtonAction{Type: "text", Label: "Peanut Butter Pickle", Payload: "\"Peanut Butter Pickle\""}, "primary"}}, + }, + false, + }, + }, + { + []string{"A", "B", "C", "D", "E"}, + &vk.Keyboard{ + true, + [][]vk.ButtonPayload{ + + { + {vk.ButtonAction{Type: "text", Label: "A", Payload: "\"A\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "B", Payload: "\"B\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "C", Payload: "\"C\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "D", Payload: "\"D\""}, "primary"}, + {vk.ButtonAction{Type: "text", Label: "E", Payload: "\"E\""}, "primary"}, + }, + }, + false, + }, + }, + } + + for _, tc := range tcs { + kb := vk.NewKeyboardFromReplies(tc.replies) + assert.Equal(t, tc.expected, kb, "keyboard mismatch for replies %v", tc.replies) + } +} diff --git a/handlers/vk/vk.go b/handlers/vk/vk.go index 8fc2bbc43..1f39c9a85 100644 --- a/handlers/vk/vk.go +++ b/handlers/vk/vk.go @@ -19,6 +19,7 @@ import ( "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/gocommon/urns" ) @@ -191,23 +192,6 @@ type mediaUploadInfoPayload struct { OwnerId int64 `json:"owner_id"` } -type Keyboard struct { - One_Time bool `json:"one_time"` - Buttons [][]buttonPayload `json:"buttons"` - Inline bool `json:"inline"` -} - -type buttonPayload struct { - Action buttonAction `json:"action"` - Color string `json:"color"` -} - -type buttonAction struct { - Type string `json:"type"` - Label string `json:"label"` - Payload string `json:"payload"` -} - // receiveEvent handles request event type func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { // read request body @@ -413,10 +397,10 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat params.Set(paramAttachments, attachments) if len(msg.QuickReplies()) != 0 { - keyboard := buildQuickRepliesParams(msg) - keyboardString := ToJSON(keyboard) + qrs := msg.QuickReplies() + keyboard := NewKeyboardFromReplies(qrs) - params.Set(paramKeyboard, keyboardString) + params.Set(paramKeyboard, string(jsonx.MustMarshal(keyboard))) } req, err := http.NewRequest(http.MethodPost, apiBaseURL+actionSendMessage, nil) @@ -445,58 +429,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat return status, nil } -// ToJSON returns the JSON encoding of Keyboard. -func ToJSON(keyboard Keyboard) string { - b, _ := json.Marshal(keyboard) - return string(b) -} - -//keyboard -func buildQuickRepliesParams(msg courier.Msg) Keyboard { - - var keyboard Keyboard - keyboard = AddRowKeyboard(keyboard) - - for _, qr := range msg.QuickReplies() { - keyboard = AddTextButton(qr, qr, "secondary", keyboard) - } - return keyboard -} - -func AddRowKeyboard(keyboard Keyboard) Keyboard { - if len(keyboard.Buttons) == 0 { - keyboard.Buttons = make([][]buttonPayload, 1) - } else { - row := make([]buttonPayload, 0) - keyboard.Buttons = append(keyboard.Buttons, row) - - } - keyboard.One_Time = true - - return keyboard -} - -func AddTextButton(label string, payload interface{}, color string, keyboard Keyboard) Keyboard { - b, err := json.Marshal(payload) - if err != nil { - panic(err) - } - - button := buttonPayload{ - Action: buttonAction{ - Type: "text", - Label: label, - Payload: string(b), - }, - Color: color, - } - - lastRow := len(keyboard.Buttons) - 1 - keyboard.Buttons[lastRow] = append(keyboard.Buttons[lastRow], button) - - return keyboard -} - // buildTextAndAttachmentParams builds msg text with attachment links (if needed) and attachments list param, also returns the errors that occurred func buildTextAndAttachmentParams(msg courier.Msg, status courier.MsgStatus) (string, string) { var msgAttachments []string diff --git a/handlers/vk/vk_test.go b/handlers/vk/vk_test.go index dbca85cfb..980755132 100644 --- a/handlers/vk/vk_test.go +++ b/handlers/vk/vk_test.go @@ -225,7 +225,7 @@ const msgKeyboard = `{ "secret": "abc123xyz" }` -const keyboardJson = `{"one_time":true,"buttons":[[{"action":{"type":"text","label":"Yes","payload":"\"Yes\""},"color":"secondary"}]],"inline":false}` +const keyboardJson = `{"one_time":true,"buttons":[[{"action":{"type":"text","label":"A","payload":"\"A\""},"color":"primary"},{"action":{"type":"text","label":"B","payload":"\"B\""},"color":"primary"},{"action":{"type":"text","label":"C","payload":"\"C\""},"color":"primary"},{"action":{"type":"text","label":"D","payload":"\"D\""},"color":"primary"},{"action":{"type":"text","label":"E","payload":"\"E\""},"color":"primary"}]],"inline":false}` var testCases = []ChannelHandleTestCase{ { @@ -476,7 +476,7 @@ var sendTestCases = []ChannelSendTestCase{ Label: "Send keyboard", Text: "Send keyboard", URN: "vk:123456789", - QuickReplies: []string{"Yes"}, + QuickReplies: []string{"A", "B", "C", "D", "E"}, Status: "S", SendPrep: setSendURL, ExternalID: "1", From 238050c36e57d55e32f7a994d4cac1aae90cbc68 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 4 Jan 2022 16:32:09 -0300 Subject: [PATCH 029/146] Remove unused Instagram-type configuration variables --- config.go | 94 +++++++++++++++++++++++------------------------- handlers/test.go | 2 -- 2 files changed, 45 insertions(+), 51 deletions(-) diff --git a/config.go b/config.go index 336664e96..5e6447690 100644 --- a/config.go +++ b/config.go @@ -4,33 +4,31 @@ import "github.com/nyaruka/ezconf" // Config is our top level configuration object type Config struct { - Backend string `help:"the backend that will be used by courier (currently only rapidpro is supported)"` - SentryDSN string `help:"the DSN used for logging errors to Sentry"` - Domain string `help:"the domain courier is exposed on"` - Address string `help:"the network interface address courier will bind to"` - Port int `help:"the port courier will listen on"` - DB string `help:"URL describing how to connect to the RapidPro database"` - Redis string `help:"URL describing how to connect to Redis"` - SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"` - S3Endpoint string `help:"the S3 endpoint we will write attachments to"` - S3Region string `help:"the S3 region we will write attachments to"` - S3MediaBucket string `help:"the S3 bucket we will write attachments to"` - S3MediaPrefix string `help:"the prefix that will be added to attachment filenames"` - S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"` - S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"` - AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` - AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` - FacebookApplicationSecret string `help:"the Facebook app secret"` - FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` - InstagramApplicationSecret string `help:"the Instagram app secret"` - InstagramWebhookSecret string `help:"the secret for Instagram webhook URL verification"` - MaxWorkers int `help:"the maximum number of go routines that will be used for sending (set to 0 to disable sending)"` - LibratoUsername string `help:"the username that will be used to authenticate to Librato"` - LibratoToken string `help:"the token that will be used to authenticate to Librato"` - StatusUsername string `help:"the username that is needed to authenticate against the /status endpoint"` - StatusPassword string `help:"the password that is needed to authenticate against the /status endpoint"` - LogLevel string `help:"the logging level courier should use"` - Version string `help:"the version that will be used in request and response headers"` + Backend string `help:"the backend that will be used by courier (currently only rapidpro is supported)"` + SentryDSN string `help:"the DSN used for logging errors to Sentry"` + Domain string `help:"the domain courier is exposed on"` + Address string `help:"the network interface address courier will bind to"` + Port int `help:"the port courier will listen on"` + DB string `help:"URL describing how to connect to the RapidPro database"` + Redis string `help:"URL describing how to connect to Redis"` + SpoolDir string `help:"the local directory where courier will write statuses or msgs that need to be retried (needs to be writable)"` + S3Endpoint string `help:"the S3 endpoint we will write attachments to"` + S3Region string `help:"the S3 region we will write attachments to"` + S3MediaBucket string `help:"the S3 bucket we will write attachments to"` + S3MediaPrefix string `help:"the prefix that will be added to attachment filenames"` + S3DisableSSL bool `help:"whether we disable SSL when accessing S3. Should always be set to False unless you're hosting an S3 compatible service within a secure internal network"` + S3ForcePathStyle bool `help:"whether we force S3 path style. Should generally need to default to False unless you're hosting an S3 compatible service"` + AWSAccessKeyID string `help:"the access key id to use when authenticating S3"` + AWSSecretAccessKey string `help:"the secret access key id to use when authenticating S3"` + FacebookApplicationSecret string `help:"the Facebook app secret"` + FacebookWebhookSecret string `help:"the secret for Facebook webhook URL verification"` + MaxWorkers int `help:"the maximum number of go routines that will be used for sending (set to 0 to disable sending)"` + LibratoUsername string `help:"the username that will be used to authenticate to Librato"` + LibratoToken string `help:"the token that will be used to authenticate to Librato"` + StatusUsername string `help:"the username that is needed to authenticate against the /status endpoint"` + StatusPassword string `help:"the password that is needed to authenticate against the /status endpoint"` + LogLevel string `help:"the logging level courier should use"` + Version string `help:"the version that will be used in request and response headers"` // IncludeChannels is the list of channels to enable, empty means include all IncludeChannels []string @@ -42,28 +40,26 @@ type Config struct { // NewConfig returns a new default configuration object func NewConfig() *Config { return &Config{ - Backend: "rapidpro", - Domain: "localhost", - Address: "", - Port: 8080, - DB: "postgres://temba:temba@localhost/temba?sslmode=disable", - Redis: "redis://localhost:6379/15", - SpoolDir: "/var/spool/courier", - S3Endpoint: "https://s3.amazonaws.com", - S3Region: "us-east-1", - S3MediaBucket: "courier-media", - S3MediaPrefix: "/media/", - S3DisableSSL: false, - S3ForcePathStyle: false, - AWSAccessKeyID: "", - AWSSecretAccessKey: "", - FacebookApplicationSecret: "missing_facebook_app_secret", - FacebookWebhookSecret: "missing_facebook_webhook_secret", - InstagramApplicationSecret: "missing_instagram_app_secret", - InstagramWebhookSecret: "missing_instagram_webhook_secret", - MaxWorkers: 32, - LogLevel: "error", - Version: "Dev", + Backend: "rapidpro", + Domain: "localhost", + Address: "", + Port: 8080, + DB: "postgres://temba:temba@localhost/temba?sslmode=disable", + Redis: "redis://localhost:6379/15", + SpoolDir: "/var/spool/courier", + S3Endpoint: "https://s3.amazonaws.com", + S3Region: "us-east-1", + S3MediaBucket: "courier-media", + S3MediaPrefix: "/media/", + S3DisableSSL: false, + S3ForcePathStyle: false, + AWSAccessKeyID: "", + AWSSecretAccessKey: "", + FacebookApplicationSecret: "missing_facebook_app_secret", + FacebookWebhookSecret: "missing_facebook_webhook_secret", + MaxWorkers: 32, + LogLevel: "error", + Version: "Dev", } } diff --git a/handlers/test.go b/handlers/test.go index bd1354628..9f5a88e3a 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -200,8 +200,6 @@ func newServer(backend courier.Backend) courier.Server { config := courier.NewConfig() config.FacebookWebhookSecret = "fb_webhook_secret" config.FacebookApplicationSecret = "fb_app_secret" - config.InstagramWebhookSecret = "ig_webhook_secret" - config.InstagramApplicationSecret = "ig_app_secret" return courier.NewServerWithLogger(config, backend, logger) From 42951fd54406b511d15f464b2ffa7420a7573eb2 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 4 Jan 2022 16:58:21 -0300 Subject: [PATCH 030/146] Add story mention skip test coverage --- handlers/facebookapp/facebookapp.go | 3 +-- handlers/facebookapp/facebookapp_test.go | 27 ++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index aa4c5b4eb..f9c1705b8 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -142,7 +142,6 @@ type moPayload struct { IsEcho bool `json:"is_echo"` MID string `json:"mid"` Text string `json:"text"` - IsDeleted bool `json:"is_deleted"` Attachments []struct { Type string `json:"type"` Payload *struct { @@ -228,7 +227,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } - // // is not a 'page' and 'instagram' object? ignore it + // is not a 'page' and 'instagram' object? ignore it if payload.Object != "page" && payload.Object != "instagram" { return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request") } diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 008a2d840..95b06189a 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -639,6 +639,32 @@ var unkownMessagingEntryIG = `{ }] }` +var storyMentionIG = `{ + "object":"instagram", + "entry": [{ + "id": "12345", + "messaging": [{ + "message": { + "mid": "external_id", + "attachments":[{ + "type":"story_mention", + "payload":{ + "url":"https://story-url" + } + }] + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + }], + "time": 1459991487970 + }] +}` + var notJSON = `blargh` var testCasesFBA = []ChannelHandleTestCase{ @@ -733,6 +759,7 @@ var testCasesIG = []ChannelHandleTestCase{ {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, + {Label: "Story Mention", URL: "/c/ig/receive", Data: storyMentionIG, Status: 200, Response: `ignoring story_mention`, PrepRequest: addValidSignature}, } func addValidSignature(r *http.Request) { From df30df8ad24bfa8192da1da4ebde82718797e740 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 5 Jan 2022 11:56:38 -0300 Subject: [PATCH 031/146] Update to gocommon v1.15.1 --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ccb11b67c..324f55a31 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/lib/pq v1.0.0 github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.14.1 + github.com/nyaruka/gocommon v1.15.1 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/patrickmn/go-cache v2.1.0+incompatible diff --git a/go.sum b/go.sum index 606fbc4d4..3826c538a 100644 --- a/go.sum +++ b/go.sum @@ -64,6 +64,8 @@ github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= github.com/nyaruka/gocommon v1.14.1 h1:/ScvLmg4zzVAuZ78TaENrvSEvW3WnUdqRd/t9hX7z7E= github.com/nyaruka/gocommon v1.14.1/go.mod h1:R1Vr7PwrYCSu+vcU0t8t/5C4TsCwcWoqiuIQCxcMqxs= +github.com/nyaruka/gocommon v1.15.1 h1:iMbI/CtCBNKSTl7ez+3tg+TGqQ1KqtIY4i4O5+dl1Tc= +github.com/nyaruka/gocommon v1.15.1/go.mod h1:R1Vr7PwrYCSu+vcU0t8t/5C4TsCwcWoqiuIQCxcMqxs= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= From 140b70c4be1bf2e85d95ff0f3117644bfa26cb2c Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 5 Jan 2022 12:07:59 -0300 Subject: [PATCH 032/146] Change of urn in instagram tests --- handlers/facebookapp/facebookapp.go | 16 +++++++++--- handlers/facebookapp/facebookapp_test.go | 32 ++++++++++++------------ 2 files changed, 29 insertions(+), 19 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index f9c1705b8..cbb6769b9 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -266,11 +266,21 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h sender = msg.Sender.ID } + var urn urns.URN + // create our URN - urn, err := urns.NewFacebookURN(sender) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + if payload.Object == "instagram" { + urn, err = urns.NewInstagramURN(sender) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + } else { + urn, err = urns.NewFacebookURN(sender) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } } + if msg.OptIn != nil { // this is an opt in, if we have a user_ref, use that as our URN (this is a checkbox plugin) // TODO: diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 95b06189a..7b75f97af 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -729,25 +729,25 @@ var testCasesFBA = []ChannelHandleTestCase{ } var testCasesIG = []ChannelHandleTestCase{ {Label: "Receive Message", URL: "/c/ig/receive", Data: helloMsgIG, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, - Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, {Label: "Receive Invalid Signature", URL: "/c/ig/receive", Data: helloMsgIG, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, {Label: "No Duplicate Receive Message", URL: "/c/ig/receive", Data: duplicateMsgIG, Status: 200, Response: "Handled", - Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, {Label: "Receive Attachment", URL: "/c/ig/receive", Data: attachmentIG, Status: 200, Response: "Handled", - Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, {Label: "Receive Like Heart", URL: "/c/ig/receive", Data: like_heart, Status: 200, Response: "Handled", - Text: Sp(""), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), + Text: Sp(""), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, {Label: "Receive Icebreaker Get Started", URL: "/c/ig/receive", Data: icebreakerGetStarted, Status: 200, Response: "Handled", - URN: Sp("facebook:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), + URN: Sp("instagram:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), ChannelEventExtra: map[string]interface{}{"title": "icebreaker question", "payload": "get_started"}, PrepRequest: addValidSignature}, @@ -758,7 +758,7 @@ var testCasesIG = []ChannelHandleTestCase{ {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntriesIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, - {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, {Label: "Story Mention", URL: "/c/ig/receive", Data: storyMentionIG, Status: 200, Response: `ignoring story_mention`, PrepRequest: addValidSignature}, } @@ -952,63 +952,63 @@ var SendTestCasesFBA = []ChannelSendTestCase{ var SendTestCasesIG = []ChannelSendTestCase{ {Label: "Plain Send", - Text: "Simple Message", URN: "facebook:12345", + Text: "Simple Message", URN: "instagram:12345", Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Plain Response", - Text: "Simple Message", URN: "facebook:12345", + Text: "Simple Message", URN: "instagram:12345", Status: "W", ExternalID: "mid.133", ResponseToExternalID: "23526", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"RESPONSE","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Quick Reply", - Text: "Are you happy?", URN: "facebook:12345", QuickReplies: []string{"Yes", "No"}, + Text: "Are you happy?", URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"Are you happy?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, SendPrep: setSendURL}, {Label: "Long Message", Text: "This is a long message which spans more than one part, what will actually be sent in the end if we exceed the max length?", - URN: "facebook:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", + URN: "instagram:12345", QuickReplies: []string{"Yes", "No"}, Topic: "agent", Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"we exceed the max length?","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, SendPrep: setSendURL}, {Label: "Send Photo", - URN: "facebook:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"image","payload":{"url":"https://foo.bar/image.jpg","is_reusable":true}}}}`, SendPrep: setSendURL}, {Label: "Send caption and photo with Quick Reply", Text: "This is some text.", - URN: "facebook:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + URN: "instagram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, QuickReplies: []string{"Yes", "No"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"text":"This is some text.","quick_replies":[{"title":"Yes","payload":"Yes","content_type":"text"},{"title":"No","payload":"No","content_type":"text"}]}}`, SendPrep: setSendURL}, {Label: "Tag Human Agent", - Text: "Simple Message", URN: "facebook:12345", + Text: "Simple Message", URN: "instagram:12345", Status: "W", ExternalID: "mid.133", Topic: "agent", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"MESSAGE_TAG","tag":"HUMAN_AGENT","recipient":{"id":"12345"},"message":{"text":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Send Document", - URN: "facebook:12345", Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + URN: "instagram:12345", Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, Status: "W", ExternalID: "mid.133", ResponseBody: `{"message_id": "mid.133"}`, ResponseStatus: 200, RequestBody: `{"messaging_type":"UPDATE","recipient":{"id":"12345"},"message":{"attachment":{"type":"file","payload":{"url":"https://foo.bar/document.pdf","is_reusable":true}}}}`, SendPrep: setSendURL}, {Label: "ID Error", - Text: "ID Error", URN: "facebook:12345", + Text: "ID Error", URN: "instagram:12345", Status: "E", ResponseBody: `{ "is_error": true }`, ResponseStatus: 200, SendPrep: setSendURL}, {Label: "Error", - Text: "Error", URN: "facebook:12345", + Text: "Error", URN: "instagram:12345", Status: "E", ResponseBody: `{ "is_error": true }`, ResponseStatus: 403, SendPrep: setSendURL}, From 4a7fcc92021fafebf8e64226baac9f564ed12f02 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 5 Jan 2022 15:07:00 -0300 Subject: [PATCH 033/146] Rename validateSignatures parameter to useUUIDRoutes --- handlers/facebookapp/facebookapp.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index cbb6769b9..d39a122a3 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -56,8 +56,8 @@ const ( payloadKey = "payload" ) -func newHandler(channelType courier.ChannelType, name string, validateSignatures bool) courier.ChannelHandler { - return &handler{handlers.NewBaseHandlerWithParams(channelType, name, validateSignatures)} +func newHandler(channelType courier.ChannelType, name string, useUUIDRoutes bool) courier.ChannelHandler { + return &handler{handlers.NewBaseHandlerWithParams(channelType, name, useUUIDRoutes)} } func init() { From eb6d0aa1ac91618d66fd212c84454a9cd0e9c154 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 5 Jan 2022 15:18:58 -0300 Subject: [PATCH 034/146] Separate TestDescribe for type IG and FBA --- handlers/facebookapp/facebookapp_test.go | 74 +++++++++++------------- 1 file changed, 33 insertions(+), 41 deletions(-) diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 7b75f97af..3123df7fe 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -16,7 +16,7 @@ import ( "github.com/stretchr/testify/assert" ) -var testChannels = []courier.Channel{ +var testChannelsFBA = []courier.Channel{ courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), } @@ -797,51 +797,43 @@ func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { return server } -func TestDescribe(t *testing.T) { - var testCases [][]ChannelHandleTestCase - testCases = append(testCases, testCasesFBA) - testCases = append(testCases, testCasesIG) - - for i, tc := range testCases { - fbGraph := buildMockFBGraph(tc) - defer fbGraph.Close() - - if i == 0 { - handler := newHandler("FBA", "Facebook", false).(courier.URNDescriber) - tcs := []struct { - urn urns.URN - metadata map[string]string - }{ - {"facebook:1337", map[string]string{"name": "John Doe"}}, - {"facebook:4567", map[string]string{"name": ""}}, - } +func TestDescribeFBA(t *testing.T) { + fbGraph := buildMockFBGraph(testCasesFBA) + defer fbGraph.Close() - for _, tc := range tcs { - metadata, _ := handler.DescribeURN(context.Background(), testChannels[0], tc.urn) - assert.Equal(t, metadata, tc.metadata) - } - } else { - handler := newHandler("IG", "Instagram", false).(courier.URNDescriber) - tcs := []struct { - urn urns.URN - metadata map[string]string - }{ - {"facebook:1337", map[string]string{"name": "John Doe"}}, - {"facebook:4567", map[string]string{"name": ""}}, - } + handler := newHandler("FBA", "Facebook", false).(courier.URNDescriber) + tcs := []struct { + urn urns.URN + metadata map[string]string + }{{"facebook:1337", map[string]string{"name": "John Doe"}}, + {"facebook:4567", map[string]string{"name": ""}}, + {"facebook:ref:1337", map[string]string{}}} + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannelsFBA[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) + } +} - for _, tc := range tcs { - metadata, _ := handler.DescribeURN(context.Background(), testChannelsIG[0], tc.urn) - assert.Equal(t, metadata, tc.metadata) - } - } +func TestDescribeIG(t *testing.T) { + fbGraph := buildMockFBGraph(testCasesIG) + defer fbGraph.Close() + handler := newHandler("IG", "Instagram", false).(courier.URNDescriber) + tcs := []struct { + urn urns.URN + metadata map[string]string + }{{"instagram:1337", map[string]string{"name": "John Doe"}}, + {"instagram:4567", map[string]string{"name": ""}}} + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannelsIG[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) } - } func TestHandler(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler("FBA", "Facebook", false), testCasesFBA) + RunChannelTestCases(t, testChannelsFBA, newHandler("FBA", "Facebook", false), testCasesFBA) RunChannelTestCases(t, testChannelsIG, newHandler("IG", "Instagram", false), testCasesIG) } @@ -849,7 +841,7 @@ func TestHandler(t *testing.T) { func BenchmarkHandler(b *testing.B) { fbService := buildMockFBGraph(testCasesFBA) - RunChannelBenchmarks(b, testChannels, newHandler("FBA", "Facebook", false), testCasesFBA) + RunChannelBenchmarks(b, testChannelsFBA, newHandler("FBA", "Facebook", false), testCasesFBA) fbService.Close() fbServiceIG := buildMockFBGraph(testCasesIG) @@ -860,7 +852,7 @@ func BenchmarkHandler(b *testing.B) { func TestVerify(t *testing.T) { - RunChannelTestCases(t, testChannels, newHandler("FBA", "Facebook", false), []ChannelHandleTestCase{ + RunChannelTestCases(t, testChannelsFBA, newHandler("FBA", "Facebook", false), []ChannelHandleTestCase{ {Label: "Valid Secret", URL: "/c/fba/receive?hub.mode=subscribe&hub.verify_token=fb_webhook_secret&hub.challenge=yarchallenge", Status: 200, Response: "yarchallenge", NoQueueErrorCheck: true, NoInvalidChannelCheck: true}, {Label: "Verify No Mode", URL: "/c/fba/receive", Status: 400, Response: "unknown request"}, From 4ade345ff956b19a1ae1cff147afab2cb0f63226 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 7 Jan 2022 13:26:42 -0500 Subject: [PATCH 035/146] Use dbutil package from gocommon --- backends/rapidpro/backend.go | 20 +++++++- backends/rapidpro/backend_test.go | 5 +- batch/batch.go | 82 ++----------------------------- batch/batch_test.go | 27 +++------- go.mod | 16 +++--- go.sum | 31 +++++++----- utils/misc.go | 10 ---- utils/misc_test.go | 5 -- 8 files changed, 57 insertions(+), 139 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 364d0d3fe..32e56c061 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -19,6 +19,7 @@ import ( "github.com/nyaruka/courier/batch" "github.com/nyaruka/courier/queue" "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/dbutil" "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/librato" @@ -651,7 +652,15 @@ func (b *backend) Start() error { // create our status committer and start it b.statusCommitter = batch.NewCommitter("status committer", b.db, bulkUpdateMsgStatusSQL, time.Millisecond*500, b.committerWG, func(err error, value batch.Value) { - logrus.WithField("comp", "status committer").WithError(err).Error("error writing status") + log := logrus.WithField("comp", "status committer") + + if qerr := dbutil.AsQueryError(err); qerr != nil { + query, params := qerr.Query() + log = log.WithFields(logrus.Fields{"sql": query, "sql_params": params}) + } + + log.WithError(err).Error("error writing status") + err = courier.WriteToSpool(b.config.SpoolDir, "statuses", value) if err != nil { logrus.WithField("comp", "status committer").WithError(err).Error("error writing status to spool") @@ -662,7 +671,14 @@ func (b *backend) Start() error { // create our log committer and start it b.logCommitter = batch.NewCommitter("log committer", b.db, insertLogSQL, time.Millisecond*500, b.committerWG, func(err error, value batch.Value) { - logrus.WithField("comp", "log committer").WithError(err).Error("error writing channel log") + log := logrus.WithField("comp", "log committer") + + if qerr := dbutil.AsQueryError(err); qerr != nil { + query, params := qerr.Query() + log = log.WithFields(logrus.Fields{"sql": query, "sql_params": params}) + } + + log.WithError(err).Error("error writing channel log") }) b.logCommitter.Start() diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index 74de96d55..df34f0af5 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -17,6 +17,7 @@ import ( "github.com/nyaruka/courier" "github.com/nyaruka/courier/queue" + "github.com/nyaruka/gocommon/dbutil/assertdb" "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/null" @@ -1172,9 +1173,7 @@ func (ts *BackendTestSuite) TestSessionTimeout() { ts.NoError(err) // make sure that took - count := 0 - ts.b.db.Get(&count, "SELECT count(*) from flows_flowsession WHERE timeout_on > NOW()") - ts.Equal(1, count) + assertdb.Query(ts.T(), ts.b.db, `SELECT count(*) from flows_flowsession WHERE timeout_on > NOW()`).Returns(1) } func (ts *BackendTestSuite) TestMailroomEvents() { diff --git a/batch/batch.go b/batch/batch.go index bfec78c46..1848ce911 100644 --- a/batch/batch.go +++ b/batch/batch.go @@ -2,11 +2,11 @@ package batch import ( "context" - "strings" "sync" "time" "github.com/jmoiron/sqlx" + "github.com/nyaruka/gocommon/dbutil" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -147,7 +147,7 @@ func (c *committer) flush(size int) bool { err = batchSQL(ctx, c.label, c.db, c.sql, []interface{}{v}) if err != nil { if c.callback != nil { - c.callback(errors.Wrapf(err, "%s: error comitting value", c.label), v.(Value)) + c.callback(errors.Wrapf(err, "%s: error committing value", c.label), v.(Value)) } } } @@ -180,86 +180,12 @@ func batchSQL(ctx context.Context, label string, db *sqlx.DB, sql string, vs []i start := time.Now() - // this will be our SQL placeholders ($1, $2,..) for values in our final query, built dynamically - values := strings.Builder{} - values.Grow(7 * len(vs)) - - // this will be each of the arguments to match the positional values above - args := make([]interface{}, 0, len(vs)*5) - - // for each value we build a bound SQL statement, then extract the values clause - for i, value := range vs { - valueSQL, valueArgs, err := sqlx.Named(sql, value) - if err != nil { - return errors.Wrapf(err, "error converting bulk insert args") - } - - args = append(args, valueArgs...) - argValues, err := extractValues(valueSQL) - if err != nil { - return errors.Wrapf(err, "error extracting values from sql: %s", valueSQL) - } - - // append to our global values, adding comma if necessary - values.WriteString(argValues) - if i+1 < len(vs) { - values.WriteString(",") - } - } - - valuesSQL, err := extractValues(sql) - if err != nil { - return errors.Wrapf(err, "error extracting values from sql: %s", sql) - } - - bulkInsert := db.Rebind(strings.Replace(sql, valuesSQL, values.String(), -1)) - - // insert them all at once - rows, err := db.QueryxContext(ctx, bulkInsert, args...) + err := dbutil.BulkQuery(ctx, db, sql, vs) if err != nil { - return errors.Wrapf(err, "error during bulk insert") - } - defer rows.Close() - - // iterate our remaining rows - for rows.Next() { - } - - // check for any error - if rows.Err() != nil { - return errors.Wrapf(rows.Err(), "error in row cursor") + return err } logrus.WithField("elapsed", time.Since(start)).WithField("rows", len(vs)).Infof("%s bulk sql complete", label) return nil } - -// extractValues extracts the portion between `VALUE(` and `)` in the passed in string. (leaving VALUE but not the parentheses) -func extractValues(sql string) (string, error) { - startValues := strings.Index(sql, "VALUES(") - if startValues <= 0 { - return "", errors.Errorf("unable to find VALUES( in bulk insert SQL: %s", sql) - } - - // find the matching end parentheses, we need to count balanced parentheses here - openCount := 1 - endValues := -1 - for i, r := range sql[startValues+7:] { - if r == '(' { - openCount++ - } else if r == ')' { - openCount-- - if openCount == 0 { - endValues = i + startValues + 7 - break - } - } - } - - if endValues <= 0 { - return "", errors.Errorf("unable to find end of VALUES() in bulk insert sql: %s", sql) - } - - return sql[startValues+6 : endValues+1], nil -} diff --git a/batch/batch_test.go b/batch/batch_test.go index 73947201d..0c2de2d2d 100644 --- a/batch/batch_test.go +++ b/batch/batch_test.go @@ -8,6 +8,7 @@ import ( "github.com/jmoiron/sqlx" _ "github.com/lib/pq" + "github.com/nyaruka/gocommon/dbutil/assertdb" "github.com/stretchr/testify/assert" ) @@ -44,9 +45,7 @@ func TestBatchInsert(t *testing.T) { time.Sleep(time.Second) assert.NoError(t, callbackErr) - count := 0 - db.Get(&count, "SELECT count(*) FROM labels;") - assert.Equal(t, 3, count) + assertdb.Query(t, db, `SELECT count(*) FROM labels;`).Returns(3) committer.Queue(&Label{0, "label4"}) committer.Queue(&Label{0, "label3"}) @@ -54,9 +53,8 @@ func TestBatchInsert(t *testing.T) { time.Sleep(time.Second) assert.Error(t, callbackErr) - assert.Equal(t, `labels: error comitting value: error during bulk insert: pq: duplicate key value violates unique constraint "labels_label_key"`, callbackErr.Error()) - db.Get(&count, "SELECT count(*) FROM labels;") - assert.Equal(t, 4, count) + assert.Equal(t, `labels: error committing value: error making bulk query: pq: duplicate key value violates unique constraint "labels_label_key"`, callbackErr.Error()) + assertdb.Query(t, db, `SELECT count(*) FROM labels;`).Returns(4) } func TestBatchUpdate(t *testing.T) { @@ -94,17 +92,8 @@ func TestBatchUpdate(t *testing.T) { time.Sleep(time.Second) assert.NoError(t, callbackErr) - count := 0 - db.Get(&count, "SELECT count(*) FROM labels;") - assert.Equal(t, 3, count) - - label := "" - db.Get(&label, "SELECT label FROM labels WHERE id = 1;") - assert.Equal(t, "label001", label) - - db.Get(&label, "SELECT label FROM labels WHERE id = 2;") - assert.Equal(t, "label02", label) - - db.Get(&label, "SELECT label FROM labels WHERE id = 3;") - assert.Equal(t, "label03", label) + assertdb.Query(t, db, `SELECT count(*) FROM labels;`).Returns(3) + assertdb.Query(t, db, `SELECT label FROM labels WHERE id = 1`).Returns("label001") + assertdb.Query(t, db, `SELECT label FROM labels WHERE id = 2`).Returns("label02") + assertdb.Query(t, db, `SELECT label FROM labels WHERE id = 3`).Returns("label03") } diff --git a/go.mod b/go.mod index ccb11b67c..6f95063a3 100644 --- a/go.mod +++ b/go.mod @@ -11,19 +11,17 @@ require ( github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 // indirect github.com/go-chi/chi v4.1.2+incompatible github.com/go-errors/errors v1.0.1 - github.com/go-playground/locales v0.11.2 // indirect - github.com/go-playground/universal-translator v0.16.0 // indirect - github.com/go-sql-driver/mysql v1.5.0 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect github.com/gofrs/uuid v3.3.0+incompatible github.com/gomodule/redigo v2.0.0+incompatible github.com/gorilla/schema v1.0.2 - github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 + github.com/jmoiron/sqlx v1.3.4 github.com/kr/pretty v0.1.0 // indirect github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect - github.com/lib/pq v1.0.0 - github.com/mattn/go-sqlite3 v1.10.0 // indirect + github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.14.1 + github.com/nyaruka/gocommon v1.16.0 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/patrickmn/go-cache v2.1.0+incompatible @@ -32,8 +30,7 @@ require ( github.com/stretchr/testify v1.7.0 golang.org/x/mod v0.4.2 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - gopkg.in/go-playground/assert.v1 v1.2.1 // indirect - gopkg.in/go-playground/validator.v9 v9.11.0 + gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/h2non/filetype.v1 v1.0.5 ) @@ -43,6 +40,7 @@ require ( github.com/golang/protobuf v1.3.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect + github.com/leodido/go-urn v1.2.1 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.1 // indirect github.com/nyaruka/phonenumbers v1.0.71 // indirect diff --git a/go.sum b/go.sum index 606fbc4d4..23c139a6c 100644 --- a/go.sum +++ b/go.sum @@ -23,10 +23,10 @@ github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyN github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-playground/locales v0.11.2 h1:wH6Ksuvzk0SU9M6wUeGz/EaRWnavAHCOsFre1njzgi8= -github.com/go-playground/locales v0.11.2/go.mod h1:IUMDtCfWo/w/mtMfIE/IG2K+Ey3ygWanZIBtBW0W2TM= -github.com/go-playground/universal-translator v0.16.0 h1:X++omBR/4cE2MNg91AoC3rmGrCjJ8eAeUP/K/EKx4DM= -github.com/go-playground/universal-translator v0.16.0/go.mod h1:1AnU7NaIRDWWzGEKwgtJRd2xk99HeFyHw3yid4rvQIY= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= @@ -41,8 +41,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0 h1:5B0uxl2lzNRVkJVg+uGHxWtRt4C0Wjc6kJKo5XYx8xE= -github.com/jmoiron/sqlx v0.0.0-20180614180643-0dae4fefe7c0/go.mod h1:IiEW3SEiiErVyFdH8NTuWjSifiEQKUoyK3LNqr2kCHU= +github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= +github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= @@ -52,18 +52,22 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= -github.com/lib/pq v1.0.0 h1:X5PMW56eZitiTeO7tKzZxFCSpbFZJtkMMooicw2us9A= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= -github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.14.1 h1:/ScvLmg4zzVAuZ78TaENrvSEvW3WnUdqRd/t9hX7z7E= -github.com/nyaruka/gocommon v1.14.1/go.mod h1:R1Vr7PwrYCSu+vcU0t8t/5C4TsCwcWoqiuIQCxcMqxs= +github.com/nyaruka/gocommon v1.16.0 h1:F2DXo8075ErYm3pIJ5209HXRIyGWvwH8mtyxjwRYN0w= +github.com/nyaruka/gocommon v1.16.0/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= @@ -85,6 +89,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -115,8 +120,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v9 v9.11.0 h1:ER548TqE6ZknRRDDo0/tP8I12UHYxNlIfss8tMd4iCo= -gopkg.in/go-playground/validator.v9 v9.11.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= +gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= +gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y= gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/utils/misc.go b/utils/misc.go index 222b00251..8f73b9a15 100644 --- a/utils/misc.go +++ b/utils/misc.go @@ -5,7 +5,6 @@ import ( "crypto/hmac" "crypto/sha256" "encoding/hex" - "encoding/json" "net/url" "path" "regexp" @@ -22,15 +21,6 @@ func SignHMAC256(privateKey string, value string) string { return signedParams } -// MapAsJSON serializes the given map as a JSON string -func MapAsJSON(m map[string]string) []byte { - bytes, err := json.Marshal(m) - if err != nil { - panic(err) - } - return bytes -} - // JoinNonEmpty takes a vararg of strings and return the join of all the non-empty strings with a delimiter between them func JoinNonEmpty(delim string, strings ...string) string { var buf bytes.Buffer diff --git a/utils/misc_test.go b/utils/misc_test.go index 23c30c26d..6e760b825 100644 --- a/utils/misc_test.go +++ b/utils/misc_test.go @@ -14,11 +14,6 @@ func TestSignHMAC256(t *testing.T) { assert.Len(t, utils.SignHMAC256("ZXwAumfRSejDxJGa", "newValueToEncrypt"), 64) } -func TestMapAsJSON(t *testing.T) { - assert.Equal(t, "{}", string(utils.MapAsJSON(map[string]string{}))) - assert.Equal(t, "{\"foo\":\"bar\"}", string(utils.MapAsJSON(map[string]string{"foo": "bar"}))) -} - func TestJoinNonEmpty(t *testing.T) { assert.Equal(t, "", utils.JoinNonEmpty(" ")) assert.Equal(t, "hello world", utils.JoinNonEmpty(" ", "", "hello", "", "world")) From 9ce5e88d648c5bb8eab265f00f702f1a13d8810f Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 7 Jan 2022 15:46:24 -0300 Subject: [PATCH 036/146] Change 'EntryID' to 'entryID' --- handlers/facebookapp/facebookapp.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index d39a122a3..54fd3099e 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -185,13 +185,13 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan return nil, fmt.Errorf("no entries found") } - EntryID := payload.Entry[0].ID + entryID := payload.Entry[0].ID //if object is 'page' returns type FBA, if object is 'instagram' returns type IG if payload.Object == "page" { - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(EntryID)) + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(entryID)) } else { - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(EntryID)) + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(entryID)) } } From eef3716d879b9fc3d06570f24114e5220bcec7fa Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 7 Jan 2022 15:48:07 -0300 Subject: [PATCH 037/146] Fix variable names --- handlers/facebookapp/facebookapp_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 3123df7fe..72ce44003 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -607,7 +607,7 @@ var noMessagingEntriesIG = `{ }] }` -var unkownMessagingEntryFBA = `{ +var unknownMessagingEntryFBA = `{ "object":"page", "entry": [{ "id": "12345", @@ -623,7 +623,7 @@ var unkownMessagingEntryFBA = `{ }] }` -var unkownMessagingEntryIG = `{ +var unknownMessagingEntryIG = `{ "object":"instagram", "entry": [{ "id": "12345", From c97ea0b1bb1d0585aaf409fb2fab2ccd53902188 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 7 Jan 2022 16:03:10 -0300 Subject: [PATCH 038/146] Rename variable in test cases --- handlers/facebookapp/facebookapp_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 72ce44003..625664971 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -723,7 +723,7 @@ var testCasesFBA = []ChannelHandleTestCase{ {Label: "Not Page", URL: "/c/fba/receive", Data: notPage, Status: 400, Response: "object expected 'page' or 'instagram', found notpage", PrepRequest: addValidSignature}, {Label: "No Entries", URL: "/c/fba/receive", Data: noEntriesFBA, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: noMessagingEntriesFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unkownMessagingEntryFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unknownMessagingEntryFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/fba/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, {Label: "Invalid URN", URL: "/c/fba/receive", Data: invalidURNFBA, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, } @@ -756,7 +756,7 @@ var testCasesIG = []ChannelHandleTestCase{ {Label: "No Entries", URL: "/c/ig/receive", Data: noEntriesIG, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, {Label: "Not Instagram", URL: "/c/ig/receive", Data: notInstagram, Status: 400, Response: "object expected 'page' or 'instagram', found notinstagram", PrepRequest: addValidSignature}, {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntriesIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unkownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unknownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, {Label: "Story Mention", URL: "/c/ig/receive", Data: storyMentionIG, Status: 200, Response: `ignoring story_mention`, PrepRequest: addValidSignature}, From 3163cff466705fae27c580ad5a137e436e6ac209 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 7 Jan 2022 14:20:43 -0500 Subject: [PATCH 039/146] Update CHANGELOG.md for v7.1.7 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a481824da..55cc8365e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.1.7 +---------- + * Use dbutil package from gocommon + * Add quick replies for vk + v7.1.6 ---------- * Throttle WA queues when we get 429 responses From db8727b3e9f678109e9697ca1c8a9bef073d8891 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 7 Jan 2022 15:09:09 -0500 Subject: [PATCH 040/146] Do more error wrapping when creating contacts and URNs --- backends/rapidpro/contact.go | 24 ++++++++++++------------ backends/rapidpro/msg.go | 7 ++++--- backends/rapidpro/urn.go | 9 +++++---- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/backends/rapidpro/contact.go b/backends/rapidpro/contact.go index 4384436af..722ab4c76 100644 --- a/backends/rapidpro/contact.go +++ b/backends/rapidpro/contact.go @@ -9,13 +9,14 @@ import ( "unicode/utf8" "github.com/nyaruka/courier" + "github.com/nyaruka/gocommon/dbutil" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/librato" "github.com/nyaruka/null" + "github.com/pkg/errors" "github.com/jmoiron/sqlx" - "github.com/lib/pq" "github.com/sirupsen/logrus" ) @@ -102,7 +103,7 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne err := b.db.GetContext(ctx, contact, lookupContactFromURNSQL, urn.Identity(), org) if err != nil && err != sql.ErrNoRows { logrus.WithError(err).WithField("urn", urn.Identity()).WithField("org_id", org).Error("error looking up contact") - return nil, err + return nil, errors.Wrap(err, "error looking up contact by URN") } // we found it, return it @@ -111,14 +112,14 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne tx, err := b.db.BeginTxx(ctx, nil) if err != nil { logrus.WithError(err).WithField("urn", urn.Identity()).WithField("org_id", org).Error("error looking up contact") - return nil, err + return nil, errors.Wrap(err, "error beginning transaction") } err = setDefaultURN(tx, channel, contact, urn, auth) if err != nil { logrus.WithError(err).WithField("urn", urn.Identity()).WithField("org_id", org).Error("error looking up contact") tx.Rollback() - return nil, err + return nil, errors.Wrap(err, "error setting default URN for contact") } return contact, tx.Commit() } @@ -166,13 +167,13 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne // insert it tx, err := b.db.BeginTxx(ctx, nil) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error beginning transaction") } err = insertContact(tx, contact) if err != nil { tx.Rollback() - return nil, err + return nil, errors.Wrap(err, "error inserting contact") } // used for unit testing contact races @@ -186,13 +187,12 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne contactURN, err := contactURNForURN(tx, channel, contact.ID_, urn, auth) if err != nil { tx.Rollback() - if pqErr, ok := err.(*pq.Error); ok { + + if dbutil.IsUniqueViolation(err) { // if this was a duplicate URN, start over with a contact lookup - if pqErr.Code.Name() == "unique_violation" { - return contactForURN(ctx, b, org, channel, urn, auth, name) - } + return contactForURN(ctx, b, org, channel, urn, auth, name) } - return nil, err + return nil, errors.Wrap(err, "error getting URN for contact") } // we stole the URN from another contact, roll back and start over @@ -204,7 +204,7 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne // all is well, we created the new contact, commit and move forward err = tx.Commit() if err != nil { - return nil, err + return nil, errors.Wrap(err, "error commiting transaction") } // store this URN on our contact diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index a194a9363..cd96c5b4e 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -15,6 +15,7 @@ import ( "time" "github.com/buger/jsonparser" + "github.com/pkg/errors" "mime" @@ -134,7 +135,7 @@ func writeMsgToDB(ctx context.Context, b *backend, m *DBMsg) error { // our db is down, write to the spool, we will write/queue this later if err != nil { - return err + return errors.Wrap(err, "error getting contact for message") } // set our contact and urn ids from our contact @@ -143,14 +144,14 @@ func writeMsgToDB(ctx context.Context, b *backend, m *DBMsg) error { rows, err := b.db.NamedQueryContext(ctx, insertMsgSQL, m) if err != nil { - return err + return errors.Wrap(err, "error inserting message") } defer rows.Close() rows.Next() err = rows.Scan(&m.ID_) if err != nil { - return err + return errors.Wrap(err, "error scanning for inserted message id") } // queue this up to be handled by RapidPro diff --git a/backends/rapidpro/urn.go b/backends/rapidpro/urn.go index 4e25cddd1..6e71e9402 100644 --- a/backends/rapidpro/urn.go +++ b/backends/rapidpro/urn.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/nyaruka/null" + "github.com/pkg/errors" "github.com/jmoiron/sqlx" "github.com/nyaruka/courier" @@ -209,14 +210,14 @@ func contactURNForURN(db *sqlx.Tx, channel *DBChannel, contactID ContactID, urn } err := db.Get(contactURN, selectOrgURN, channel.OrgID(), urn.Identity()) if err != nil && err != sql.ErrNoRows { - return nil, err + return nil, errors.Wrap(err, "error looking up URN by identity") } // we didn't find it, let's insert it if err == sql.ErrNoRows { err = insertContactURN(db, contactURN) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error inserting URN") } } @@ -232,7 +233,7 @@ func contactURNForURN(db *sqlx.Tx, channel *DBChannel, contactID ContactID, urn contactURN.Display = display err = updateContactURN(db, contactURN) if err != nil { - return nil, err + return nil, errors.Wrap(err, "error updating URN") } } @@ -242,7 +243,7 @@ func contactURNForURN(db *sqlx.Tx, channel *DBChannel, contactID ContactID, urn err = updateContactURN(db, contactURN) } - return contactURN, err + return contactURN, errors.Wrap(err, "error updating URN auth") } const insertURN = ` From 207d7bf93b1c699fadf644defb8f6be7bbf163bd Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 09:24:59 -0500 Subject: [PATCH 041/146] Update CHANGELOG.md for v7.1.8 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55cc8365e..3148d5de0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.8 +---------- + * Do more error wrapping when creating contacts and URNs + v7.1.7 ---------- * Use dbutil package from gocommon From 50064d1cbb3494b807a505a8a3783c12b91e29ba Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 10:26:19 -0500 Subject: [PATCH 042/146] Fix bulk status updates --- backends/rapidpro/status.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backends/rapidpro/status.go b/backends/rapidpro/status.go index a707bb7a4..998101042 100644 --- a/backends/rapidpro/status.go +++ b/backends/rapidpro/status.go @@ -305,7 +305,7 @@ WHERE msgs_msg.channel_id = s.channel_id::int AND msgs_msg.direction = 'O' RETURNING - msgs_msg.id + msgs_msg.id AS msg_id ` //----------------------------------------------------------------------------- From 2fd9ac0773b1e47bf5702978daa246e0fed59cde Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 11:01:17 -0500 Subject: [PATCH 043/146] Update CHANGELOG.md for v7.1.9 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3148d5de0..0668c1f39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.9 +---------- + * Fix bulk status updates + v7.1.8 ---------- * Do more error wrapping when creating contacts and URNs From 63e6361dd4c3bd801d67f9e51b4193bcf1945c45 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 13:26:48 -0500 Subject: [PATCH 044/146] Update to latest gocommon --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6f95063a3..469cef140 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.16.0 + github.com/nyaruka/gocommon v1.16.1 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/patrickmn/go-cache v2.1.0+incompatible diff --git a/go.sum b/go.sum index 23c139a6c..1126f1e6d 100644 --- a/go.sum +++ b/go.sum @@ -66,8 +66,8 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.16.0 h1:F2DXo8075ErYm3pIJ5209HXRIyGWvwH8mtyxjwRYN0w= -github.com/nyaruka/gocommon v1.16.0/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= +github.com/nyaruka/gocommon v1.16.1 h1:RvIEx/w6a2c0T3eO+SaOGZjiajK4+t8S8Utqd3PnoGg= +github.com/nyaruka/gocommon v1.16.1/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= From 4a8a9e6919d6c2057b0add50a1a4763c8efe29c5 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 13:33:32 -0500 Subject: [PATCH 045/146] Update CHANGELOG.md for v7.1.10 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0668c1f39..b62e7a96d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.10 +---------- + * Update to latest gocommon + v7.1.9 ---------- * Fix bulk status updates From 6f444e72c3940eab8fbc635ab64193a8a7b32514 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 15:40:02 -0500 Subject: [PATCH 046/146] More bulk sql tweaks --- backends/rapidpro/status.go | 2 -- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/backends/rapidpro/status.go b/backends/rapidpro/status.go index 998101042..56759fcf2 100644 --- a/backends/rapidpro/status.go +++ b/backends/rapidpro/status.go @@ -304,8 +304,6 @@ WHERE msgs_msg.id = s.msg_id::bigint AND msgs_msg.channel_id = s.channel_id::int AND msgs_msg.direction = 'O' -RETURNING - msgs_msg.id AS msg_id ` //----------------------------------------------------------------------------- diff --git a/go.mod b/go.mod index 469cef140..34edb620f 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.16.1 + github.com/nyaruka/gocommon v1.16.2 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/patrickmn/go-cache v2.1.0+incompatible diff --git a/go.sum b/go.sum index 1126f1e6d..a4af1d246 100644 --- a/go.sum +++ b/go.sum @@ -66,8 +66,8 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.16.1 h1:RvIEx/w6a2c0T3eO+SaOGZjiajK4+t8S8Utqd3PnoGg= -github.com/nyaruka/gocommon v1.16.1/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= +github.com/nyaruka/gocommon v1.16.2 h1:fvDKKRVsaES76SyvWujDttwplVWVZwHL3AJNbk+rDno= +github.com/nyaruka/gocommon v1.16.2/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= From 71d5ddf7cb05115a8e67a72b548848e29e116b46 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 10 Jan 2022 15:47:14 -0500 Subject: [PATCH 047/146] Update CHANGELOG.md for v7.1.11 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b62e7a96d..a102cf7fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.11 +---------- + * More bulk sql tweaks + v7.1.10 ---------- * Update to latest gocommon From 7e8a19845aef536f38b2fffec6f51fa2f026bdd6 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 12 Jan 2022 16:53:54 -0500 Subject: [PATCH 048/146] Update to latest gocommon --- go.mod | 9 +++++---- go.sum | 18 ++++++++++++++---- handlers/whatsapp/whatsapp.go | 7 ++++--- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 34edb620f..bb6f37861 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/nyaruka/courier +go 1.17 + require ( github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect @@ -14,16 +16,17 @@ require ( github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect github.com/gofrs/uuid v3.3.0+incompatible - github.com/gomodule/redigo v2.0.0+incompatible + github.com/gomodule/redigo v1.8.8 github.com/gorilla/schema v1.0.2 github.com/jmoiron/sqlx v1.3.4 github.com/kr/pretty v0.1.0 // indirect github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.16.2 + github.com/nyaruka/gocommon v1.17.0 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 + github.com/nyaruka/redisx v0.2.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.4.2 @@ -51,5 +54,3 @@ require ( golang.org/x/text v0.3.6 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) - -go 1.17 diff --git a/go.sum b/go.sum index a4af1d246..5db31a84a 100644 --- a/go.sum +++ b/go.sum @@ -2,6 +2,7 @@ github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 h1:4EgP6xLAdrD/TR github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa h1:lL66YnJWy1tHlhjSx8fXnpgmv8kQVYnI4ilbYpNB6Zs= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= +github.com/aws/aws-sdk-go v1.34.31/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.40.56 h1:FM2yjR0UUYFzDTMx+mH9Vyw1k1EUUxsAFzk+BjkzANA= github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456 h1:SnUWpAH4lEUoS86woR12h21VMUbDe+DYp88V646wwMI= @@ -33,8 +34,8 @@ github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6 github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/gomodule/redigo v2.0.0+incompatible h1:K/R+8tc58AaqLkqG2Ol3Qk+DR/TlNuhuh457pBFPtt0= -github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/gomodule/redigo v1.8.8 h1:f6cXq6RRfiyrOJEV7p3JhLDlmawGBVBBP1MggY8Mo4E= +github.com/gomodule/redigo v1.8.8/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA= github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -66,14 +67,18 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.16.2 h1:fvDKKRVsaES76SyvWujDttwplVWVZwHL3AJNbk+rDno= -github.com/nyaruka/gocommon v1.16.2/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= +github.com/nyaruka/gocommon v1.5.3/go.mod h1:2ZeBZF9yt20IaAJ4aC1ujojAsFhJBk2IuDvSl7KuQDw= +github.com/nyaruka/gocommon v1.17.0 h1:cTiDLSUgmYJ9OZw752jva0P2rz0utRtv5WGuKFc9kxw= +github.com/nyaruka/gocommon v1.17.0/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= github.com/nyaruka/null v1.1.1/go.mod h1:HSAFbLNOaEhHnoU0VCveCPz0GDtJ3GEtFWhvnBNkhPE= +github.com/nyaruka/phonenumbers v1.0.58/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= github.com/nyaruka/phonenumbers v1.0.71 h1:itkCGhxkQkHrJ6OyZSApdjQVlPmrWs88MF283pPvbFU= github.com/nyaruka/phonenumbers v1.0.71/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= +github.com/nyaruka/redisx v0.2.1 h1:BavpQRCsK5xV2uxPdJJ26yVmjSo+q6bdjWqeNNf0s5w= +github.com/nyaruka/redisx v0.2.1/go.mod h1:cdbAm4y/+oFWu7qFzH2ERPeqRXJC2CtgRhwcBacM4Oc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -94,21 +99,26 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index ee9f30022..a05061655 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -16,8 +16,8 @@ import ( "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" - "github.com/nyaruka/gocommon/rcache" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/redisx" "github.com/patrickmn/go-cache" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -765,7 +765,8 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri defer rc.Close() cacheKey := fmt.Sprintf(mediaCacheKeyPattern, msg.Channel().UUID().String()) - mediaID, err := rcache.Get(rc, cacheKey, mediaURL) + mediaCache := redisx.NewIntervalHash(cacheKey, time.Hour*24, 2) + mediaID, err := mediaCache.Get(rc, mediaURL) if err != nil { return "", logs, errors.Wrapf(err, "error reading media id from redis: %s : %s", cacheKey, mediaURL) } else if mediaID != "" { @@ -823,7 +824,7 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri } // put in cache - err = rcache.Set(rc, cacheKey, mediaURL, mediaID) + err = mediaCache.Set(rc, mediaURL, mediaID) if err != nil { return "", logs, errors.Wrapf(err, "error setting media id in cache") } From 966c851c1885a7478f3a52ca761f9649f0760e12 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 13 Jan 2022 09:07:33 -0500 Subject: [PATCH 049/146] Update CHANGELOG.md for v7.1.12 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a102cf7fb..97b05ce13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.1.12 +---------- + * Update to latest gocommon + * Add instagram handler + v7.1.11 ---------- * More bulk sql tweaks From 93f74a0c454b356ac2217ce41e2e4dea78ed9bfe Mon Sep 17 00:00:00 2001 From: alviriseup Date: Tue, 18 Jan 2022 18:57:46 +0600 Subject: [PATCH 050/146] Added Session Status and modified test case --- handlers/firebase/firebase.go | 12 +++++++----- handlers/firebase/firebase_test.go | 10 +++++----- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/handlers/firebase/firebase.go b/handlers/firebase/firebase.go index 70e6ced82..5e9d414e8 100644 --- a/handlers/firebase/firebase.go +++ b/handlers/firebase/firebase.go @@ -117,11 +117,12 @@ func (h *handler) registerContact(ctx context.Context, channel courier.Channel, type mtPayload struct { Data struct { - Type string `json:"type"` - Title string `json:"title"` - Message string `json:"message"` - MessageID int64 `json:"message_id"` - QuickReplies []string `json:"quick_replies,omitempty"` + Type string `json:"type"` + Title string `json:"title"` + Message string `json:"message"` + MessageID int64 `json:"message_id"` + SessionStatus string `json:"session_status"` + QuickReplies []string `json:"quick_replies,omitempty"` } `json:"data"` Notification *mtNotification `json:"notification,omitempty"` ContentAvailable bool `json:"content_available"` @@ -162,6 +163,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat payload.Data.Title = title payload.Data.Message = part payload.Data.MessageID = int64(msg.ID()) + payload.Data.SessionStatus = msg.SessionStatus() // include any quick replies on the last piece we send if i == len(msgParts)-1 { diff --git a/handlers/firebase/firebase_test.go b/handlers/firebase/firebase_test.go index b245946d7..3d4498602 100644 --- a/handlers/firebase/firebase_test.go +++ b/handlers/firebase/firebase_test.go @@ -77,7 +77,7 @@ var notificationSendTestCases = []ChannelSendTestCase{ Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10},"notification":{"title":"FCMTitle","body":"Simple Message"},"content_available":true,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10,"session_status":""},"notification":{"title":"FCMTitle","body":"Simple Message"},"content_available":true,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, } @@ -87,22 +87,22 @@ var sendTestCases = []ChannelSendTestCase{ Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10},"content_available":false,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message","message_id":10,"session_status":""},"content_available":false,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, {Label: "Long Message", Text: longMsg, - URN: "fcm:250788123123", URNAuth: "auth1", + URN: "fcm:250788123123", URNAuth: "auth1", Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"ate ac.","message_id":10},"content_available":false,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"ate ac.","message_id":10,"session_status":""},"content_available":false,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, {Label: "Quick Reply", Text: "Simple Message", URN: "fcm:250788123123", URNAuth: "auth1", QuickReplies: []string{"yes", "no"}, Attachments: []string{"image/jpeg:https://foo.bar"}, Status: "W", ExternalID: "123456", ResponseBody: `{"success":1, "multicast_id": 123456}`, ResponseStatus: 200, Headers: map[string]string{"Authorization": "key=FCMKey"}, - RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message\nhttps://foo.bar","message_id":10,"quick_replies":["yes","no"]},"content_available":false,"to":"auth1","priority":"high"}`, + RequestBody: `{"data":{"type":"rapidpro","title":"FCMTitle","message":"Simple Message\nhttps://foo.bar","message_id":10,"session_status":"","quick_replies":["yes","no"]},"content_available":false,"to":"auth1","priority":"high"}`, SendPrep: setSendURL}, {Label: "Error", Text: "Error", URN: "fcm:250788123123", URNAuth: "auth1", From a23b58d0d223b1ada1a487c95b1fc7f0cf137923 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 18 Jan 2022 10:41:51 -0500 Subject: [PATCH 051/146] Send db and redis stats to librato in backed heartbeat --- backends/rapidpro/backend.go | 43 ++++++++++++++++++++++++++----- backends/rapidpro/backend_test.go | 5 ++++ 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 32e56c061..6aa335f32 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -12,7 +12,6 @@ import ( "sync" "time" - "github.com/aws/aws-sdk-go/aws/credentials" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" "github.com/nyaruka/courier" @@ -438,10 +437,39 @@ func (b *backend) Heartbeat() error { bulkSize += count } - // log our total + // get our DB and redis stats + dbStats := b.db.Stats() + redisStats := b.redisPool.Stats() + + dbWaitDurationInPeriod := dbStats.WaitDuration - b.dbWaitDuration + dbWaitCountInPeriod := dbStats.WaitCount - b.dbWaitCount + redisWaitDurationInPeriod := redisStats.WaitDuration - b.redisWaitDuration + redisWaitCountInPeriod := redisStats.WaitCount - b.redisWaitCount + + b.dbWaitDuration = dbStats.WaitDuration + b.dbWaitCount = dbStats.WaitCount + b.redisWaitDuration = redisStats.WaitDuration + b.redisWaitCount = redisStats.WaitCount + + librato.Gauge("courier.db_busy", float64(dbStats.InUse)) + librato.Gauge("courier.db_idle", float64(dbStats.Idle)) + librato.Gauge("courier.db_wait_ms", float64(dbWaitDurationInPeriod/time.Millisecond)) + librato.Gauge("courier.db_wait_count", float64(dbWaitCountInPeriod)) + librato.Gauge("courier.redis_wait_ms", float64(redisWaitDurationInPeriod/time.Millisecond)) + librato.Gauge("courier.redis_wait_count", float64(redisWaitCountInPeriod)) librato.Gauge("courier.bulk_queue", float64(bulkSize)) librato.Gauge("courier.priority_queue", float64(prioritySize)) - logrus.WithField("bulk_queue", bulkSize).WithField("priority_queue", prioritySize).Info("heartbeat queue sizes calculated") + + logrus.WithFields(logrus.Fields{ + "db_busy": dbStats.InUse, + "db_idle": dbStats.Idle, + "db_wait_time": dbWaitDurationInPeriod, + "db_wait_count": dbWaitCountInPeriod, + "redis_wait_time": dbWaitDurationInPeriod, + "redis_wait_count": dbWaitCountInPeriod, + "priority_size": prioritySize, + "bulk_size": bulkSize, + }).Info("current analytics") return nil } @@ -753,10 +781,13 @@ type backend struct { db *sqlx.DB redisPool *redis.Pool storage storage.Storage - awsCreds *credentials.Credentials - - popScript *redis.Script stopChan chan bool waitGroup *sync.WaitGroup + + // both sqlx and redis provide wait stats which are cummulative that we need to convert into increments + dbWaitDuration time.Duration + dbWaitCount int64 + redisWaitDuration time.Duration + redisWaitCount int64 } diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index df34f0af5..07298585e 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -680,6 +680,11 @@ func (ts *BackendTestSuite) TestHealth() { ts.Equal(ts.b.Health(), "") } +func (ts *BackendTestSuite) TestHeartbeat() { + // TODO make analytics abstraction layer so we can test what we report + ts.NoError(ts.b.Heartbeat()) +} + func (ts *BackendTestSuite) TestDupes() { r := ts.b.redisPool.Get() defer r.Close() From fd95b68b39c1b1d0d66afec230fe24aef7db0d5c Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 18 Jan 2022 12:02:12 -0500 Subject: [PATCH 052/146] Update CHANGELOG.md for v7.1.13 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 97b05ce13..e2074147a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.1.13 +---------- + * Send db and redis stats to librato in backed heartbeat + * Include session_status in FCM payloads + v7.1.12 ---------- * Update to latest gocommon From 54ca176b9462a43590b4af7a02c2f2e514fe81b4 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 19 Jan 2022 12:17:35 +0200 Subject: [PATCH 053/146] Add support to receive button text from Twilio WhatsApp --- handlers/twiml/twiml.go | 8 +++++++- handlers/twiml/twiml_test.go | 16 ++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/handlers/twiml/twiml.go b/handlers/twiml/twiml.go index c3fde453f..220a1cac5 100644 --- a/handlers/twiml/twiml.go +++ b/handlers/twiml/twiml.go @@ -79,6 +79,7 @@ type moForm struct { To string `validate:"required"` ToCountry string Body string + ButtonText string NumMedia int } @@ -138,8 +139,13 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w form.Body = handlers.DecodePossibleBase64(form.Body) } + text := form.Body + if channel.IsScheme(urns.WhatsAppScheme) && form.ButtonText != "" { + text = form.ButtonText + } + // build our msg - msg := h.Backend().NewIncomingMsg(channel, urn, form.Body).WithExternalID(form.MessageSID) + msg := h.Backend().NewIncomingMsg(channel, urn, text).WithExternalID(form.MessageSID) // process any attached media for i := 0; i < form.NumMedia; i++ { diff --git a/handlers/twiml/twiml_test.go b/handlers/twiml/twiml_test.go index 19ff7d98b..9dd223e64 100644 --- a/handlers/twiml/twiml_test.go +++ b/handlers/twiml/twiml_test.go @@ -54,10 +54,11 @@ var ( twaStatusIDURL = "/c/twa/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status?id=12345" twaStatusInvalidIDURL = "/c/twa/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status?id=asdf" - receiveValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" - receiveMedia = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" - receiveMediaWithMsg = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&Body=Msg&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" - receiveBase64 = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=QmFubm9uIEV4cGxhaW5zIFRoZSBXb3JsZCAuLi4K4oCcVGhlIENhbXAgb2YgdGhlIFNhaW50c%2BKA&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" + receiveValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" + receiveButtonIgnored = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&ButtonText=Confirm&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" + receiveMedia = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" + receiveMediaWithMsg = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&Body=Msg&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" + receiveBase64 = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=QmFubm9uIEV4cGxhaW5zIFRoZSBXb3JsZCAuLi4K4oCcVGhlIENhbXAgb2YgdGhlIFNhaW50c%2BKA&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" statusInvalid = "MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&MessageStatus=huh" statusValid = "MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&MessageStatus=delivered" @@ -67,6 +68,7 @@ var ( tmsReceiveExtra = "ToCountry=US&ToState=&SmsMessageSid=SMbbf29aeb9d380ce2a1c0ae4635ff9dab&NumMedia=0&ToCity=&FromZip=27609&SmsSid=SMbbf29aeb9d380ce2a1c0ae4635ff9dab&FromState=NC&SmsStatus=received&FromCity=RALEIGH&Body=John+Cruz&FromCountry=US&To=384387&ToZip=&NumSegments=1&MessageSid=SMbbf29aeb9d380ce2a1c0ae4635ff9dab&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" waReceiveValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&FromCountry=US&To=whatsapp:%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=whatsapp:%2B14133881111&ApiVersion=2010-04-01" + waReceiveButtonValid = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=Msg&ButtonText=Confirm&FromCountry=US&To=whatsapp:%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=whatsapp:%2B14133881111&ApiVersion=2010-04-01" waReceivePrefixlessURN = "ToCountry=US&ToState=CA&SmsMessageSid=SM681a1f26d9ec591431ce406e8f399525&NumMedia=0&ToCity=&FromZip=60625&SmsSid=SM681a1f26d9ec591431ce406e8f399525&FromState=IL&SmsStatus=received&FromCity=CHICAGO&Body=Msg&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SM681a1f26d9ec591431ce406e8f399525&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" ) @@ -74,6 +76,9 @@ var testCases = []ChannelHandleTestCase{ {Label: "Receive Valid", URL: receiveURL, Data: receiveValid, Status: 200, Response: "", Text: Sp("Msg"), URN: Sp("tel:+14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), PrepRequest: addValidSignature}, + {Label: "Receive Button Ignored", URL: receiveURL, Data: receiveButtonIgnored, Status: 200, Response: "", + Text: Sp("Msg"), URN: Sp("tel:+14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), + PrepRequest: addValidSignature}, {Label: "Receive Invalid Signature", URL: receiveURL, Data: receiveValid, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, {Label: "Receive Missing Signature", URL: receiveURL, Data: receiveValid, Status: 400, Response: "missing request signature"}, @@ -199,6 +204,9 @@ var twaTestCases = []ChannelHandleTestCase{ {Label: "Receive Valid", URL: twaReceiveURL, Data: waReceiveValid, Status: 200, Response: "", Text: Sp("Msg"), URN: Sp("whatsapp:14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), PrepRequest: addValidSignature}, + {Label: "Receive Valid", URL: twaReceiveURL, Data: waReceiveButtonValid, Status: 200, Response: "", + Text: Sp("Confirm"), URN: Sp("whatsapp:14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), + PrepRequest: addValidSignature}, {Label: "Receive Prefixless URN", URL: twaReceiveURL, Data: waReceivePrefixlessURN, Status: 200, Response: "", Text: Sp("Msg"), URN: Sp("whatsapp:14133881111"), ExternalID: Sp("SM681a1f26d9ec591431ce406e8f399525"), PrepRequest: addValidSignature}, From e3f960077c4f0f6cc9703994aaf17d75d5444a9d Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 19 Jan 2022 14:51:11 +0200 Subject: [PATCH 054/146] Support sending WA quick replies when we have attachments too --- handlers/whatsapp/whatsapp.go | 72 ++++++++++++++++++++++++++---- handlers/whatsapp/whatsapp_test.go | 46 +++++++++++++++++++ 2 files changed, 109 insertions(+), 9 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index a05061655..542ddbc45 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -568,6 +568,13 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann var logs []*courier.ChannelLog var err error + parts := handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) + + qrs := msg.QuickReplies() + wppVersion := msg.Channel().ConfigForKey("version", "0").(string) + isInteractiveMsgCompatible := semver.Compare(wppVersion, interactiveMsgMinSupVersion) + isInteractiveMsg := (isInteractiveMsgCompatible >= 0) && (len(qrs) > 0) + if len(msg.Attachments()) > 0 { for attachmentCount, attachment := range msg.Attachments() { @@ -595,7 +602,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann To: msg.URN().Path(), Type: "document", } - if attachmentCount == 0 { + if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() } mediaPayload.Filename, err = utils.BasePathForURL(mediaURL) @@ -611,7 +618,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann To: msg.URN().Path(), Type: "image", } - if attachmentCount == 0 { + if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() } payload.Image = mediaPayload @@ -621,7 +628,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann To: msg.URN().Path(), Type: "video", } - if attachmentCount == 0 { + if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() } payload.Video = mediaPayload @@ -634,6 +641,59 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann break } } + + if isInteractiveMsg { + for i, part := range parts { + if i < (len(parts) - 1) { //if split into more than one message, the first parts will be text and the last interactive + payload := mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + } + payload.Text.Body = part + payloads = append(payloads, payload) + + } else { + payload := mtInteractivePayload{ + To: msg.URN().Path(), + Type: "interactive", + } + + // up to 3 qrs the interactive message will be button type, otherwise it will be list + if len(qrs) <= 3 { + payload.Interactive.Type = "button" + payload.Interactive.Body.Text = part + btns := make([]mtButton, len(qrs)) + for i, qr := range qrs { + btns[i] = mtButton{ + Type: "reply", + } + btns[i].Reply.ID = fmt.Sprint(i) + btns[i].Reply.Title = qr + } + payload.Interactive.Action.Buttons = btns + payloads = append(payloads, payload) + } else { + payload.Interactive.Type = "list" + payload.Interactive.Body.Text = part + payload.Interactive.Action.Button = "Menu" + section := mtSection{ + Rows: make([]mtSectionRow, len(qrs)), + } + for i, qr := range qrs { + section.Rows[i] = mtSectionRow{ + ID: fmt.Sprint(i), + Title: qr, + } + } + payload.Interactive.Action.Sections = []mtSection{ + section, + } + payloads = append(payloads, payload) + } + } + } + } + } else { // do we have a template? var templating *MsgTemplating @@ -684,12 +744,6 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann payloads = append(payloads, payload) } } else { - parts := handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) - - qrs := msg.QuickReplies() - wppVersion := msg.Channel().ConfigForKey("version", "0").(string) - isInteractiveMsgCompatible := semver.Compare(wppVersion, interactiveMsgMinSupVersion) - isInteractiveMsg := (isInteractiveMsgCompatible >= 0) && (len(qrs) > 0) if isInteractiveMsg { for i, part := range parts { diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 653a62051..edb54c7bb 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -605,6 +605,52 @@ var defaultSendTestCases = []ChannelSendTestCase{ ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"interactive","interactive":{"type":"list","body":{"text":"Interactive List Msg"},"action":{"button":"Menu","sections":[{"rows":[{"id":"0","title":"ROW1"},{"id":"1","title":"ROW2"},{"id":"2","title":"ROW3"},{"id":"3","title":"ROW4"}]}]}}}`, SendPrep: setSendURL}, + {Label: "Interactive Button Message Send with attachment", + Text: "Interactive Button Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"BUTTON1"}, + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"image","image":{"link":"https://foo.bar/image.jpg"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"interactive","interactive":{"type":"button","body":{"text":"Interactive Button Msg"},"action":{"buttons":[{"type":"reply","reply":{"id":"0","title":"BUTTON1"}}]}}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, + {Label: "Interactive List Message Send with attachment", + Text: "Interactive List Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"ROW1", "ROW2", "ROW3", "ROW4"}, + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"image","image":{"link":"https://foo.bar/image.jpg"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"interactive","interactive":{"type":"list","body":{"text":"Interactive List Msg"},"action":{"button":"Menu","sections":[{"rows":[{"id":"0","title":"ROW1"},{"id":"1","title":"ROW2"},{"id":"2","title":"ROW3"},{"id":"3","title":"ROW4"}]}]}}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, } var mediaCacheSendTestCases = []ChannelSendTestCase{ From 8b0131886c730175831bd6b0b12a69c3b7a9f155 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 19 Jan 2022 09:10:22 -0500 Subject: [PATCH 055/146] Allow more active redis connections --- backends/rapidpro/backend.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 6aa335f32..a36e87888 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -592,11 +592,11 @@ func (b *backend) Start() error { // create our pool redisPool := &redis.Pool{ Wait: true, // makes callers wait for a connection - MaxActive: 8, // only open this many concurrent connections at once + MaxActive: 36, // only open this many concurrent connections at once MaxIdle: 4, // only keep up to this many idle IdleTimeout: 240 * time.Second, // how long to wait before reaping a connection Dial: func() (redis.Conn, error) { - conn, err := redis.Dial("tcp", fmt.Sprintf("%s", redisURL.Host)) + conn, err := redis.Dial("tcp", redisURL.Host) if err != nil { return nil, err } From cbfd24d843f2ab48584a2ea833426692c23cac57 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 19 Jan 2022 09:59:06 -0500 Subject: [PATCH 056/146] Update CHANGELOG.md for v7.1.14 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2074147a..d0ee8624b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +v7.1.14 +---------- + * Allow more active redis connections + * Support sending WA quick replies when we have attachments too + * Add support to receive button text from Twilio WhatsApp + v7.1.13 ---------- * Send db and redis stats to librato in backed heartbeat From 3c8c8012509b0970020a8b625172ced5cd497bf0 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Tue, 18 Jan 2022 14:21:55 +0200 Subject: [PATCH 057/146] Update HX user_data parameter --- handlers/highconnection/highconnection.go | 2 +- handlers/highconnection/highconnection_test.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/handlers/highconnection/highconnection.go b/handlers/highconnection/highconnection.go index f722f17a2..c03c8c2ce 100644 --- a/handlers/highconnection/highconnection.go +++ b/handlers/highconnection/highconnection.go @@ -142,7 +142,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "to": []string{msg.URN().Path()}, "ret_id": []string{msg.ID().String()}, "datacoding": []string{"8"}, - "userdata": []string{"textit"}, + "user_data": []string{"textit"}, "ret_url": []string{statusURL}, "ret_mo_url": []string{receiveURL}, } diff --git a/handlers/highconnection/highconnection_test.go b/handlers/highconnection/highconnection_test.go index 4869abcee..61b419cff 100644 --- a/handlers/highconnection/highconnection_test.go +++ b/handlers/highconnection/highconnection_test.go @@ -64,7 +64,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ "to": "+250788383383", "ret_id": "10", "datacoding": "8", - "userdata": "textit", + "user_data": "textit", "ret_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status", "ret_mo_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive", }, @@ -81,7 +81,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ "to": "+250788383383", "ret_id": "10", "datacoding": "8", - "userdata": "textit", + "user_data": "textit", "ret_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status", "ret_mo_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive", }, @@ -98,7 +98,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ "to": "+250788383383", "ret_id": "10", "datacoding": "8", - "userdata": "textit", + "user_data": "textit", "ret_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status", "ret_mo_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive", }, @@ -116,7 +116,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ "to": "+250788383383", "ret_id": "10", "datacoding": "8", - "userdata": "textit", + "user_data": "textit", "ret_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status", "ret_mo_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive", }, From 04bd7559602da8b4ee6799e2b09c59b033feb68b Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 19 Jan 2022 11:48:31 +0200 Subject: [PATCH 058/146] Send flow name as user_data to HX --- backends/rapidpro/msg.go | 25 ++++++++++++++++ handlers/highconnection/highconnection.go | 2 +- .../highconnection/highconnection_test.go | 30 ++++++++++++++++--- handlers/test.go | 4 +++ msg.go | 5 ++++ test.go | 22 ++++++++++++++ 6 files changed, 83 insertions(+), 5 deletions(-) diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index cd96c5b4e..fca7d334f 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -518,6 +518,8 @@ type DBMsg struct { SessionWaitStartedOn_ *time.Time `json:"session_wait_started_on,omitempty"` SessionStatus_ string `json:"session_status,omitempty"` + Flow_ json.RawMessage `json:"flow,omitempty"` + channel *DBChannel workerToken queue.WorkerToken alreadyWritten bool @@ -542,6 +544,26 @@ func (m *DBMsg) IsResend() bool { return m.IsResend_ } func (m *DBMsg) Channel() courier.Channel { return m.channel } func (m *DBMsg) SessionStatus() string { return m.SessionStatus_ } +func (m *DBMsg) Flow() json.RawMessage { + return m.Flow_ +} + +func (m *DBMsg) FlowName() string { + if m.Flow_ == nil { + return "" + } + name, _, _, _ := jsonparser.Get(m.Flow_, "name") + return string(name) +} + +func (m *DBMsg) FlowUUID() string { + if m.Flow_ == nil { + return "" + } + flowUUID, _, _, _ := jsonparser.Get(m.Flow_, "uuid") + return string(flowUUID) +} + func (m *DBMsg) QuickReplies() []string { if m.quickReplies != nil { return m.quickReplies @@ -597,6 +619,9 @@ func (m *DBMsg) WithUUID(uuid courier.MsgUUID) courier.Msg { m.UUID_ = uuid; ret // WithMetadata can be used to add metadata to a Msg func (m *DBMsg) WithMetadata(metadata json.RawMessage) courier.Msg { m.Metadata_ = metadata; return m } +// WithFlow can be used to add flow to a Msg +func (m *DBMsg) WithFlow(flow json.RawMessage) courier.Msg { m.Flow_ = flow; return m } + // WithAttachment can be used to append to the media urls for a message func (m *DBMsg) WithAttachment(url string) courier.Msg { m.Attachments_ = append(m.Attachments_, url) diff --git a/handlers/highconnection/highconnection.go b/handlers/highconnection/highconnection.go index c03c8c2ce..70f2cec7c 100644 --- a/handlers/highconnection/highconnection.go +++ b/handlers/highconnection/highconnection.go @@ -142,7 +142,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "to": []string{msg.URN().Path()}, "ret_id": []string{msg.ID().String()}, "datacoding": []string{"8"}, - "user_data": []string{"textit"}, + "user_data": []string{msg.FlowName()}, "ret_url": []string{statusURL}, "ret_mo_url": []string{receiveURL}, } diff --git a/handlers/highconnection/highconnection_test.go b/handlers/highconnection/highconnection_test.go index 61b419cff..8a7b8f370 100644 --- a/handlers/highconnection/highconnection_test.go +++ b/handlers/highconnection/highconnection_test.go @@ -1,6 +1,7 @@ package highconnection import ( + "encoding/json" "net/http/httptest" "testing" "time" @@ -57,6 +58,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Text: "Simple Message", URN: "tel:+250788383383", Status: "W", + Flow: json.RawMessage(`{"uuid": "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", "name": "Favorites"}`), URLParams: map[string]string{ "accountid": "Username", "password": "Password", @@ -64,7 +66,24 @@ var defaultSendTestCases = []ChannelSendTestCase{ "to": "+250788383383", "ret_id": "10", "datacoding": "8", - "user_data": "textit", + "user_data": "Favorites", + "ret_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status", + "ret_mo_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive", + }, + ResponseStatus: 200, + SendPrep: setSendURL}, + {Label: "Plain Send without flow", + Text: "Simple Message", + URN: "tel:+250788383383", + Status: "W", + URLParams: map[string]string{ + "accountid": "Username", + "password": "Password", + "text": "Simple Message", + "to": "+250788383383", + "ret_id": "10", + "datacoding": "8", + "user_data": "", "ret_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status", "ret_mo_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive", }, @@ -74,6 +93,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Text: "☺", URN: "tel:+250788383383", Status: "W", + Flow: json.RawMessage(`{"uuid": "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", "name": "Favorites"}`), URLParams: map[string]string{ "accountid": "Username", "password": "Password", @@ -81,7 +101,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ "to": "+250788383383", "ret_id": "10", "datacoding": "8", - "user_data": "textit", + "user_data": "Favorites", "ret_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status", "ret_mo_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive", }, @@ -91,6 +111,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Text: "This is a longer message than 160 characters and will cause us to split it into two separate parts, isn't that right but it is even longer than before I say, I need to keep adding more things to make it work", URN: "tel:+250788383383", Status: "W", + Flow: json.RawMessage(`{"uuid": "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", "name": "Favorites"}`), URLParams: map[string]string{ "accountid": "Username", "password": "Password", @@ -98,7 +119,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ "to": "+250788383383", "ret_id": "10", "datacoding": "8", - "user_data": "textit", + "user_data": "Favorites", "ret_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status", "ret_mo_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive", }, @@ -109,6 +130,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, URN: "tel:+250788383383", Status: "W", + Flow: json.RawMessage(`{"uuid": "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", "name": "Favorites"}`), URLParams: map[string]string{ "accountid": "Username", "password": "Password", @@ -116,7 +138,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ "to": "+250788383383", "ret_id": "10", "datacoding": "8", - "user_data": "textit", + "user_data": "Favorites", "ret_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/status", "ret_mo_url": "https://localhost/c/hx/8eb23e93-5ecb-45ba-b726-3b064e0c56ab/receive", }, diff --git a/handlers/test.go b/handlers/test.go index 9f5a88e3a..0c67d1f4e 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -89,6 +89,7 @@ type ChannelSendTestCase struct { HighPriority bool ResponseToExternalID string Metadata json.RawMessage + Flow json.RawMessage ResponseStatus int ResponseBody string @@ -231,6 +232,9 @@ func RunChannelSendTestCases(t *testing.T, channel courier.Channel, handler cour if len(testCase.Metadata) > 0 { msg.WithMetadata(testCase.Metadata) } + if len(testCase.Flow) > 0 { + msg.WithFlow(testCase.Flow) + } var testRequest *http.Request server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/msg.go b/msg.go index 809370f20..b45b51a5b 100644 --- a/msg.go +++ b/msg.go @@ -98,6 +98,10 @@ type Msg interface { ResponseToExternalID() string IsResend() bool + Flow() json.RawMessage + FlowName() string + FlowUUID() string + Channel() Channel ReceivedOn() *time.Time @@ -113,6 +117,7 @@ type Msg interface { WithAttachment(url string) Msg WithURNAuth(auth string) Msg WithMetadata(metadata json.RawMessage) Msg + WithFlow(flow json.RawMessage) Msg EventID() int64 SessionStatus() string diff --git a/test.go b/test.go index 012c67a1b..385375e71 100644 --- a/test.go +++ b/test.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/buger/jsonparser" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/gocommon/uuids" @@ -562,6 +563,7 @@ type mockMsg struct { metadata json.RawMessage alreadyWritten bool isResend bool + flow json.RawMessage receivedOn *time.Time sentOn *time.Time @@ -570,6 +572,24 @@ type mockMsg struct { func (m *mockMsg) SessionStatus() string { return "" } +func (m *mockMsg) Flow() json.RawMessage { return m.flow } + +func (m *mockMsg) FlowName() string { + if m.flow == nil { + return "" + } + name, _, _, _ := jsonparser.Get(m.flow, "name") + return string(name) +} + +func (m *mockMsg) FlowUUID() string { + if m.flow == nil { + return "" + } + flowUUID, _, _, _ := jsonparser.Get(m.flow, "uuid") + return string(flowUUID) +} + func (m *mockMsg) Channel() Channel { return m.channel } func (m *mockMsg) ID() MsgID { return m.id } func (m *mockMsg) EventID() int64 { return int64(m.id) } @@ -603,6 +623,8 @@ func (m *mockMsg) WithAttachment(url string) Msg { } func (m *mockMsg) WithMetadata(metadata json.RawMessage) Msg { m.metadata = metadata; return m } +func (m *mockMsg) WithFlow(flow json.RawMessage) Msg { m.flow = flow; return m } + //----------------------------------------------------------------------------- // Mock status implementation //----------------------------------------------------------------------------- From dccf0dc2f27c3310396c253199fae2aed632a7a8 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 20 Jan 2022 14:11:58 +0200 Subject: [PATCH 059/146] Add and use MsgFlowRef type --- backends/rapidpro/backend_test.go | 3 +++ backends/rapidpro/msg.go | 24 +++---------------- handlers/highconnection/highconnection.go | 2 +- .../highconnection/highconnection_test.go | 21 ++++++++++++---- handlers/test.go | 10 +++++--- msg.go | 13 ++++++---- test.go | 23 +++--------------- 7 files changed, 42 insertions(+), 54 deletions(-) diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index 07298585e..aa738ae91 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -109,6 +109,7 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { "contact_id": 30, "contact_urn_id": 14, "error_count": 0, + "flow": {"uuid": "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", "name": "Favorites"}, "modified_on": "2017-07-21T19:22:23.254133Z", "id": 204, "channel_uuid": "f3ad3eb6-d00d-4dc3-92e9-9f34f32940ba", @@ -140,6 +141,8 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { ts.Equal("external-id", msg.ResponseToExternalID()) ts.True(msg.HighPriority()) ts.True(msg.IsResend()) + ts.Equal("Favorites", msg.Flow().Name) + ts.Equal("9de3663f-c5c5-4c92-9f45-ecbc09abcc85", msg.Flow().UUID) msgJSONNoQR := `{ "status": "P", diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index fca7d334f..28cd1fac8 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -518,7 +518,7 @@ type DBMsg struct { SessionWaitStartedOn_ *time.Time `json:"session_wait_started_on,omitempty"` SessionStatus_ string `json:"session_status,omitempty"` - Flow_ json.RawMessage `json:"flow,omitempty"` + Flow_ courier.MsgFlowRef `json:"flow,omitempty"` channel *DBChannel workerToken queue.WorkerToken @@ -544,25 +544,7 @@ func (m *DBMsg) IsResend() bool { return m.IsResend_ } func (m *DBMsg) Channel() courier.Channel { return m.channel } func (m *DBMsg) SessionStatus() string { return m.SessionStatus_ } -func (m *DBMsg) Flow() json.RawMessage { - return m.Flow_ -} - -func (m *DBMsg) FlowName() string { - if m.Flow_ == nil { - return "" - } - name, _, _, _ := jsonparser.Get(m.Flow_, "name") - return string(name) -} - -func (m *DBMsg) FlowUUID() string { - if m.Flow_ == nil { - return "" - } - flowUUID, _, _, _ := jsonparser.Get(m.Flow_, "uuid") - return string(flowUUID) -} +func (m *DBMsg) Flow() courier.MsgFlowRef { return m.Flow_ } func (m *DBMsg) QuickReplies() []string { if m.quickReplies != nil { @@ -620,7 +602,7 @@ func (m *DBMsg) WithUUID(uuid courier.MsgUUID) courier.Msg { m.UUID_ = uuid; ret func (m *DBMsg) WithMetadata(metadata json.RawMessage) courier.Msg { m.Metadata_ = metadata; return m } // WithFlow can be used to add flow to a Msg -func (m *DBMsg) WithFlow(flow json.RawMessage) courier.Msg { m.Flow_ = flow; return m } +func (m *DBMsg) WithFlow(flow courier.MsgFlowRef) courier.Msg { m.Flow_ = flow; return m } // WithAttachment can be used to append to the media urls for a message func (m *DBMsg) WithAttachment(url string) courier.Msg { diff --git a/handlers/highconnection/highconnection.go b/handlers/highconnection/highconnection.go index 70f2cec7c..e5fa0d418 100644 --- a/handlers/highconnection/highconnection.go +++ b/handlers/highconnection/highconnection.go @@ -142,7 +142,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "to": []string{msg.URN().Path()}, "ret_id": []string{msg.ID().String()}, "datacoding": []string{"8"}, - "user_data": []string{msg.FlowName()}, + "user_data": []string{msg.Flow().Name}, "ret_url": []string{statusURL}, "ret_mo_url": []string{receiveURL}, } diff --git a/handlers/highconnection/highconnection_test.go b/handlers/highconnection/highconnection_test.go index 8a7b8f370..9376bbb4f 100644 --- a/handlers/highconnection/highconnection_test.go +++ b/handlers/highconnection/highconnection_test.go @@ -1,7 +1,6 @@ package highconnection import ( - "encoding/json" "net/http/httptest" "testing" "time" @@ -58,7 +57,10 @@ var defaultSendTestCases = []ChannelSendTestCase{ Text: "Simple Message", URN: "tel:+250788383383", Status: "W", - Flow: json.RawMessage(`{"uuid": "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", "name": "Favorites"}`), + Flow: &struct { + UUID string + Name string + }{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, URLParams: map[string]string{ "accountid": "Username", "password": "Password", @@ -93,7 +95,10 @@ var defaultSendTestCases = []ChannelSendTestCase{ Text: "☺", URN: "tel:+250788383383", Status: "W", - Flow: json.RawMessage(`{"uuid": "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", "name": "Favorites"}`), + Flow: &struct { + UUID string + Name string + }{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, URLParams: map[string]string{ "accountid": "Username", "password": "Password", @@ -111,7 +116,10 @@ var defaultSendTestCases = []ChannelSendTestCase{ Text: "This is a longer message than 160 characters and will cause us to split it into two separate parts, isn't that right but it is even longer than before I say, I need to keep adding more things to make it work", URN: "tel:+250788383383", Status: "W", - Flow: json.RawMessage(`{"uuid": "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", "name": "Favorites"}`), + Flow: &struct { + UUID string + Name string + }{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, URLParams: map[string]string{ "accountid": "Username", "password": "Password", @@ -130,7 +138,10 @@ var defaultSendTestCases = []ChannelSendTestCase{ Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, URN: "tel:+250788383383", Status: "W", - Flow: json.RawMessage(`{"uuid": "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", "name": "Favorites"}`), + Flow: &struct { + UUID string + Name string + }{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, URLParams: map[string]string{ "accountid": "Username", "password": "Password", diff --git a/handlers/test.go b/handlers/test.go index 0c67d1f4e..c4fe86250 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -89,7 +89,10 @@ type ChannelSendTestCase struct { HighPriority bool ResponseToExternalID string Metadata json.RawMessage - Flow json.RawMessage + Flow *struct { + UUID string + Name string + } ResponseStatus int ResponseBody string @@ -232,8 +235,9 @@ func RunChannelSendTestCases(t *testing.T, channel courier.Channel, handler cour if len(testCase.Metadata) > 0 { msg.WithMetadata(testCase.Metadata) } - if len(testCase.Flow) > 0 { - msg.WithFlow(testCase.Flow) + if testCase.Flow != nil { + var flowRef = courier.MsgFlowRef{UUID: testCase.Flow.UUID, Name: testCase.Flow.Name} + msg.WithFlow(flowRef) } var testRequest *http.Request diff --git a/msg.go b/msg.go index b45b51a5b..7721d8be7 100644 --- a/msg.go +++ b/msg.go @@ -78,6 +78,13 @@ func NewMsgUUIDFromString(uuidString string) MsgUUID { return MsgUUID{uuid} } +type MsgFlowRef struct { + UUID string `json:"uuid" validate:"uuid4"` + Name string `json:"name"` +} + +var NilMsgFlowRef = MsgFlowRef{"", ""} + //----------------------------------------------------------------------------- // Msg interface //----------------------------------------------------------------------------- @@ -98,9 +105,7 @@ type Msg interface { ResponseToExternalID() string IsResend() bool - Flow() json.RawMessage - FlowName() string - FlowUUID() string + Flow() MsgFlowRef Channel() Channel @@ -117,7 +122,7 @@ type Msg interface { WithAttachment(url string) Msg WithURNAuth(auth string) Msg WithMetadata(metadata json.RawMessage) Msg - WithFlow(flow json.RawMessage) Msg + WithFlow(flow MsgFlowRef) Msg EventID() int64 SessionStatus() string diff --git a/test.go b/test.go index 385375e71..b620925a9 100644 --- a/test.go +++ b/test.go @@ -11,7 +11,6 @@ import ( "sync" "time" - "github.com/buger/jsonparser" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/gocommon/uuids" @@ -563,7 +562,7 @@ type mockMsg struct { metadata json.RawMessage alreadyWritten bool isResend bool - flow json.RawMessage + flow MsgFlowRef receivedOn *time.Time sentOn *time.Time @@ -572,23 +571,7 @@ type mockMsg struct { func (m *mockMsg) SessionStatus() string { return "" } -func (m *mockMsg) Flow() json.RawMessage { return m.flow } - -func (m *mockMsg) FlowName() string { - if m.flow == nil { - return "" - } - name, _, _, _ := jsonparser.Get(m.flow, "name") - return string(name) -} - -func (m *mockMsg) FlowUUID() string { - if m.flow == nil { - return "" - } - flowUUID, _, _, _ := jsonparser.Get(m.flow, "uuid") - return string(flowUUID) -} +func (m *mockMsg) Flow() MsgFlowRef { return m.flow } func (m *mockMsg) Channel() Channel { return m.channel } func (m *mockMsg) ID() MsgID { return m.id } @@ -623,7 +606,7 @@ func (m *mockMsg) WithAttachment(url string) Msg { } func (m *mockMsg) WithMetadata(metadata json.RawMessage) Msg { m.metadata = metadata; return m } -func (m *mockMsg) WithFlow(flow json.RawMessage) Msg { m.flow = flow; return m } +func (m *mockMsg) WithFlow(flow MsgFlowRef) Msg { m.flow = flow; return m } //----------------------------------------------------------------------------- // Mock status implementation From 3de06510bd063051a1be8370a88dd0121a5270f8 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 20 Jan 2022 11:22:47 -0300 Subject: [PATCH 060/146] Fix: Gujarati whatsapp language code --- handlers/whatsapp/whatsapp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 542ddbc45..7c0ac4889 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -1152,7 +1152,7 @@ var languageMap = map[string]string{ "fra": "fr", // French "deu": "de", // German "ell": "el", // Greek - "gul": "gu", // Gujarati + "guj": "gu", // Gujarati "hau": "ha", // Hausa "enb": "he", // Hebrew "hin": "hi", // Hindi From 62917587210cc1ec3d616fff966522106886af40 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Mon, 24 Jan 2022 09:40:52 +0200 Subject: [PATCH 061/146] Rename FlowReference and use pointer --- backends/rapidpro/backend_test.go | 9 +++++++-- backends/rapidpro/msg.go | 20 +++++++++++++++++--- handlers/highconnection/highconnection.go | 2 +- handlers/test.go | 4 ++-- msg.go | 10 +++++----- test.go | 21 ++++++++++++++++++--- 6 files changed, 50 insertions(+), 16 deletions(-) diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index aa738ae91..993080438 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -141,8 +141,10 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { ts.Equal("external-id", msg.ResponseToExternalID()) ts.True(msg.HighPriority()) ts.True(msg.IsResend()) - ts.Equal("Favorites", msg.Flow().Name) - ts.Equal("9de3663f-c5c5-4c92-9f45-ecbc09abcc85", msg.Flow().UUID) + flow_ref := courier.FlowReference{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"} + ts.Equal(msg.Flow(), &flow_ref) + ts.Equal("Favorites", msg.FlowName()) + ts.Equal("9de3663f-c5c5-4c92-9f45-ecbc09abcc85", msg.FlowUUID()) msgJSONNoQR := `{ "status": "P", @@ -176,6 +178,9 @@ func (ts *BackendTestSuite) TestMsgUnmarshal() { ts.Equal("", msg.Topic()) ts.Equal("", msg.ResponseToExternalID()) ts.False(msg.IsResend()) + ts.Nil(msg.Flow()) + ts.Equal("", msg.FlowName()) + ts.Equal("", msg.FlowUUID()) } func (ts *BackendTestSuite) TestCheckMsgExists() { diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index 28cd1fac8..92460a9a5 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -518,7 +518,7 @@ type DBMsg struct { SessionWaitStartedOn_ *time.Time `json:"session_wait_started_on,omitempty"` SessionStatus_ string `json:"session_status,omitempty"` - Flow_ courier.MsgFlowRef `json:"flow,omitempty"` + Flow_ *courier.FlowReference `json:"flow,omitempty"` channel *DBChannel workerToken queue.WorkerToken @@ -544,7 +544,21 @@ func (m *DBMsg) IsResend() bool { return m.IsResend_ } func (m *DBMsg) Channel() courier.Channel { return m.channel } func (m *DBMsg) SessionStatus() string { return m.SessionStatus_ } -func (m *DBMsg) Flow() courier.MsgFlowRef { return m.Flow_ } +func (m *DBMsg) Flow() *courier.FlowReference { return m.Flow_ } + +func (m *DBMsg) FlowName() string { + if m.Flow_ == nil { + return "" + } + return m.Flow_.Name +} + +func (m *DBMsg) FlowUUID() string { + if m.Flow_ == nil { + return "" + } + return m.Flow_.UUID +} func (m *DBMsg) QuickReplies() []string { if m.quickReplies != nil { @@ -602,7 +616,7 @@ func (m *DBMsg) WithUUID(uuid courier.MsgUUID) courier.Msg { m.UUID_ = uuid; ret func (m *DBMsg) WithMetadata(metadata json.RawMessage) courier.Msg { m.Metadata_ = metadata; return m } // WithFlow can be used to add flow to a Msg -func (m *DBMsg) WithFlow(flow courier.MsgFlowRef) courier.Msg { m.Flow_ = flow; return m } +func (m *DBMsg) WithFlow(flow *courier.FlowReference) courier.Msg { m.Flow_ = flow; return m } // WithAttachment can be used to append to the media urls for a message func (m *DBMsg) WithAttachment(url string) courier.Msg { diff --git a/handlers/highconnection/highconnection.go b/handlers/highconnection/highconnection.go index e5fa0d418..70f2cec7c 100644 --- a/handlers/highconnection/highconnection.go +++ b/handlers/highconnection/highconnection.go @@ -142,7 +142,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "to": []string{msg.URN().Path()}, "ret_id": []string{msg.ID().String()}, "datacoding": []string{"8"}, - "user_data": []string{msg.Flow().Name}, + "user_data": []string{msg.FlowName()}, "ret_url": []string{statusURL}, "ret_mo_url": []string{receiveURL}, } diff --git a/handlers/test.go b/handlers/test.go index c4fe86250..31ffb22b6 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -236,8 +236,8 @@ func RunChannelSendTestCases(t *testing.T, channel courier.Channel, handler cour msg.WithMetadata(testCase.Metadata) } if testCase.Flow != nil { - var flowRef = courier.MsgFlowRef{UUID: testCase.Flow.UUID, Name: testCase.Flow.Name} - msg.WithFlow(flowRef) + var flowRef = courier.FlowReference{UUID: testCase.Flow.UUID, Name: testCase.Flow.Name} + msg.WithFlow(&flowRef) } var testRequest *http.Request diff --git a/msg.go b/msg.go index 7721d8be7..c94dd6491 100644 --- a/msg.go +++ b/msg.go @@ -78,13 +78,11 @@ func NewMsgUUIDFromString(uuidString string) MsgUUID { return MsgUUID{uuid} } -type MsgFlowRef struct { +type FlowReference struct { UUID string `json:"uuid" validate:"uuid4"` Name string `json:"name"` } -var NilMsgFlowRef = MsgFlowRef{"", ""} - //----------------------------------------------------------------------------- // Msg interface //----------------------------------------------------------------------------- @@ -105,7 +103,9 @@ type Msg interface { ResponseToExternalID() string IsResend() bool - Flow() MsgFlowRef + Flow() *FlowReference + FlowName() string + FlowUUID() string Channel() Channel @@ -122,7 +122,7 @@ type Msg interface { WithAttachment(url string) Msg WithURNAuth(auth string) Msg WithMetadata(metadata json.RawMessage) Msg - WithFlow(flow MsgFlowRef) Msg + WithFlow(flow *FlowReference) Msg EventID() int64 SessionStatus() string diff --git a/test.go b/test.go index b620925a9..69bcef185 100644 --- a/test.go +++ b/test.go @@ -562,7 +562,8 @@ type mockMsg struct { metadata json.RawMessage alreadyWritten bool isResend bool - flow MsgFlowRef + + flow *FlowReference receivedOn *time.Time sentOn *time.Time @@ -571,7 +572,21 @@ type mockMsg struct { func (m *mockMsg) SessionStatus() string { return "" } -func (m *mockMsg) Flow() MsgFlowRef { return m.flow } +func (m *mockMsg) Flow() *FlowReference { return m.flow } + +func (m *mockMsg) FlowName() string { + if m.flow == nil { + return "" + } + return m.flow.Name +} + +func (m *mockMsg) FlowUUID() string { + if m.flow == nil { + return "" + } + return m.flow.UUID +} func (m *mockMsg) Channel() Channel { return m.channel } func (m *mockMsg) ID() MsgID { return m.id } @@ -606,7 +621,7 @@ func (m *mockMsg) WithAttachment(url string) Msg { } func (m *mockMsg) WithMetadata(metadata json.RawMessage) Msg { m.metadata = metadata; return m } -func (m *mockMsg) WithFlow(flow MsgFlowRef) Msg { m.flow = flow; return m } +func (m *mockMsg) WithFlow(flow *FlowReference) Msg { m.flow = flow; return m } //----------------------------------------------------------------------------- // Mock status implementation From 950141dcc8dcf8f5162e5eadc91d979d37a234b2 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Tue, 25 Jan 2022 10:00:11 +0200 Subject: [PATCH 062/146] use FlowReference type --- .../highconnection/highconnection_test.go | 20 ++++--------------- handlers/test.go | 8 ++------ 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/handlers/highconnection/highconnection_test.go b/handlers/highconnection/highconnection_test.go index 9376bbb4f..8a6d90897 100644 --- a/handlers/highconnection/highconnection_test.go +++ b/handlers/highconnection/highconnection_test.go @@ -57,10 +57,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Text: "Simple Message", URN: "tel:+250788383383", Status: "W", - Flow: &struct { - UUID string - Name string - }{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, + Flow: &courier.FlowReference{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, URLParams: map[string]string{ "accountid": "Username", "password": "Password", @@ -95,10 +92,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Text: "☺", URN: "tel:+250788383383", Status: "W", - Flow: &struct { - UUID string - Name string - }{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, + Flow: &courier.FlowReference{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, URLParams: map[string]string{ "accountid": "Username", "password": "Password", @@ -116,10 +110,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Text: "This is a longer message than 160 characters and will cause us to split it into two separate parts, isn't that right but it is even longer than before I say, I need to keep adding more things to make it work", URN: "tel:+250788383383", Status: "W", - Flow: &struct { - UUID string - Name string - }{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, + Flow: &courier.FlowReference{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, URLParams: map[string]string{ "accountid": "Username", "password": "Password", @@ -138,10 +129,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, URN: "tel:+250788383383", Status: "W", - Flow: &struct { - UUID string - Name string - }{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, + Flow: &courier.FlowReference{UUID: "9de3663f-c5c5-4c92-9f45-ecbc09abcc85", Name: "Favorites"}, URLParams: map[string]string{ "accountid": "Username", "password": "Password", diff --git a/handlers/test.go b/handlers/test.go index 31ffb22b6..9d245c0d2 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -89,10 +89,7 @@ type ChannelSendTestCase struct { HighPriority bool ResponseToExternalID string Metadata json.RawMessage - Flow *struct { - UUID string - Name string - } + Flow *courier.FlowReference ResponseStatus int ResponseBody string @@ -236,8 +233,7 @@ func RunChannelSendTestCases(t *testing.T, channel courier.Channel, handler cour msg.WithMetadata(testCase.Metadata) } if testCase.Flow != nil { - var flowRef = courier.FlowReference{UUID: testCase.Flow.UUID, Name: testCase.Flow.Name} - msg.WithFlow(&flowRef) + msg.WithFlow(testCase.Flow) } var testRequest *http.Request From 58a64d9ce0668ddd13ca6aed17a90a262f6dd073 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 25 Jan 2022 09:09:13 -0800 Subject: [PATCH 063/146] Update CHANGELOG.md for v7.1.15 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d0ee8624b..4790c3fe5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.1.15 +---------- + * Fix Gujarati whatsapp language code + * Send flow name as user_data to HX + v7.1.14 ---------- * Allow more active redis connections From d30a0dd3f6279f7c0243cf6bc87b2cf6bcabf9ee Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 3 Feb 2022 18:36:36 +0200 Subject: [PATCH 064/146] Add support for channel paused bulk queue by redis key --- queue/queue.go | 8 ++++++++ queue/queue_test.go | 27 ++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/queue/queue.go b/queue/queue.go index da6b96e83..1f62f7d74 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -123,6 +123,14 @@ var luaPop = redis.NewScript(2, `-- KEYS: [EpochMS QueueType] -- if we didn't find one, try again from our bulk queue if not result[1] or isFutureResult then + -- check if we are paused for bulk queue + local pausedBulkKey = "paused_bulk:" .. queueName + local pausedBulk = redis.call("get", pausedBulkKey) + if pausedBulk then + return {"retry", ""} + end + + -- we are not pause check our bulk queue local bulkQueue = queue .. "/0" local bulkResult = redis.call("zrangebyscore", bulkQueue, 0, "+inf", "WITHSCORES", "LIMIT", 0, 1) diff --git a/queue/queue_test.go b/queue/queue_test.go index 81cd415a3..8e6641382 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -60,6 +60,19 @@ func TestLua(t *testing.T) { delay := time.Second*2 - time.Duration(time.Now().UnixNano()%int64(time.Second)) time.Sleep(delay) + conn.Do("set", "paused_bulk:chan1", "engaged") + conn.Do("EXPIRE", "paused_bulk:chan1", 5) + + // we have the rate limit set, + queue, value, err := PopFromQueue(conn, "msgs") + assert.NoError(err) + if value != "" && queue != EmptyQueue { + t.Fatal("Should be paused") + } + + // When the redis paused key is remove, we get the values from bulk queue/low priority + conn.Do("DEL", "paused_bulk:chan1") + // pop 10 items off for i := 0; i < 10; i++ { queue, value, err := PopFromQueue(conn, "msgs") @@ -69,7 +82,8 @@ func TestLua(t *testing.T) { } // next value should be throttled - queue, value, err := PopFromQueue(conn, "msgs") + queue, value, err = PopFromQueue(conn, "msgs") + assert.NoError(err) if value != "" && queue != EmptyQueue { t.Fatal("Should be throttled") } @@ -85,6 +99,7 @@ func TestLua(t *testing.T) { // adding more items shouldn't change that queue, value, err = PopFromQueue(conn, "msgs") + assert.NoError(err) if value != "" && queue != EmptyQueue { t.Fatal("Should be throttled") } @@ -104,11 +119,18 @@ func TestLua(t *testing.T) { err = PushOntoQueue(conn, "msgs", "chan1", rate, `[{"id":31}]`, HighPriority) assert.NoError(err) + // make sure pause bulk key do not prevent use to get from the high priority queue + conn.Do("set", "paused_bulk:chan1", "engaged") + conn.Do("EXPIRE", "paused_bulk:chan1", 5) + queue, value, err = PopFromQueue(conn, "msgs") assert.NoError(err) assert.Equal(WorkerToken("msgs:chan1|10"), queue) assert.Equal(`{"id":31}`, value) + // make sure paused is not present for more tests + conn.Do("DEL", "paused_bulk:chan1") + // should get next five bulk msgs fine for i := 10; i < 15; i++ { queue, value, err := PopFromQueue(conn, "msgs") @@ -119,6 +141,7 @@ func TestLua(t *testing.T) { // push on a compound message err = PushOntoQueue(conn, "msgs", "chan1", rate, `[{"id":32}, {"id":33}]`, HighPriority) + assert.NoError(err) queue, value, err = PopFromQueue(conn, "msgs") assert.NoError(err) @@ -176,12 +199,14 @@ func TestLua(t *testing.T) { // we have the rate limit set, queue, value, err = PopFromQueue(conn, "msgs") + assert.NoError(err) if value != "" && queue != EmptyQueue { t.Fatal("Should be throttled") } time.Sleep(2 * time.Second) queue, value, err = PopFromQueue(conn, "msgs") + assert.NoError(err) if value != "" && queue != EmptyQueue { t.Fatal("Should be throttled") } From a349d84d9ea0f6719d3d005f7fdf9af6ad655931 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 3 Feb 2022 18:50:51 +0200 Subject: [PATCH 065/146] Pause WA channel bulk queue when we hit the spam rate limit --- handlers/whatsapp/whatsapp.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 7c0ac4889..b766b929b 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -918,6 +918,18 @@ func sendWhatsAppMsg(rc redis.Conn, msg courier.Msg, sendPath *url.URL, payload // handle send msg errors if err == nil && len(errPayload.Errors) > 0 { + if hasTiersError(*errPayload) { + pausedBulkKey := fmt.Sprintf("paused_bulk:%s", msg.Channel().UUID().String()) + rc.Do("set", pausedBulkKey, "engaged") + + // The WA tiers spam rate limit hit + // We pause the bulk queue for 24 hours and 5min + rc.Do("expire", pausedBulkKey, (60*60*24)+(5*60)) + + err := errors.Errorf("received error from send endpoint: %s", errPayload.Errors[0].Title) + return "", "", []*courier.ChannelLog{log}, err + } + if !hasWhatsAppContactError(*errPayload) { err := errors.Errorf("received error from send endpoint: %s", errPayload.Errors[0].Title) return "", "", []*courier.ChannelLog{log}, err @@ -1022,6 +1034,15 @@ func buildWhatsAppHeaders(channel courier.Channel) http.Header { return header } +func hasTiersError(payload mtErrorPayload) bool { + for _, err := range payload.Errors { + if err.Code == 471 && err.Title == "Spam rate limit hit" { + return true + } + } + return false +} + func hasWhatsAppContactError(payload mtErrorPayload) bool { for _, err := range payload.Errors { if err.Code == 1006 && err.Title == "Resource not found" && err.Details == "unknown contact" { From 7a9d220f9dd4e3434b54af68d83f6c146edfdf6e Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 3 Feb 2022 19:41:34 +0200 Subject: [PATCH 066/146] Use UPPPERCASE for redis commands --- handlers/whatsapp/whatsapp.go | 8 ++++---- queue/queue_test.go | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index b766b929b..b9cfc8544 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -901,12 +901,12 @@ func sendWhatsAppMsg(rc redis.Conn, msg courier.Msg, sendPath *url.URL, payload if rr.StatusCode == 429 || rr.StatusCode == 503 { rateLimitKey := fmt.Sprintf("rate_limit:%s", msg.Channel().UUID().String()) - rc.Do("set", rateLimitKey, "engaged") + rc.Do("SET", rateLimitKey, "engaged") // The rate limit is 50 requests per second // We pause sending 2 seconds so the limit count is reset // TODO: In the future we should the header value when available - rc.Do("expire", rateLimitKey, 2) + rc.Do("EXPIRE", rateLimitKey, 2) log := courier.NewChannelLogFromRR("rate limit engaged", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) return "", "", []*courier.ChannelLog{log}, err @@ -920,11 +920,11 @@ func sendWhatsAppMsg(rc redis.Conn, msg courier.Msg, sendPath *url.URL, payload if err == nil && len(errPayload.Errors) > 0 { if hasTiersError(*errPayload) { pausedBulkKey := fmt.Sprintf("paused_bulk:%s", msg.Channel().UUID().String()) - rc.Do("set", pausedBulkKey, "engaged") + rc.Do("SET", pausedBulkKey, "engaged") // The WA tiers spam rate limit hit // We pause the bulk queue for 24 hours and 5min - rc.Do("expire", pausedBulkKey, (60*60*24)+(5*60)) + rc.Do("EXPIRE", pausedBulkKey, (60*60*24)+(5*60)) err := errors.Errorf("received error from send endpoint: %s", errPayload.Errors[0].Title) return "", "", []*courier.ChannelLog{log}, err diff --git a/queue/queue_test.go b/queue/queue_test.go index 8e6641382..b7906ada7 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -60,7 +60,7 @@ func TestLua(t *testing.T) { delay := time.Second*2 - time.Duration(time.Now().UnixNano()%int64(time.Second)) time.Sleep(delay) - conn.Do("set", "paused_bulk:chan1", "engaged") + conn.Do("SET", "paused_bulk:chan1", "engaged") conn.Do("EXPIRE", "paused_bulk:chan1", 5) // we have the rate limit set, @@ -120,7 +120,7 @@ func TestLua(t *testing.T) { assert.NoError(err) // make sure pause bulk key do not prevent use to get from the high priority queue - conn.Do("set", "paused_bulk:chan1", "engaged") + conn.Do("SET", "paused_bulk:chan1", "engaged") conn.Do("EXPIRE", "paused_bulk:chan1", 5) queue, value, err = PopFromQueue(conn, "msgs") @@ -194,7 +194,7 @@ func TestLua(t *testing.T) { err = PushOntoQueue(conn, "msgs", "chan1", rate, `[{"id":34}]`, HighPriority) assert.NoError(err) - conn.Do("set", "rate_limit:chan1", "engaged") + conn.Do("SET", "rate_limit:chan1", "engaged") conn.Do("EXPIRE", "rate_limit:chan1", 5) // we have the rate limit set, From d9bc7ac422a3a7e8d93cbec6928a3751e5342849 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 3 Feb 2022 19:44:25 +0200 Subject: [PATCH 067/146] Rely only on the payload error code --- handlers/whatsapp/whatsapp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index b9cfc8544..19778cab5 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -1036,7 +1036,7 @@ func buildWhatsAppHeaders(channel courier.Channel) http.Header { func hasTiersError(payload mtErrorPayload) bool { for _, err := range payload.Errors { - if err.Code == 471 && err.Title == "Spam rate limit hit" { + if err.Code == 471 { return true } } From 6ea5e12c8123b251584a9004688ead6ce7095713 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 3 Feb 2022 19:53:46 +0200 Subject: [PATCH 068/146] Use rate_limit_bulk as key --- handlers/whatsapp/whatsapp.go | 6 +++--- queue/queue.go | 8 ++++---- queue/queue_test.go | 12 ++++++------ 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 19778cab5..6b9ae0f3a 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -919,12 +919,12 @@ func sendWhatsAppMsg(rc redis.Conn, msg courier.Msg, sendPath *url.URL, payload // handle send msg errors if err == nil && len(errPayload.Errors) > 0 { if hasTiersError(*errPayload) { - pausedBulkKey := fmt.Sprintf("paused_bulk:%s", msg.Channel().UUID().String()) - rc.Do("SET", pausedBulkKey, "engaged") + rateLimitBulkKey := fmt.Sprintf("rate_limit_bulk:%s", msg.Channel().UUID().String()) + rc.Do("SET", rateLimitBulkKey, "engaged") // The WA tiers spam rate limit hit // We pause the bulk queue for 24 hours and 5min - rc.Do("EXPIRE", pausedBulkKey, (60*60*24)+(5*60)) + rc.Do("EXPIRE", rateLimitBulkKey, (60*60*24)+(5*60)) err := errors.Errorf("received error from send endpoint: %s", errPayload.Errors[0].Title) return "", "", []*courier.ChannelLog{log}, err diff --git a/queue/queue.go b/queue/queue.go index 1f62f7d74..02fc25930 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -123,10 +123,10 @@ var luaPop = redis.NewScript(2, `-- KEYS: [EpochMS QueueType] -- if we didn't find one, try again from our bulk queue if not result[1] or isFutureResult then - -- check if we are paused for bulk queue - local pausedBulkKey = "paused_bulk:" .. queueName - local pausedBulk = redis.call("get", pausedBulkKey) - if pausedBulk then + -- check if we are rate limited for bulk queue + local rateLimitBulkKey = "rate_limit_bulk:" .. queueName + local rateLimitBulk = redis.call("get", rateLimitBulkKey) + if rateLimitBulk then return {"retry", ""} end diff --git a/queue/queue_test.go b/queue/queue_test.go index b7906ada7..8a88fef40 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -60,8 +60,8 @@ func TestLua(t *testing.T) { delay := time.Second*2 - time.Duration(time.Now().UnixNano()%int64(time.Second)) time.Sleep(delay) - conn.Do("SET", "paused_bulk:chan1", "engaged") - conn.Do("EXPIRE", "paused_bulk:chan1", 5) + conn.Do("SET", "rate_limit_bulk:chan1", "engaged") + conn.Do("EXPIRE", "rate_limit_bulk:chan1", 5) // we have the rate limit set, queue, value, err := PopFromQueue(conn, "msgs") @@ -71,7 +71,7 @@ func TestLua(t *testing.T) { } // When the redis paused key is remove, we get the values from bulk queue/low priority - conn.Do("DEL", "paused_bulk:chan1") + conn.Do("DEL", "rate_limit_bulk:chan1") // pop 10 items off for i := 0; i < 10; i++ { @@ -120,8 +120,8 @@ func TestLua(t *testing.T) { assert.NoError(err) // make sure pause bulk key do not prevent use to get from the high priority queue - conn.Do("SET", "paused_bulk:chan1", "engaged") - conn.Do("EXPIRE", "paused_bulk:chan1", 5) + conn.Do("SET", "rate_limit_bulk:chan1", "engaged") + conn.Do("EXPIRE", "rate_limit_bulk:chan1", 5) queue, value, err = PopFromQueue(conn, "msgs") assert.NoError(err) @@ -129,7 +129,7 @@ func TestLua(t *testing.T) { assert.Equal(`{"id":31}`, value) // make sure paused is not present for more tests - conn.Do("DEL", "paused_bulk:chan1") + conn.Do("DEL", "rate_limit_bulk:chan1") // should get next five bulk msgs fine for i := 10; i < 15; i++ { From 638d6021162fff80cd2fa890ad0f19000e0524b1 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 3 Feb 2022 13:17:48 -0500 Subject: [PATCH 069/146] Update to latest gocommon --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index bb6f37861..0a157e171 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect - github.com/aws/aws-sdk-go v1.40.56 + github.com/aws/aws-sdk-go v1.40.56 // indirect github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect github.com/dghubble/oauth1 v0.4.0 @@ -23,7 +23,7 @@ require ( github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.17.0 + github.com/nyaruka/gocommon v1.17.1 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/nyaruka/redisx v0.2.1 diff --git a/go.sum b/go.sum index 5db31a84a..0d9c2bbb1 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4 github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= github.com/nyaruka/gocommon v1.5.3/go.mod h1:2ZeBZF9yt20IaAJ4aC1ujojAsFhJBk2IuDvSl7KuQDw= -github.com/nyaruka/gocommon v1.17.0 h1:cTiDLSUgmYJ9OZw752jva0P2rz0utRtv5WGuKFc9kxw= -github.com/nyaruka/gocommon v1.17.0/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0= +github.com/nyaruka/gocommon v1.17.1 h1:4bbNp+0/BIbne4VDiKOxh3kcbdvEu/WsrsZiG/VyRZ8= +github.com/nyaruka/gocommon v1.17.1/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= From 6a519a6d3b72327bf1ebbe4e6b20a92ec59afb6a Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 3 Feb 2022 13:18:28 -0500 Subject: [PATCH 070/146] Update CHANGELOG.md for v7.1.16 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4790c3fe5..eb940cb16 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.1.16 +---------- + * Update to latest gocommon + * Pause WA channel bulk queue when we hit the spam rate limit + v7.1.15 ---------- * Fix Gujarati whatsapp language code From bad4f0f833a368c938a48f9bf3446adc98d797a9 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 3 Feb 2022 20:23:52 +0200 Subject: [PATCH 071/146] Use UPPERCASE for redis commands --- backends/rapidpro/backend.go | 12 ++++++------ backends/rapidpro/task.go | 2 +- celery/celery_test.go | 2 +- cmd/fuzzer/main.go | 4 ++-- handlers/jiochat/jiochat.go | 4 ++-- handlers/jiochat/jiochat_test.go | 6 +++--- handlers/wechat/wechat.go | 4 ++-- handlers/wechat/wechat_test.go | 6 +++--- queue/queue_test.go | 12 ++++++------ 9 files changed, 26 insertions(+), 26 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index a36e87888..b62c07ef1 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -409,11 +409,11 @@ func (b *backend) Heartbeat() error { rc := b.redisPool.Get() defer rc.Close() - active, err := redis.Strings(rc.Do("zrange", fmt.Sprintf("%s:active", msgQueueName), "0", "-1")) + active, err := redis.Strings(rc.Do("ZRANGE", fmt.Sprintf("%s:active", msgQueueName), "0", "-1")) if err != nil { return errors.Wrapf(err, "error getting active queues") } - throttled, err := redis.Strings(rc.Do("zrange", fmt.Sprintf("%s:throttled", msgQueueName), "0", "-1")) + throttled, err := redis.Strings(rc.Do("ZRANGE", fmt.Sprintf("%s:throttled", msgQueueName), "0", "-1")) if err != nil { return errors.Wrapf(err, "error getting throttled queues") } @@ -423,14 +423,14 @@ func (b *backend) Heartbeat() error { bulkSize := 0 for _, queue := range queues { q := fmt.Sprintf("%s/1", queue) - count, err := redis.Int(rc.Do("zcard", q)) + count, err := redis.Int(rc.Do("ZCARD", q)) if err != nil { return errors.Wrapf(err, "error getting size of priority queue: %s", q) } prioritySize += count q = fmt.Sprintf("%s/0", queue) - count, err = redis.Int(rc.Do("zcard", q)) + count, err = redis.Int(rc.Do("ZCARD", q)) if err != nil { return errors.Wrapf(err, "error getting size of bulk queue: %s", q) } @@ -526,13 +526,13 @@ func (b *backend) Status() string { } // get # of items in our normal queue - size, err := redis.Int64(rc.Do("zcard", fmt.Sprintf("%s:%s/1", msgQueueName, queue))) + size, err := redis.Int64(rc.Do("ZCARD", fmt.Sprintf("%s:%s/1", msgQueueName, queue))) if err != nil { return fmt.Sprintf("error reading queue size: %v", err) } // get # of items in the bulk queue - bulkSize, err := redis.Int64(rc.Do("zcard", fmt.Sprintf("%s:%s/0", msgQueueName, queue))) + bulkSize, err := redis.Int64(rc.Do("ZCARD", fmt.Sprintf("%s:%s/0", msgQueueName, queue))) if err != nil { return fmt.Sprintf("error reading bulk queue size: %v", err) } diff --git a/backends/rapidpro/task.go b/backends/rapidpro/task.go index d96d60e7d..b13808cbe 100644 --- a/backends/rapidpro/task.go +++ b/backends/rapidpro/task.go @@ -122,7 +122,7 @@ func queueMailroomTask(rc redis.Conn, taskType string, orgID OrgID, contactID Co rc.Send("rpush", contactQueue, eventJSON) rc.Send("zadd", fmt.Sprintf("handler:%d", orgID), fmt.Sprintf("%.5f", epochFloat-10000000), contactJSON) rc.Send("zincrby", "handler:active", 0, orgID) - _, err = rc.Do("exec") + _, err = rc.Do("EXEC") return err } diff --git a/celery/celery_test.go b/celery/celery_test.go index b8baf60f7..f44acad69 100644 --- a/celery/celery_test.go +++ b/celery/celery_test.go @@ -47,7 +47,7 @@ func TestQueue(t *testing.T) { if err != nil { t.Error(err) } - _, err = conn.Do("exec") + _, err = conn.Do("EXEC") if err != nil { t.Error(err) } diff --git a/cmd/fuzzer/main.go b/cmd/fuzzer/main.go index 21e9376f4..ac99250ad 100644 --- a/cmd/fuzzer/main.go +++ b/cmd/fuzzer/main.go @@ -53,11 +53,11 @@ func main() { // insert our messages for i := 0; i < 1000; i++ { json := fmt.Sprintf(msgJSON, i) - _, err := conn.Do("zadd", "msgs:"+channelUUID, 0.0, json) + _, err := conn.Do("ZADD", "msgs:"+channelUUID, 0.0, json) if err != nil { log.Fatalf("err inserting msg: %s", err) } - _, err = conn.Do("zincrby", "msgs:active", 0.0, channelUUID) + _, err = conn.Do("ZINCRBY", "msgs:active", 0.0, channelUUID) if err != nil { log.Fatalf("err incrementing active: %s", err) } diff --git a/handlers/jiochat/jiochat.go b/handlers/jiochat/jiochat.go index 9673095e4..ac49ec7a8 100644 --- a/handlers/jiochat/jiochat.go +++ b/handlers/jiochat/jiochat.go @@ -208,7 +208,7 @@ func (h *handler) fetchAccessToken(ctx context.Context, channel courier.Channel) defer rc.Close() cacheKey := fmt.Sprintf("jiochat_channel_access_token:%s", channel.UUID().String()) - _, err = rc.Do("set", cacheKey, accessToken, 7200) + _, err = rc.Do("SET", cacheKey, accessToken, 7200) if err != nil { logrus.WithError(err).Error("error setting the access token to redis") @@ -221,7 +221,7 @@ func (h *handler) getAccessToken(channel courier.Channel) (string, error) { defer rc.Close() cacheKey := fmt.Sprintf("jiochat_channel_access_token:%s", channel.UUID().String()) - accessToken, err := redis.String(rc.Do(http.MethodGet, cacheKey)) + accessToken, err := redis.String(rc.Do("GET", cacheKey)) if err != nil { return "", err } diff --git a/handlers/jiochat/jiochat_test.go b/handlers/jiochat/jiochat_test.go index 3b69ff069..6f67cfa6f 100644 --- a/handlers/jiochat/jiochat_test.go +++ b/handlers/jiochat/jiochat_test.go @@ -261,7 +261,7 @@ func TestDescribe(t *testing.T) { mb := courier.NewMockBackend() conn := mb.RedisPool().Get() - _, err := conn.Do("Set", "jiochat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") + _, err := conn.Do("SET", "jiochat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") if err != nil { log.Fatal(err) } @@ -290,7 +290,7 @@ func TestBuildMediaRequest(t *testing.T) { mb := courier.NewMockBackend() conn := mb.RedisPool().Get() - _, err := conn.Do("Set", "jiochat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") + _, err := conn.Do("SET", "jiochat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") if err != nil { log.Fatal(err) } @@ -376,7 +376,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ func setupBackend(mb *courier.MockBackend) { conn := mb.RedisPool().Get() - _, err := conn.Do("Set", "jiochat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") + _, err := conn.Do("SET", "jiochat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") if err != nil { log.Fatal(err) } diff --git a/handlers/wechat/wechat.go b/handlers/wechat/wechat.go index bc9a3a0e6..97f185477 100644 --- a/handlers/wechat/wechat.go +++ b/handlers/wechat/wechat.go @@ -134,7 +134,7 @@ func (h *handler) fetchAccessToken(ctx context.Context, channel courier.Channel) defer rc.Close() cacheKey := fmt.Sprintf("wechat_channel_access_token:%s", channel.UUID().String()) - _, err = rc.Do("set", cacheKey, accessToken, expiration) + _, err = rc.Do("SET", cacheKey, accessToken, expiration) if err != nil { logrus.WithError(err).Error("error setting the access token to redis") @@ -147,7 +147,7 @@ func (h *handler) getAccessToken(channel courier.Channel) (string, error) { defer rc.Close() cacheKey := fmt.Sprintf("wechat_channel_access_token:%s", channel.UUID().String()) - accessToken, err := redis.String(rc.Do(http.MethodGet, cacheKey)) + accessToken, err := redis.String(rc.Do("GET", cacheKey)) if err != nil { return "", err } diff --git a/handlers/wechat/wechat_test.go b/handlers/wechat/wechat_test.go index e2865e4b3..bb27d9604 100644 --- a/handlers/wechat/wechat_test.go +++ b/handlers/wechat/wechat_test.go @@ -257,7 +257,7 @@ func TestDescribe(t *testing.T) { mb := courier.NewMockBackend() conn := mb.RedisPool().Get() - _, err := conn.Do("Set", "wechat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") + _, err := conn.Do("SET", "wechat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") if err != nil { log.Fatal(err) } @@ -286,7 +286,7 @@ func TestBuildMediaRequest(t *testing.T) { mb := courier.NewMockBackend() conn := mb.RedisPool().Get() - _, err := conn.Do("Set", "wechat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") + _, err := conn.Do("SET", "wechat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") if err != nil { log.Fatal(err) } @@ -366,7 +366,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ func setupBackend(mb *courier.MockBackend) { conn := mb.RedisPool().Get() - _, err := conn.Do("Set", "wechat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") + _, err := conn.Do("SET", "wechat_channel_access_token:8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "ACCESS_TOKEN") if err != nil { log.Fatal(err) } diff --git a/queue/queue_test.go b/queue/queue_test.go index 8a88fef40..7a0a69a61 100644 --- a/queue/queue_test.go +++ b/queue/queue_test.go @@ -89,11 +89,11 @@ func TestLua(t *testing.T) { } // check our redis state - count, err := redis.Int(conn.Do("zcard", "msgs:throttled")) + count, err := redis.Int(conn.Do("ZCARD", "msgs:throttled")) assert.NoError(err) assert.Equal(1, count, "Expected chan1 to be throttled") - count, err = redis.Int(conn.Do("zcard", "msgs:active")) + count, err = redis.Int(conn.Do("ZCARD", "msgs:active")) assert.NoError(err) assert.Equal(0, count, "Expected chan1 to not be active") @@ -106,11 +106,11 @@ func TestLua(t *testing.T) { err = PushOntoQueue(conn, "msgs", "chan1", rate, `[{"id":30}]`, LowPriority) assert.NoError(err) - count, err = redis.Int(conn.Do("zcard", "msgs:throttled")) + count, err = redis.Int(conn.Do("ZCARD", "msgs:throttled")) assert.NoError(err) assert.Equal(1, count, "Expected chan1 to be throttled") - count, err = redis.Int(conn.Do("zcard", "msgs:active")) + count, err = redis.Int(conn.Do("ZCARD", "msgs:active")) assert.NoError(err) assert.Equal(0, count, "Expected chan1 to not be active") @@ -211,11 +211,11 @@ func TestLua(t *testing.T) { t.Fatal("Should be throttled") } - count, err = redis.Int(conn.Do("zcard", "msgs:throttled")) + count, err = redis.Int(conn.Do("ZCARD", "msgs:throttled")) assert.NoError(err) assert.Equal(1, count, "Expected chan1 to be throttled") - count, err = redis.Int(conn.Do("zcard", "msgs:active")) + count, err = redis.Int(conn.Do("ZCARD", "msgs:active")) assert.NoError(err) assert.Equal(0, count, "Expected chan1 to not be active") From b15e40edb4c99e86acbd5f8ccae9e70977e76d64 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 4 Feb 2022 13:03:01 +0200 Subject: [PATCH 072/146] Support for deleting messages when we get an event for the message being deleted on IG --- backend.go | 3 +++ backends/rapidpro/backend.go | 20 ++++++++++++++++++ handlers/facebookapp/facebookapp.go | 7 ++++++ handlers/facebookapp/facebookapp_test.go | 27 ++++++++++++++++++++++++ test.go | 5 +++++ 5 files changed, 62 insertions(+) diff --git a/backend.go b/backend.go index 9516a0cdc..9a740e7ec 100644 --- a/backend.go +++ b/backend.go @@ -38,6 +38,9 @@ type Backend interface { // RemoveURNFromcontact removes a URN from the passed in contact RemoveURNfromContact(context context.Context, channel Channel, contact Contact, urn urns.URN) (urns.URN, error) + // DeleteMsgWithExternalID delete a message we receive an event that it should be deleted + DeleteMsgWithExternalID(ctx context.Context, channel Channel, externalID string) error + // NewIncomingMsg creates a new message from the given params NewIncomingMsg(channel Channel, urn urns.URN, text string) Msg diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index a36e87888..ecfd115b5 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -101,6 +101,26 @@ func (b *backend) RemoveURNfromContact(ctx context.Context, c courier.Channel, c return urn, nil } +const updateMsgVisibilityDeleted = ` +UPDATE + msgs_msg +SET + visibility = 'D' +WHERE + msgs_msg.id = (SELECT m."id" FROM "msgs_msg" m INNER JOIN "channels_channel" c ON (m."channel_id" = c."id") WHERE (c."uuid" = $1 AND m."external_id" = $2 AND m."direction" = 'I') +RETURNING + msgs_msg.id +` + +// DeleteMsgWithExternalID delete a message we receive an event that it should be deleted +func (b *backend) DeleteMsgWithExternalID(ctx context.Context, channel courier.Channel, externalID string) error { + _, err := b.db.ExecContext(ctx, updateMsgVisibilityDeleted, channel.UUID().String(), externalID) + if err != nil { + return err + } + return nil +} + // NewIncomingMsg creates a new message from the given params func (b *backend) NewIncomingMsg(channel courier.Channel, urn urns.URN, text string) courier.Msg { // remove any control characters diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 54fd3099e..0b8a0f733 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -142,6 +142,7 @@ type moPayload struct { IsEcho bool `json:"is_echo"` MID string `json:"mid"` Text string `json:"text"` + IsDeleted string `json:"is_deleted"` Attachments []struct { Type string `json:"type"` Payload *struct { @@ -384,6 +385,12 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h continue } + if msg.Message.IsDeleted == "true" { + h.Backend().DeleteMsgWithExternalID(ctx, channel, msg.Message.MID) + data = append(data, courier.NewInfoData("msg deleted")) + continue + } + text := msg.Message.Text attachmentURLs := make([]string, 0, 2) diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 625664971..4557b76a2 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -573,6 +573,32 @@ var dlr = `{ }] }` +var unsentMsgIG = `{ + "object":"instagram", + "entry":[ + { + "id":"12345", + "time":1459991487970, + "messaging":[ + { + "sender":{ + "id":"5678" + }, + "recipient":{ + "id":"12345" + }, + "timestamp":1569262485349, + "message":{ + "mid":"external_id", + "is_deleted":"true" + } + } + ] + } + ] + } +` + var notPage = `{ "object":"notpage", "entry": [{}] @@ -760,6 +786,7 @@ var testCasesIG = []ChannelHandleTestCase{ {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, {Label: "Story Mention", URL: "/c/ig/receive", Data: storyMentionIG, Status: 200, Response: `ignoring story_mention`, PrepRequest: addValidSignature}, + {Label: "Message unsent", URL: "/c/ig/receive", Data: unsentMsgIG, Status: 200, Response: `msg deleted`, PrepRequest: addValidSignature}, } func addValidSignature(r *http.Request) { diff --git a/test.go b/test.go index 69bcef185..69a634862 100644 --- a/test.go +++ b/test.go @@ -113,6 +113,11 @@ func (mb *MockBackend) GetLastContactName() string { return mb.lastContactName } +// DeleteMsgWithExternalID delete a message we receive an event that it should be deleted +func (mb *MockBackend) DeleteMsgWithExternalID(ctx context.Context, channel Channel, externalID string) error { + return nil +} + // NewIncomingMsg creates a new message from the given params func (mb *MockBackend) NewIncomingMsg(channel Channel, urn urns.URN, text string) Msg { return &mockMsg{channel: channel, urn: urn, text: text} From 9d1d72cabf726d1aa185eeecd28964d93a3f055e Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 4 Feb 2022 13:06:09 +0200 Subject: [PATCH 073/146] Do not save any message when receiving IG story mentions --- handlers/facebookapp/facebookapp.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 0b8a0f733..3af75ce7e 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -391,6 +391,8 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h continue } + has_story_mentions := false + text := msg.Message.Text attachmentURLs := make([]string, 0, 2) @@ -407,6 +409,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h if att.Type == "story_mention" { data = append(data, courier.NewInfoData("ignoring story_mention")) + has_story_mentions = true continue } @@ -416,6 +419,11 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h } + // if we have a story mention, skip and do not save any message + if has_story_mentions { + continue + } + // create our message ev := h.Backend().NewIncomingMsg(channel, urn, text).WithExternalID(msg.Message.MID).WithReceivedOn(date) event := h.Backend().CheckExternalIDSeen(ev) From 256679504f5c1db92d21066f6a887f2f0d731c74 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Mon, 7 Feb 2022 14:36:46 -0300 Subject: [PATCH 074/146] Add kyrgyzstan language code wpp --- handlers/whatsapp/whatsapp.go | 1 + 1 file changed, 1 insertion(+) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 542ddbc45..558bffabd 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -1164,6 +1164,7 @@ var languageMap = map[string]string{ "kan": "kn", // Kannada "kaz": "kk", // Kazakh "kor": "ko", // Korean + "kir": "ky_KG", // Kyrgyzstan "lao": "lo", // Lao "lav": "lv", // Latvian "lit": "lt", // Lithuanian From f2dcb840121be8d2e52fe102542ddf8494bf15a8 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 8 Feb 2022 14:44:10 -0300 Subject: [PATCH 075/146] Add missing languages from whatsapp template --- handlers/whatsapp/whatsapp.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 0dab712fc..bb9b801b1 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -1171,6 +1171,7 @@ var languageMap = map[string]string{ "fil": "fil", // Filipino "fin": "fi", // Finnish "fra": "fr", // French + "kat": "ka", // Georgian "deu": "de", // German "ell": "el", // Greek "guj": "gu", // Gujarati @@ -1184,6 +1185,7 @@ var languageMap = map[string]string{ "jpn": "ja", // Japanese "kan": "kn", // Kannada "kaz": "kk", // Kazakh + "kin": "rw_RW", // Kinyarwanda "kor": "ko", // Korean "kir": "ky_KG", // Kyrgyzstan "lao": "lo", // Lao From 7d54d0fd46afb465009ad5f247ca073dd4153094 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 11 Feb 2022 16:32:16 +0200 Subject: [PATCH 076/146] Use deleted by sender visibity for message unsent on IG channels --- backends/rapidpro/backend.go | 10 ++++++---- backends/rapidpro/backend_test.go | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 50bde2c33..654e84e39 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -101,20 +101,22 @@ func (b *backend) RemoveURNfromContact(ctx context.Context, c courier.Channel, c return urn, nil } -const updateMsgVisibilityDeleted = ` +const updateMsgVisibilityDeletedBySender = ` UPDATE msgs_msg SET - visibility = 'D' + visibility = 'X', + text = '', + attachments = '{}' WHERE - msgs_msg.id = (SELECT m."id" FROM "msgs_msg" m INNER JOIN "channels_channel" c ON (m."channel_id" = c."id") WHERE (c."uuid" = $1 AND m."external_id" = $2 AND m."direction" = 'I') + msgs_msg.id = (SELECT m."id" FROM "msgs_msg" m INNER JOIN "channels_channel" c ON (m."channel_id" = c."id") WHERE (c."uuid" = $1 AND m."external_id" = $2 AND m."direction" = 'I')) RETURNING msgs_msg.id ` // DeleteMsgWithExternalID delete a message we receive an event that it should be deleted func (b *backend) DeleteMsgWithExternalID(ctx context.Context, channel courier.Channel, externalID string) error { - _, err := b.db.ExecContext(ctx, updateMsgVisibilityDeleted, channel.UUID().String(), externalID) + _, err := b.db.ExecContext(ctx, updateMsgVisibilityDeletedBySender, string(channel.UUID().String()), externalID) if err != nil { return err } diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index 993080438..c84f5c8b1 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -212,6 +212,35 @@ func (ts *BackendTestSuite) TestCheckMsgExists() { ts.Nil(err) } +func (ts *BackendTestSuite) TestDeleteMsgWithExternalID() { + knChannel := ts.getChannel("KN", "dbc126ed-66bc-4e28-b67b-81dc3327c95d") + + ctx := context.Background() + + // no error for invalid external ID + err := ts.b.DeleteMsgWithExternalID(ctx, knChannel, "ext-invalid") + ts.Nil(err) + + // cannot change out going messages + err = ts.b.DeleteMsgWithExternalID(ctx, knChannel, "ext1") + ts.Nil(err) + + m := readMsgFromDB(ts.b, courier.NewMsgID(10000)) + ts.Equal(m.Text_, "test message") + ts.Equal(len(m.Attachments()), 0) + ts.Equal(m.Visibility_, MsgVisibility("V")) + + // for incoming messages mark them deleted by sender and readact their text and clear their attachments + err = ts.b.DeleteMsgWithExternalID(ctx, knChannel, "ext2") + ts.Nil(err) + + m = readMsgFromDB(ts.b, courier.NewMsgID(10002)) + ts.Equal(m.Text_, "") + ts.Equal(len(m.Attachments()), 0) + ts.Equal(m.Visibility_, MsgVisibility("X")) + +} + func (ts *BackendTestSuite) TestContact() { knChannel := ts.getChannel("KN", "dbc126ed-66bc-4e28-b67b-81dc3327c95d") urn, _ := urns.NewTelURNForCountry("12065551518", "US") From e870ad8cbe8f118978af27bfeae827af58401edd Mon Sep 17 00:00:00 2001 From: Rafael Soares Date: Tue, 15 Feb 2022 12:42:36 -0300 Subject: [PATCH 077/146] fix whatsapp uploaded attachment file name --- handlers/whatsapp/whatsapp.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index bb9b801b1..8c16bbaa8 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -586,6 +586,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann if err != nil { logrus.WithField("channel_uuid", msg.Channel().UUID().String()).WithError(err).Error("error while uploading media to whatsapp") } + fileURL := mediaURL if err == nil && mediaID != "" { mediaURL = "" } @@ -605,7 +606,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() } - mediaPayload.Filename, err = utils.BasePathForURL(mediaURL) + mediaPayload.Filename, err = utils.BasePathForURL(fileURL) // Logging error if err != nil { From 50a68d619d30931cba6d8fdefac39ca3b375de4b Mon Sep 17 00:00:00 2001 From: Rafael Soares Date: Wed, 16 Feb 2022 18:43:28 -0300 Subject: [PATCH 078/146] test whatsapp upload and cache document send cases --- handlers/whatsapp/whatsapp_test.go | 43 ++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index edb54c7bb..70c2dc684 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -738,6 +738,49 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ }, SendPrep: setSendURL, }, + { + Label: "Document Upload OK", + Text: "document caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"application/pdf:https://foo.bar/document2.pdf"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/media", + Body: "media bytes", + }: MockedResponse{ + Status: 200, + Body: `{ "media" : [{"id": "25c484d1-1283-4b94-988d-7276bdec4ef3"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"document","document":{"id":"25c484d1-1283-4b94-988d-7276bdec4ef3","caption":"document caption","filename":"document2.pdf"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL, + }, + {Label: "Cached Document", + Text: "document caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"application/pdf:https://foo.bar/document2.pdf"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"document","document":{"id":"25c484d1-1283-4b94-988d-7276bdec4ef3","caption":"document caption","filename":"document2.pdf"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL, + }, } var hsmSupportSendTestCases = []ChannelSendTestCase{ From 8b26f615647aba04b8902a26271b89afac76e9db Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 17 Feb 2022 11:08:43 -0500 Subject: [PATCH 079/146] Update CHANGELOG.md for v7.1.17 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index eb940cb16..0a25a864e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +v7.1.17 +---------- + * Fix whatsapp uploaded attachment file name + * Use deleted by sender visibity for message unsent on IG channels + * Add missing languages from whatsapp template + * Do not save any message when receiving IG story mentions + v7.1.16 ---------- * Update to latest gocommon From 812a1b64338b273e96be40e71f0eead025df57ee Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 18 Feb 2022 10:50:29 -0300 Subject: [PATCH 080/146] Fix metadata fetching for new Facebook contacts --- handlers/facebookapp/facebookapp.go | 36 ++++++++++++++++-------- handlers/facebookapp/facebookapp_test.go | 35 +++++++++++++++++++---- 2 files changed, 55 insertions(+), 16 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 54fd3099e..1f2ac8f20 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -662,20 +662,34 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn base, _ := url.Parse(graphURL) path, _ := url.Parse(fmt.Sprintf("/%s", urn.Path())) u := base.ResolveReference(path) - query := url.Values{} - query.Set("access_token", accessToken) - u.RawQuery = query.Encode() - req, _ := http.NewRequest(http.MethodGet, u.String(), nil) - rr, err := utils.MakeHTTPRequest(req) - if err != nil { - return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) - } - // read our name - name, _ := jsonparser.GetString(rr.Body, "name") + if fmt.Sprint(channel.ChannelType()) == "FBA" { + query.Set("fields", "first_name,last_name") + query.Set("access_token", accessToken) - return map[string]string{"name": name}, nil + u.RawQuery = query.Encode() + req, _ := http.NewRequest(http.MethodGet, u.String(), nil) + rr, err := utils.MakeHTTPRequest(req) + if err != nil { + return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) + } + // read our first and last name + firstName, _ := jsonparser.GetString(rr.Body, "first_name") + lastName, _ := jsonparser.GetString(rr.Body, "last_name") + return map[string]string{"name": utils.JoinNonEmpty(" ", firstName, lastName)}, nil + } else { + query.Set("access_token", accessToken) + u.RawQuery = query.Encode() + req, _ := http.NewRequest(http.MethodGet, u.String(), nil) + rr, err := utils.MakeHTTPRequest(req) + if err != nil { + return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) + } + // read our name + name, _ := jsonparser.GetString(rr.Body, "name") + return map[string]string{"name": name}, nil + } } // see https://developers.facebook.com/docs/messenger-platform/webhook#security diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 625664971..3163190b0 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -773,7 +773,31 @@ func addInvalidSignature(r *http.Request) { } // mocks the call to the Facebook graph API -func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { +func buildMockFBGraphFBA(testCases []ChannelHandleTestCase) *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + accessToken := r.URL.Query().Get("access_token") + defer r.Body.Close() + + // invalid auth token + if accessToken != "a123" { + http.Error(w, "invalid auth token", 403) + } + + // user has a name + if strings.HasSuffix(r.URL.Path, "1337") { + w.Write([]byte(`{ "first_name": "John", "last_name": "Doe"}`)) + return + } + // no name + w.Write([]byte(`{ "first_name": "", "last_name": ""}`)) + })) + graphURL = server.URL + + return server +} + +// mocks the call to the Facebook graph API +func buildMockFBGraphIG(testCases []ChannelHandleTestCase) *httptest.Server { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { accessToken := r.URL.Query().Get("access_token") defer r.Body.Close() @@ -797,8 +821,9 @@ func buildMockFBGraph(testCases []ChannelHandleTestCase) *httptest.Server { return server } + func TestDescribeFBA(t *testing.T) { - fbGraph := buildMockFBGraph(testCasesFBA) + fbGraph := buildMockFBGraphFBA(testCasesFBA) defer fbGraph.Close() handler := newHandler("FBA", "Facebook", false).(courier.URNDescriber) @@ -816,7 +841,7 @@ func TestDescribeFBA(t *testing.T) { } func TestDescribeIG(t *testing.T) { - fbGraph := buildMockFBGraph(testCasesIG) + fbGraph := buildMockFBGraphIG(testCasesIG) defer fbGraph.Close() handler := newHandler("IG", "Instagram", false).(courier.URNDescriber) @@ -839,12 +864,12 @@ func TestHandler(t *testing.T) { } func BenchmarkHandler(b *testing.B) { - fbService := buildMockFBGraph(testCasesFBA) + fbService := buildMockFBGraphFBA(testCasesFBA) RunChannelBenchmarks(b, testChannelsFBA, newHandler("FBA", "Facebook", false), testCasesFBA) fbService.Close() - fbServiceIG := buildMockFBGraph(testCasesIG) + fbServiceIG := buildMockFBGraphIG(testCasesIG) RunChannelBenchmarks(b, testChannelsIG, newHandler("IG", "Instagram", false), testCasesIG) fbServiceIG.Close() From aa3bff3a89aad8930358326c7679aea7ea4657c5 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 18 Feb 2022 12:17:01 -0300 Subject: [PATCH 081/146] Refactor the metadata search --- handlers/facebookapp/facebookapp.go | 38 ++++++++++++++--------------- 1 file changed, 18 insertions(+), 20 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 1f2ac8f20..940ac71af 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -663,33 +663,31 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn path, _ := url.Parse(fmt.Sprintf("/%s", urn.Path())) u := base.ResolveReference(path) query := url.Values{} + var name string if fmt.Sprint(channel.ChannelType()) == "FBA" { query.Set("fields", "first_name,last_name") - query.Set("access_token", accessToken) + } - u.RawQuery = query.Encode() - req, _ := http.NewRequest(http.MethodGet, u.String(), nil) - rr, err := utils.MakeHTTPRequest(req) - if err != nil { - return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) - } - // read our first and last name + query.Set("access_token", accessToken) + u.RawQuery = query.Encode() + req, _ := http.NewRequest(http.MethodGet, u.String(), nil) + rr, err := utils.MakeHTTPRequest(req) + if err != nil { + return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) + } + + // read our first and last name or complete name + if fmt.Sprint(channel.ChannelType()) == "FBA" { firstName, _ := jsonparser.GetString(rr.Body, "first_name") lastName, _ := jsonparser.GetString(rr.Body, "last_name") - return map[string]string{"name": utils.JoinNonEmpty(" ", firstName, lastName)}, nil - } else { - query.Set("access_token", accessToken) - u.RawQuery = query.Encode() - req, _ := http.NewRequest(http.MethodGet, u.String(), nil) - rr, err := utils.MakeHTTPRequest(req) - if err != nil { - return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) - } - // read our name - name, _ := jsonparser.GetString(rr.Body, "name") - return map[string]string{"name": name}, nil + name = utils.JoinNonEmpty(" ", firstName, lastName) + }else{ + name, _ = jsonparser.GetString(rr.Body, "name") } + + return map[string]string{"name": name}, nil + } // see https://developers.facebook.com/docs/messenger-platform/webhook#security From 6e08af505ecdea5ccb3fcc0a8f53c9d2ab9c7a20 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 23 Feb 2022 16:40:14 +0200 Subject: [PATCH 082/146] Fix type for IsDeleted field for events --- handlers/facebookapp/facebookapp.go | 12 ++++++------ handlers/facebookapp/facebookapp_test.go | 3 +-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 97a31f33b..b4c879a0b 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -142,7 +142,7 @@ type moPayload struct { IsEcho bool `json:"is_echo"` MID string `json:"mid"` Text string `json:"text"` - IsDeleted string `json:"is_deleted"` + IsDeleted bool `json:"is_deleted"` Attachments []struct { Type string `json:"type"` Payload *struct { @@ -385,7 +385,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h continue } - if msg.Message.IsDeleted == "true" { + if msg.Message.IsDeleted { h.Backend().DeleteMsgWithExternalID(ctx, channel, msg.Message.MID) data = append(data, courier.NewInfoData("msg deleted")) continue @@ -691,18 +691,18 @@ func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn if err != nil { return nil, fmt.Errorf("unable to look up contact data:%s\n%s", err, rr.Response) } - + // read our first and last name or complete name if fmt.Sprint(channel.ChannelType()) == "FBA" { firstName, _ := jsonparser.GetString(rr.Body, "first_name") lastName, _ := jsonparser.GetString(rr.Body, "last_name") name = utils.JoinNonEmpty(" ", firstName, lastName) - }else{ + } else { name, _ = jsonparser.GetString(rr.Body, "name") } - + return map[string]string{"name": name}, nil - + } // see https://developers.facebook.com/docs/messenger-platform/webhook#security diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 19d5f95fc..d814f04ef 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -590,7 +590,7 @@ var unsentMsgIG = `{ "timestamp":1569262485349, "message":{ "mid":"external_id", - "is_deleted":"true" + "is_deleted": true } } ] @@ -848,7 +848,6 @@ func buildMockFBGraphIG(testCases []ChannelHandleTestCase) *httptest.Server { return server } - func TestDescribeFBA(t *testing.T) { fbGraph := buildMockFBGraphFBA(testCasesFBA) defer fbGraph.Close() From f28d4be070964155ec15ca7747d46355dc45b725 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 23 Feb 2022 09:45:24 -0500 Subject: [PATCH 083/146] Update CHANGELOG.md for v7.1.18 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a25a864e..281cc8480 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.1.18 +---------- + * Fix type for IsDeleted field for IG unsend events + * Fix metadata fetching for new Facebook contacts + v7.1.17 ---------- * Fix whatsapp uploaded attachment file name From 89125d1c89cb94caa828edd59c019cd72c9b4fe0 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 25 Feb 2022 16:31:47 +0200 Subject: [PATCH 084/146] Update to support D3 contact check for whatspp contact not in contact store --- handlers/whatsapp/whatsapp.go | 2 +- handlers/whatsapp/whatsapp_test.go | 32 ++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 8c16bbaa8..bdd568f2c 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -1046,7 +1046,7 @@ func hasTiersError(payload mtErrorPayload) bool { func hasWhatsAppContactError(payload mtErrorPayload) bool { for _, err := range payload.Errors { - if err.Code == 1006 && err.Title == "Resource not found" && err.Details == "unknown contact" { + if err.Code == 1006 && err.Title == "Resource not found" && (err.Details == "unknown contact" || err.Details == "Could not retrieve phone number from contact store") { return true } } diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 70c2dc684..e125e0247 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -561,6 +561,38 @@ var defaultSendTestCases = []ChannelSendTestCase{ }, SendPrep: setSendURL, }, + {Label: "Try Messaging Again After WhatsApp Contact Check", + Text: "try again", URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"text","text":{"body":"try again"}}`, + }: MockedResponse{ + Status: 404, + Body: `{"errors": [{"code": 1006, "title": "Resource not found", "details": "Could not retrieve phone number from contact store"}]}`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/contacts", + Body: `{"blocking":"wait","contacts":["+250788123123"],"force_check":true}`, + }: MockedResponse{ + Status: 200, + Body: `{"contacts": [{"input": "+250788123123", "status": "valid", "wa_id": "250788123123"}]}`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + RawQuery: "retry=1", + Body: `{"to":"250788123123","type":"text","text":{"body":"try again"}}`, + }: MockedResponse{ + Status: 201, + Body: `{"messages": [{"id": "157b5e14568e8"}]}`, + }, + }, + SendPrep: setSendURL, + }, {Label: "Try Messaging Again After WhatsApp Contact Check With Returned WhatsApp ID", Text: "try again", URN: "whatsapp:5582999887766", Status: "W", ExternalID: "157b5e14568e8", From 29f3b410d3c3d986f4945eed5aa8f5d7cecfba80 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 25 Feb 2022 11:09:06 -0500 Subject: [PATCH 085/146] Update CHANGELOG.md for v7.1.19 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 281cc8480..30afb763a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.1.19 +---------- + * Update D3 handler to support check for whatsapp contact not in contact store + v7.1.18 ---------- * Fix type for IsDeleted field for IG unsend events From 1a46ce7a508c7eb22ed4abd70e48933f4e528dfd Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 7 Mar 2022 12:18:30 -0500 Subject: [PATCH 086/146] CI testing with go 1.17.7 --- .github/workflows/ci.yml | 100 +++++++++++++++++++-------------------- README.md | 1 + 2 files changed, 51 insertions(+), 50 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d43f9d7f3..1d713fc6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,74 +1,74 @@ name: CI on: [push, pull_request] env: - go-version: '1.17.2' # https://github.com/golang/go/issues/49366 - redis-version: '3.2.4' + go-version: "1.17.7" + redis-version: "3.2.4" jobs: test: name: Test strategy: matrix: - pg-version: ['12', '13'] + pg-version: ["12", "13"] runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v1 + - name: Checkout code + uses: actions/checkout@v1 - - name: Install Redis - uses: zhulik/redis-action@v1.0.0 - with: - redis version: ${{ env.redis-version }} + - name: Install Redis + uses: zhulik/redis-action@v1.0.0 + with: + redis version: ${{ env.redis-version }} - - name: Install PostgreSQL - uses: harmon758/postgresql-action@v1 - with: - postgresql version: ${{ matrix.pg-version }} - postgresql db: courier_test - postgresql user: courier - postgresql password: courier + - name: Install PostgreSQL + uses: harmon758/postgresql-action@v1 + with: + postgresql version: ${{ matrix.pg-version }} + postgresql db: courier_test + postgresql user: courier + postgresql password: courier - - name: Install Go - uses: actions/setup-go@v1 - with: - go-version: ${{ env.go-version }} + - name: Install Go + uses: actions/setup-go@v1 + with: + go-version: ${{ env.go-version }} - - name: Run tests - run: go test -p=1 -coverprofile=coverage.text -covermode=atomic ./... + - name: Run tests + run: go test -p=1 -coverprofile=coverage.text -covermode=atomic ./... + + - name: Upload coverage + if: success() + uses: codecov/codecov-action@v1 + with: + fail_ci_if_error: true - - name: Upload coverage - if: success() - uses: codecov/codecov-action@v1 - with: - fail_ci_if_error: true - release: name: Release needs: [test] if: startsWith(github.ref, 'refs/tags/') runs-on: ubuntu-latest steps: - - name: Checkout code - uses: actions/checkout@v1 + - name: Checkout code + uses: actions/checkout@v1 - - name: Install Go - uses: actions/setup-go@v1 - with: - go-version: ${{ env.go-version }} + - name: Install Go + uses: actions/setup-go@v1 + with: + go-version: ${{ env.go-version }} - - name: Publish release - uses: goreleaser/goreleaser-action@v1 - if: ${{ !contains(github.ref, '-') }} - with: - version: v0.147.2 - args: release --rm-dist - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Publish release + uses: goreleaser/goreleaser-action@v1 + if: ${{ !contains(github.ref, '-') }} + with: + version: v0.147.2 + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Publish non-master release - uses: goreleaser/goreleaser-action@v1 - if: contains(github.ref, '-') - with: - version: v0.147.2 - args: release --rm-dist --skip-validate - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Publish non-master release + uses: goreleaser/goreleaser-action@v1 + if: contains(github.ref, '-') + with: + version: v0.147.2 + args: release --rm-dist --skip-validate + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 0b1d7f6e2..0e8d402a8 100644 --- a/README.md +++ b/README.md @@ -26,6 +26,7 @@ behind a reverse proxy such as nginx or Elastic Load Balancer that provides HTTP # Configuration Courier uses a tiered configuration system, each option takes precendence over the ones above it: + 1. The configuration file 2. Environment variables starting with `COURIER_` 3. Command line parameters From 43157eadf922f40b880f69fc374f63b10f84f61d Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 7 Mar 2022 12:19:12 -0500 Subject: [PATCH 087/146] Update CHANGELOG.md for v7.2.0 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30afb763a..9072df787 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.2.0 +---------- + * CI testing with go 1.17.7 + v7.1.19 ---------- * Update D3 handler to support check for whatsapp contact not in contact store From 30fb76e493ce5de72faa0953c1236f4d3f63c726 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 17 Mar 2022 13:17:23 +0200 Subject: [PATCH 088/146] Support stopping contacts when we get stop events on status callbacks --- handlers/twiml/twiml.go | 13 +++++++++++++ handlers/twiml/twiml_test.go | 10 ++++++++++ 2 files changed, 23 insertions(+) diff --git a/handlers/twiml/twiml.go b/handlers/twiml/twiml.go index 220a1cac5..f6bd9eaac 100644 --- a/handlers/twiml/twiml.go +++ b/handlers/twiml/twiml.go @@ -87,6 +87,7 @@ type statusForm struct { MessageSID string `validate:"required"` MessageStatus string `validate:"required"` ErrorCode string + To string } var statusMapping = map[string]courier.MsgStatusValue{ @@ -195,6 +196,18 @@ func (h *handler) receiveStatus(ctx context.Context, channel courier.Channel, w if status == nil { status = h.Backend().NewMsgStatusForExternalID(channel, form.MessageSID, msgStatus) } + + errorCode, _ := strconv.ParseInt(form.ErrorCode, 10, 64) + if errorCode == errorStopped { + // create a stop channel event + channelEvent := h.Backend().NewChannelEvent(channel, courier.StopContact, urns.URN(form.To)) + err = h.Backend().WriteChannelEvent(ctx, channelEvent) + if err != nil { + return nil, err + } + + } + return handlers.WriteMsgStatusAndResponse(ctx, h, channel, status, w, r) } diff --git a/handlers/twiml/twiml_test.go b/handlers/twiml/twiml_test.go index 9dd223e64..9aa4f0656 100644 --- a/handlers/twiml/twiml_test.go +++ b/handlers/twiml/twiml_test.go @@ -60,6 +60,8 @@ var ( receiveMediaWithMsg = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=2&ToCity=&Body=Msg&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01&MediaUrl0=cat.jpg&MediaUrl1=dog.jpg" receiveBase64 = "ToCountry=US&ToState=District+Of+Columbia&SmsMessageSid=SMe287d7109a5a925f182f0e07fe5b223b&NumMedia=0&ToCity=&FromZip=01022&SmsSid=SMe287d7109a5a925f182f0e07fe5b223b&FromState=MA&SmsStatus=received&FromCity=CHICOPEE&Body=QmFubm9uIEV4cGxhaW5zIFRoZSBXb3JsZCAuLi4K4oCcVGhlIENhbXAgb2YgdGhlIFNhaW50c%2BKA&FromCountry=US&To=%2B12028831111&ToZip=&NumSegments=1&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&AccountSid=acctid&From=%2B14133881111&ApiVersion=2010-04-01" + statusStop = "ErrorCode=21610&MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&MessageStatus=failed&To=%2B12028831111" + statusInvalid = "MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&MessageStatus=huh" statusValid = "MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&MessageStatus=delivered" statusRead = "MessageSid=SMe287d7109a5a925f182f0e07fe5b223b&MessageStatus=read" @@ -93,6 +95,8 @@ var testCases = []ChannelHandleTestCase{ {Label: "Receive Base64", URL: receiveURL, Data: receiveBase64, Status: 200, Response: "", Text: Sp("Bannon Explains The World ...\n“The Camp of the Saints"), URN: Sp("tel:+14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), PrepRequest: addValidSignature}, + {Label: "Status Stop contact", URL: statusURL, Data: statusStop, Status: 200, Response: `"status":"F"`, + PrepRequest: addValidSignature}, {Label: "Status No Params", URL: statusURL, Data: " ", Status: 200, Response: "no msg status, ignoring", PrepRequest: addValidSignature}, {Label: "Status Invalid Status", URL: statusURL, Data: statusInvalid, Status: 400, Response: "unknown status 'huh'", @@ -128,6 +132,8 @@ var tmsTestCases = []ChannelHandleTestCase{ {Label: "Receive Base64", URL: tmsReceiveURL, Data: receiveBase64, Status: 200, Response: "", Text: Sp("Bannon Explains The World ...\n“The Camp of the Saints"), URN: Sp("tel:+14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), PrepRequest: addValidSignature}, + {Label: "Status Stop contact", URL: tmsStatusURL, Data: statusStop, Status: 200, Response: `"status":"F"`, + PrepRequest: addValidSignature}, {Label: "Status TMS extra", URL: tmsStatusURL, Data: tmsStatusExtra, Status: 200, Response: `"status":"S"`, ExternalID: Sp("SM0b6e2697aae04182a9f5b5c7a8994c7f"), PrepRequest: addValidSignature}, {Label: "Status No Params", URL: tmsStatusURL, Data: " ", Status: 200, Response: "no msg status, ignoring", @@ -165,6 +171,8 @@ var twTestCases = []ChannelHandleTestCase{ {Label: "Receive Base64", URL: twReceiveURL, Data: receiveBase64, Status: 200, Response: "", Text: Sp("Bannon Explains The World ...\n“The Camp of the Saints"), URN: Sp("tel:+14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), PrepRequest: addValidSignature}, + {Label: "Status Stop contact", URL: twStatusURL, Data: statusStop, Status: 200, Response: `"status":"F"`, + PrepRequest: addValidSignature}, {Label: "Status No Params", URL: twStatusURL, Data: " ", Status: 200, Response: "no msg status, ignoring", PrepRequest: addValidSignature}, {Label: "Status Invalid Status", URL: twStatusURL, Data: statusInvalid, Status: 400, Response: "unknown status 'huh'", @@ -187,6 +195,8 @@ var swTestCases = []ChannelHandleTestCase{ Text: Sp("Msg"), URN: Sp("tel:+14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b"), Attachments: []string{"cat.jpg", "dog.jpg"}}, {Label: "Receive Base64", URL: swReceiveURL, Data: receiveBase64, Status: 200, Response: "", Text: Sp("Bannon Explains The World ...\n“The Camp of the Saints"), URN: Sp("tel:+14133881111"), ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b")}, + {Label: "Status Stop contact", URL: swStatusURL, Data: statusStop, Status: 200, Response: `"status":"F"`, + PrepRequest: addValidSignature}, {Label: "Status No Params", URL: swStatusURL, Data: " ", Status: 200, Response: "no msg status, ignoring"}, {Label: "Status Invalid Status", URL: swStatusURL, Data: statusInvalid, Status: 400, Response: "unknown status 'huh'"}, {Label: "Status Valid", URL: swStatusURL, Data: statusValid, Status: 200, Response: `"status":"D"`, ExternalID: Sp("SMe287d7109a5a925f182f0e07fe5b223b")}, From 17f7da477258cdf3a704ffa6cd443a7d39297e72 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 18 Mar 2022 12:05:21 -0500 Subject: [PATCH 089/146] Update CHANGELOG.md for v7.3.0 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9072df787..d6b770ce7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.3.0 +---------- + * Support stopping contacts when we get stop events on status callbacks + v7.2.0 ---------- * CI testing with go 1.17.7 From 6868a5c40a1536e4f862d3c8ee685a7d33b07ed8 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 21 Mar 2022 11:57:10 -0500 Subject: [PATCH 090/146] Fix handling stops via status callbacks on Twilio --- handlers/twiml/twiml.go | 44 +++++++++++++++++++++++------------------ 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/handlers/twiml/twiml.go b/handlers/twiml/twiml.go index f6bd9eaac..ce39b0769 100644 --- a/handlers/twiml/twiml.go +++ b/handlers/twiml/twiml.go @@ -113,24 +113,7 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } - // create our URN - var urn urns.URN - if channel.IsScheme(urns.WhatsAppScheme) { - // Twilio Whatsapp from is in the form: whatsapp:+12211414154 or +12211414154 - var fromTel string - parts := strings.Split(form.From, ":") - if len(parts) > 1 { - fromTel = parts[1] - } else { - fromTel = parts[0] - } - - // trim off left +, official whatsapp IDs dont have that - urn, err = urns.NewWhatsAppURN(strings.TrimLeft(fromTel, "+")) - } else { - urn, err = urns.NewTelURNForCountry(form.From, form.FromCountry) - } - + urn, err := h.parseURN(channel, form.From, form.FromCountry) if err != nil { return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } @@ -199,8 +182,13 @@ func (h *handler) receiveStatus(ctx context.Context, channel courier.Channel, w errorCode, _ := strconv.ParseInt(form.ErrorCode, 10, 64) if errorCode == errorStopped { + urn, err := h.parseURN(channel, form.To, "") + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + // create a stop channel event - channelEvent := h.Backend().NewChannelEvent(channel, courier.StopContact, urns.URN(form.To)) + channelEvent := h.Backend().NewChannelEvent(channel, courier.StopContact, urn) err = h.Backend().WriteChannelEvent(ctx, channelEvent) if err != nil { return nil, err @@ -326,6 +314,24 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat return status, nil } +func (h *handler) parseURN(channel courier.Channel, text, country string) (urns.URN, error) { + if channel.IsScheme(urns.WhatsAppScheme) { + // Twilio Whatsapp from is in the form: whatsapp:+12211414154 or +12211414154 + var fromTel string + parts := strings.Split(text, ":") + if len(parts) > 1 { + fromTel = parts[1] + } else { + fromTel = parts[0] + } + + // trim off left +, official whatsapp IDs dont have that + return urns.NewWhatsAppURN(strings.TrimLeft(fromTel, "+")) + } + + return urns.NewTelURNForCountry(text, country) +} + func (h *handler) baseURL(c courier.Channel) string { // Twilio channels use the Twili base URL if c.ChannelType() == "T" || c.ChannelType() == "TMS" || c.ChannelType() == "TWA" { From 087a320bdd38bdf51ddaf63396e8ed2b9f83a4ef Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 21 Mar 2022 13:10:23 -0500 Subject: [PATCH 091/146] Update CHANGELOG.md for v7.3.1 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d6b770ce7..e71877ff7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.3.1 +---------- + * Fix handling stops via status callbacks on Twilio + v7.3.0 ---------- * Support stopping contacts when we get stop events on status callbacks From 2d19d676ace03a26d6e8b6faebd47894fabcd060 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Tue, 22 Mar 2022 14:52:02 -0500 Subject: [PATCH 092/146] Update golang.org/x/sys --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 0a157e171..c5a8de616 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect - golang.org/x/sys v0.0.0-20210423082822-04245dca01da // indirect + golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect golang.org/x/text v0.3.6 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) diff --git a/go.sum b/go.sum index 0d9c2bbb1..85e0c7856 100644 --- a/go.sum +++ b/go.sum @@ -114,8 +114,9 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 1eed670755119c0a2d2b6144f9a62fbd81d53d8b Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 8 Apr 2022 19:02:23 +0200 Subject: [PATCH 093/146] Refactor FBA tests --- handlers/facebookapp/facebookapp_test.go | 744 +----------------- .../testdata/fba/attachmentFBA.json | 31 + .../testdata/fba/differentPageFBA.json | 24 + handlers/facebookapp/testdata/fba/dlr.json | 27 + .../testdata/fba/duplicateMsgFBA.json | 43 + .../facebookapp/testdata/fba/echoFBA.json | 23 + .../facebookapp/testdata/fba/helloMsgFBA.json | 24 + .../testdata/fba/invalidURNFBA.json | 24 + .../testdata/fba/locationAttachment.json | 34 + .../testdata/fba/noEntriesFBA.json | 4 + .../testdata/fba/noMessagingEntriesFBA.json | 8 + .../facebookapp/testdata/fba/notPage.json | 6 + handlers/facebookapp/testdata/fba/optIn.json | 23 + .../testdata/fba/optInUserRef.json | 24 + .../facebookapp/testdata/fba/postback.json | 29 + .../testdata/fba/postbackGetStarted.json | 24 + .../testdata/fba/postbackReferral.json | 30 + .../facebookapp/testdata/fba/referral.json | 27 + .../facebookapp/testdata/fba/thumbsUp.json | 32 + .../fba/unknownMessagingEntryFBA.json | 19 + .../facebookapp/testdata/ig/attachmentIG.json | 31 + .../testdata/ig/differentPageIG.json | 24 + .../testdata/ig/duplicateMsgIG.json | 43 + handlers/facebookapp/testdata/ig/echoIG.json | 23 + .../facebookapp/testdata/ig/helloMsgIG.json | 24 + .../testdata/ig/icebreakerGetStarted.json | 24 + .../facebookapp/testdata/ig/invalidURNIG.json | 24 + .../facebookapp/testdata/ig/like_heart.json | 28 + .../facebookapp/testdata/ig/noEntriesIG.json | 4 + .../testdata/ig/noMessagingEntriesIG.json | 8 + .../facebookapp/testdata/ig/notInstagram.json | 6 + .../testdata/ig/storyMentionIG.json | 31 + .../testdata/ig/unknownMessagingEntryIG.json | 19 + .../facebookapp/testdata/ig/unsentMsgIG.json | 24 + test.go | 9 + 35 files changed, 815 insertions(+), 707 deletions(-) create mode 100644 handlers/facebookapp/testdata/fba/attachmentFBA.json create mode 100644 handlers/facebookapp/testdata/fba/differentPageFBA.json create mode 100644 handlers/facebookapp/testdata/fba/dlr.json create mode 100644 handlers/facebookapp/testdata/fba/duplicateMsgFBA.json create mode 100644 handlers/facebookapp/testdata/fba/echoFBA.json create mode 100644 handlers/facebookapp/testdata/fba/helloMsgFBA.json create mode 100644 handlers/facebookapp/testdata/fba/invalidURNFBA.json create mode 100644 handlers/facebookapp/testdata/fba/locationAttachment.json create mode 100644 handlers/facebookapp/testdata/fba/noEntriesFBA.json create mode 100644 handlers/facebookapp/testdata/fba/noMessagingEntriesFBA.json create mode 100644 handlers/facebookapp/testdata/fba/notPage.json create mode 100644 handlers/facebookapp/testdata/fba/optIn.json create mode 100644 handlers/facebookapp/testdata/fba/optInUserRef.json create mode 100644 handlers/facebookapp/testdata/fba/postback.json create mode 100644 handlers/facebookapp/testdata/fba/postbackGetStarted.json create mode 100644 handlers/facebookapp/testdata/fba/postbackReferral.json create mode 100644 handlers/facebookapp/testdata/fba/referral.json create mode 100644 handlers/facebookapp/testdata/fba/thumbsUp.json create mode 100644 handlers/facebookapp/testdata/fba/unknownMessagingEntryFBA.json create mode 100644 handlers/facebookapp/testdata/ig/attachmentIG.json create mode 100644 handlers/facebookapp/testdata/ig/differentPageIG.json create mode 100644 handlers/facebookapp/testdata/ig/duplicateMsgIG.json create mode 100644 handlers/facebookapp/testdata/ig/echoIG.json create mode 100644 handlers/facebookapp/testdata/ig/helloMsgIG.json create mode 100644 handlers/facebookapp/testdata/ig/icebreakerGetStarted.json create mode 100644 handlers/facebookapp/testdata/ig/invalidURNIG.json create mode 100644 handlers/facebookapp/testdata/ig/like_heart.json create mode 100644 handlers/facebookapp/testdata/ig/noEntriesIG.json create mode 100644 handlers/facebookapp/testdata/ig/noMessagingEntriesIG.json create mode 100644 handlers/facebookapp/testdata/ig/notInstagram.json create mode 100644 handlers/facebookapp/testdata/ig/storyMentionIG.json create mode 100644 handlers/facebookapp/testdata/ig/unknownMessagingEntryIG.json create mode 100644 handlers/facebookapp/testdata/ig/unsentMsgIG.json diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index d814f04ef..9ec8576c0 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -24,769 +24,99 @@ var testChannelsIG = []courier.Channel{ courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), } -var helloMsgFBA = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var helloMsgIG = `{ - "object":"instagram", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var duplicateMsgFBA = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }, - { - "id": "12345", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var duplicateMsgIG = `{ - "object":"instagram", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }, - { - "id": "12345", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var invalidURNFBA = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "abc5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var invalidURNIG = `{ - "object":"instagram", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "abc5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var attachmentFBA = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "mid": "external_id", - "attachments":[{ - "type":"image", - "payload":{ - "url":"https://image-url/foo.png" - } - }] - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var attachmentIG = `{ - "object":"instagram", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "mid": "external_id", - "attachments":[{ - "type":"image", - "payload":{ - "url":"https://image-url/foo.png" - } - }] - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var locationAttachment = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "mid": "external_id", - "attachments":[{ - "type":"location", - "payload":{ - "coordinates": { - "lat": 1.2, - "long": -1.3 - } - } - }] - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var thumbsUp = `{ - "object":"page", - "entry":[{ - "id":"12345", - "time":1459991487970, - "messaging":[{ - "sender":{"id":"5678"}, - "recipient":{"id":"12345"}, - "timestamp":1459991487970, - "message":{ - "mid":"external_id", - "attachments":[{ - "type":"image", - "payload":{ - "sticker_id": 369239263222822, - "url":"https://scontent.xx.fbcdn.net/v/arst" - } - }] - } - }] - }] -}` - -var like_heart = `{ - "object":"instagram", - "entry":[{ - "id":"12345", - "messaging":[{ - "sender":{"id":"5678"}, - "recipient":{"id":"12345"}, - "timestamp":1459991487970, - "message":{ - "mid":"external_id", - "attachments":[{ - "type":"like_heart" - }] - } - }], - "time":1459991487970 - }] -}` - -var differentPageIG = `{ - "object":"instagram", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1235" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var differentPageFBA = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "text": "Hello World", - "mid": "external_id" - }, - "recipient": { - "id": "1235" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var echoFBA = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970, - "message": { - "is_echo": true, - "mid": "qT7ywaK" - } - }] - }] -}` - -var echoIG = `{ - "object":"instagram", - "entry": [{ - "id": "12345", - "messaging": [{ - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970, - "message": { - "is_echo": true, - "mid": "qT7ywaK" - } - }] - }] -}` - -var icebreakerGetStarted = `{ - "object":"instagram", - "entry": [{ - "id": "12345", - "messaging": [{ - "postback": { - "title": "icebreaker question", - "payload": "get_started" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var optInUserRef = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "optin": { - "ref": "optin_ref", - "user_ref": "optin_user_ref" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var optIn = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "optin": { - "ref": "optin_ref" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var postback = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "postback": { - "title": "postback title", - "payload": "postback payload", - "referral": { - "ref": "postback ref", - "source": "postback source", - "type": "postback type" - } - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var postbackReferral = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "postback": { - "title": "postback title", - "payload": "get_started", - "referral": { - "ref": "postback ref", - "source": "postback source", - "type": "postback type", - "ad_id": "ad id" - } - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var postbackGetStarted = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "postback": { - "title": "postback title", - "payload": "get_started" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var referral = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "referral": { - "ref": "referral id", - "ad_id": "ad id", - "source": "referral source", - "type": "referral type" - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678", - "user_ref": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var dlr = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "delivery":{ - "mids":[ - "mid.1458668856218:ed81099e15d3f4f233" - ], - "watermark":1458668856253, - "seq":37 - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var unsentMsgIG = `{ - "object":"instagram", - "entry":[ - { - "id":"12345", - "time":1459991487970, - "messaging":[ - { - "sender":{ - "id":"5678" - }, - "recipient":{ - "id":"12345" - }, - "timestamp":1569262485349, - "message":{ - "mid":"external_id", - "is_deleted": true - } - } - ] - } - ] - } -` - -var notPage = `{ - "object":"notpage", - "entry": [{}] -}` - -var notInstagram = `{ - "object":"notinstagram", - "entry": [{}] -}` - -var noEntriesFBA = `{ - "object":"page", - "entry": [] -}` - -var noEntriesIG = `{ - "object":"instagram", - "entry": [] -}` - -var noMessagingEntriesFBA = `{ - "object":"page", - "entry": [{ - "id": "12345" - }] -}` - -var noMessagingEntriesIG = `{ - "object":"instagram", - "entry": [{ - "id": "12345" - }] -}` - -var unknownMessagingEntryFBA = `{ - "object":"page", - "entry": [{ - "id": "12345", - "messaging": [{ - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }] - }] -}` - -var unknownMessagingEntryIG = `{ - "object":"instagram", - "entry": [{ - "id": "12345", - "messaging": [{ - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }] - }] -}` - -var storyMentionIG = `{ - "object":"instagram", - "entry": [{ - "id": "12345", - "messaging": [{ - "message": { - "mid": "external_id", - "attachments":[{ - "type":"story_mention", - "payload":{ - "url":"https://story-url" - } - }] - }, - "recipient": { - "id": "12345" - }, - "sender": { - "id": "5678" - }, - "timestamp": 1459991487970 - }], - "time": 1459991487970 - }] -}` - -var notJSON = `blargh` - var testCasesFBA = []ChannelHandleTestCase{ - {Label: "Receive Message FBA", URL: "/c/fba/receive", Data: helloMsgFBA, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + {Label: "Receive Message FBA", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/helloMsgFBA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Invalid Signature", URL: "/c/fba/receive", Data: helloMsgFBA, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, + {Label: "Receive Invalid Signature", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/helloMsgFBA.json")), Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, - {Label: "No Duplicate Receive Message", URL: "/c/fba/receive", Data: duplicateMsgFBA, Status: 200, Response: "Handled", + {Label: "No Duplicate Receive Message", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/duplicateMsgFBA.json")), Status: 200, Response: "Handled", Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Attachment", URL: "/c/fba/receive", Data: attachmentFBA, Status: 200, Response: "Handled", + {Label: "Receive Attachment", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/attachmentFBA.json")), Status: 200, Response: "Handled", Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Location", URL: "/c/fba/receive", Data: locationAttachment, Status: 200, Response: "Handled", + {Label: "Receive Location", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/locationAttachment.json")), Status: 200, Response: "Handled", Text: Sp(""), Attachments: []string{"geo:1.200000,-1.300000"}, URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Thumbs Up", URL: "/c/fba/receive", Data: thumbsUp, Status: 200, Response: "Handled", + {Label: "Receive Thumbs Up", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/thumbsUp.json")), Status: 200, Response: "Handled", Text: Sp("👍"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive OptIn UserRef", URL: "/c/fba/receive", Data: optInUserRef, Status: 200, Response: "Handled", + {Label: "Receive OptIn UserRef", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/optInUserRef.json")), Status: 200, Response: "Handled", URN: Sp("facebook:ref:optin_user_ref"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.Referral), ChannelEventExtra: map[string]interface{}{"referrer_id": "optin_ref"}, PrepRequest: addValidSignature}, - {Label: "Receive OptIn", URL: "/c/fba/receive", Data: optIn, Status: 200, Response: "Handled", + {Label: "Receive OptIn", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/optIn.json")), Status: 200, Response: "Handled", URN: Sp("facebook:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.Referral), ChannelEventExtra: map[string]interface{}{"referrer_id": "optin_ref"}, PrepRequest: addValidSignature}, - {Label: "Receive Get Started", URL: "/c/fba/receive", Data: postbackGetStarted, Status: 200, Response: "Handled", + {Label: "Receive Get Started", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/postbackGetStarted.json")), Status: 200, Response: "Handled", URN: Sp("facebook:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), ChannelEventExtra: map[string]interface{}{"title": "postback title", "payload": "get_started"}, PrepRequest: addValidSignature}, - {Label: "Receive Referral Postback", URL: "/c/fba/receive", Data: postback, Status: 200, Response: "Handled", + {Label: "Receive Referral Postback", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/postback.json")), Status: 200, Response: "Handled", URN: Sp("facebook:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.Referral), ChannelEventExtra: map[string]interface{}{"title": "postback title", "payload": "postback payload", "referrer_id": "postback ref", "source": "postback source", "type": "postback type"}, PrepRequest: addValidSignature}, - {Label: "Receive Referral", URL: "/c/fba/receive", Data: postbackReferral, Status: 200, Response: "Handled", + {Label: "Receive Referral", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/postbackReferral.json")), Status: 200, Response: "Handled", URN: Sp("facebook:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.Referral), ChannelEventExtra: map[string]interface{}{"title": "postback title", "payload": "get_started", "referrer_id": "postback ref", "source": "postback source", "type": "postback type", "ad_id": "ad id"}, PrepRequest: addValidSignature}, - {Label: "Receive Referral", URL: "/c/fba/receive", Data: referral, Status: 200, Response: `"referrer_id":"referral id"`, + {Label: "Receive Referral", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/referral.json")), Status: 200, Response: `"referrer_id":"referral id"`, URN: Sp("facebook:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.Referral), ChannelEventExtra: map[string]interface{}{"referrer_id": "referral id", "source": "referral source", "type": "referral type", "ad_id": "ad id"}, PrepRequest: addValidSignature}, - {Label: "Receive DLR", URL: "/c/fba/receive", Data: dlr, Status: 200, Response: "Handled", + {Label: "Receive DLR", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/dlr.json")), Status: 200, Response: "Handled", Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), MsgStatus: Sp(courier.MsgDelivered), ExternalID: Sp("mid.1458668856218:ed81099e15d3f4f233"), PrepRequest: addValidSignature}, - {Label: "Different Page", URL: "/c/fba/receive", Data: differentPageFBA, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, - {Label: "Echo", URL: "/c/fba/receive", Data: echoFBA, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, - {Label: "Not Page", URL: "/c/fba/receive", Data: notPage, Status: 400, Response: "object expected 'page' or 'instagram', found notpage", PrepRequest: addValidSignature}, - {Label: "No Entries", URL: "/c/fba/receive", Data: noEntriesFBA, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, - {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: noMessagingEntriesFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: unknownMessagingEntryFBA, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Not JSON", URL: "/c/fba/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, - {Label: "Invalid URN", URL: "/c/fba/receive", Data: invalidURNFBA, Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, + {Label: "Different Page", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/differentPageFBA.json")), Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, + {Label: "Echo", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/echoFBA.json")), Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, + {Label: "Not Page", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/notPage.json")), Status: 400, Response: "object expected 'page' or 'instagram', found notpage", PrepRequest: addValidSignature}, + {Label: "No Entries", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/noEntriesFBA.json")), Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, + {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/noMessagingEntriesFBA.json")), Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/unknownMessagingEntryFBA.json")), Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Not JSON", URL: "/c/fba/receive", Data: "not JSON", Status: 400, Response: "Error", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/invalidURNFBA.json")), Status: 400, Response: "invalid facebook id", PrepRequest: addValidSignature}, } var testCasesIG = []ChannelHandleTestCase{ - {Label: "Receive Message", URL: "/c/ig/receive", Data: helloMsgIG, Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + {Label: "Receive Message", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/helloMsgIG.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Invalid Signature", URL: "/c/ig/receive", Data: helloMsgIG, Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, + {Label: "Receive Invalid Signature", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/helloMsgIG.json")), Status: 400, Response: "invalid request signature", PrepRequest: addInvalidSignature}, - {Label: "No Duplicate Receive Message", URL: "/c/ig/receive", Data: duplicateMsgIG, Status: 200, Response: "Handled", + {Label: "No Duplicate Receive Message", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/duplicateMsgIG.json")), Status: 200, Response: "Handled", Text: Sp("Hello World"), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Attachment", URL: "/c/ig/receive", Data: attachmentIG, Status: 200, Response: "Handled", + {Label: "Receive Attachment", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/attachmentIG.json")), Status: 200, Response: "Handled", Text: Sp(""), Attachments: []string{"https://image-url/foo.png"}, URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Like Heart", URL: "/c/ig/receive", Data: like_heart, Status: 200, Response: "Handled", + {Label: "Receive Like Heart", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/like_heart.json")), Status: 200, Response: "Handled", Text: Sp(""), URN: Sp("instagram:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Icebreaker Get Started", URL: "/c/ig/receive", Data: icebreakerGetStarted, Status: 200, Response: "Handled", + {Label: "Receive Icebreaker Get Started", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/icebreakerGetStarted.json")), Status: 200, Response: "Handled", URN: Sp("instagram:5678"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), ChannelEvent: Sp(courier.NewConversation), ChannelEventExtra: map[string]interface{}{"title": "icebreaker question", "payload": "get_started"}, PrepRequest: addValidSignature}, - - {Label: "Different Page", URL: "/c/ig/receive", Data: differentPageIG, Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, - {Label: "Echo", URL: "/c/ig/receive", Data: echoIG, Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, - {Label: "No Entries", URL: "/c/ig/receive", Data: noEntriesIG, Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, - {Label: "Not Instagram", URL: "/c/ig/receive", Data: notInstagram, Status: 400, Response: "object expected 'page' or 'instagram', found notinstagram", PrepRequest: addValidSignature}, - {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: noMessagingEntriesIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: unknownMessagingEntryIG, Status: 200, Response: "Handled", PrepRequest: addValidSignature}, - {Label: "Not JSON", URL: "/c/ig/receive", Data: notJSON, Status: 400, Response: "Error", PrepRequest: addValidSignature}, - {Label: "Invalid URN", URL: "/c/ig/receive", Data: invalidURNIG, Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, - {Label: "Story Mention", URL: "/c/ig/receive", Data: storyMentionIG, Status: 200, Response: `ignoring story_mention`, PrepRequest: addValidSignature}, - {Label: "Message unsent", URL: "/c/ig/receive", Data: unsentMsgIG, Status: 200, Response: `msg deleted`, PrepRequest: addValidSignature}, + {Label: "Different Page", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/differentPageIG.json")), Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, + {Label: "Echo", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/echoIG.json")), Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, + {Label: "No Entries", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/noEntriesIG.json")), Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, + {Label: "Not Instagram", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/notInstagram.json")), Status: 400, Response: "object expected 'page' or 'instagram', found notinstagram", PrepRequest: addValidSignature}, + {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/noMessagingEntriesIG.json")), Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/unknownMessagingEntryIG.json")), Status: 200, Response: "Handled", PrepRequest: addValidSignature}, + {Label: "Not JSON", URL: "/c/ig/receive", Data: "not JSON", Status: 400, Response: "Error", PrepRequest: addValidSignature}, + {Label: "Invalid URN", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/invalidURNIG.json")), Status: 400, Response: "invalid instagram id", PrepRequest: addValidSignature}, + {Label: "Story Mention", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/storyMentionIG.json")), Status: 200, Response: `ignoring story_mention`, PrepRequest: addValidSignature}, + {Label: "Message unsent", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/unsentMsgIG.json")), Status: 200, Response: `msg deleted`, PrepRequest: addValidSignature}, } func addValidSignature(r *http.Request) { diff --git a/handlers/facebookapp/testdata/fba/attachmentFBA.json b/handlers/facebookapp/testdata/fba/attachmentFBA.json new file mode 100644 index 000000000..85c94addf --- /dev/null +++ b/handlers/facebookapp/testdata/fba/attachmentFBA.json @@ -0,0 +1,31 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "mid": "external_id", + "attachments": [ + { + "type": "image", + "payload": { + "url": "https://image-url/foo.png" + } + } + ] + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/differentPageFBA.json b/handlers/facebookapp/testdata/fba/differentPageFBA.json new file mode 100644 index 000000000..380fbfc6e --- /dev/null +++ b/handlers/facebookapp/testdata/fba/differentPageFBA.json @@ -0,0 +1,24 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1235" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/dlr.json b/handlers/facebookapp/testdata/fba/dlr.json new file mode 100644 index 000000000..2a252d1e9 --- /dev/null +++ b/handlers/facebookapp/testdata/fba/dlr.json @@ -0,0 +1,27 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "delivery": { + "mids": [ + "mid.1458668856218:ed81099e15d3f4f233" + ], + "watermark": 1458668856253, + "seq": 37 + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/duplicateMsgFBA.json b/handlers/facebookapp/testdata/fba/duplicateMsgFBA.json new file mode 100644 index 000000000..a857e48de --- /dev/null +++ b/handlers/facebookapp/testdata/fba/duplicateMsgFBA.json @@ -0,0 +1,43 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + }, + { + "id": "12345", + "messaging": [ + { + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/echoFBA.json b/handlers/facebookapp/testdata/fba/echoFBA.json new file mode 100644 index 000000000..e90e8408e --- /dev/null +++ b/handlers/facebookapp/testdata/fba/echoFBA.json @@ -0,0 +1,23 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970, + "message": { + "is_echo": true, + "mid": "qT7ywaK" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/helloMsgFBA.json b/handlers/facebookapp/testdata/fba/helloMsgFBA.json new file mode 100644 index 000000000..183b7e7c9 --- /dev/null +++ b/handlers/facebookapp/testdata/fba/helloMsgFBA.json @@ -0,0 +1,24 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/invalidURNFBA.json b/handlers/facebookapp/testdata/fba/invalidURNFBA.json new file mode 100644 index 000000000..95abe574b --- /dev/null +++ b/handlers/facebookapp/testdata/fba/invalidURNFBA.json @@ -0,0 +1,24 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "abc5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/locationAttachment.json b/handlers/facebookapp/testdata/fba/locationAttachment.json new file mode 100644 index 000000000..aca25918a --- /dev/null +++ b/handlers/facebookapp/testdata/fba/locationAttachment.json @@ -0,0 +1,34 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "mid": "external_id", + "attachments": [ + { + "type": "location", + "payload": { + "coordinates": { + "lat": 1.2, + "long": -1.3 + } + } + } + ] + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/noEntriesFBA.json b/handlers/facebookapp/testdata/fba/noEntriesFBA.json new file mode 100644 index 000000000..4a862f4ed --- /dev/null +++ b/handlers/facebookapp/testdata/fba/noEntriesFBA.json @@ -0,0 +1,4 @@ +{ + "object": "page", + "entry": [] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/noMessagingEntriesFBA.json b/handlers/facebookapp/testdata/fba/noMessagingEntriesFBA.json new file mode 100644 index 000000000..b99c06f1c --- /dev/null +++ b/handlers/facebookapp/testdata/fba/noMessagingEntriesFBA.json @@ -0,0 +1,8 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345" + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/notPage.json b/handlers/facebookapp/testdata/fba/notPage.json new file mode 100644 index 000000000..1743fb31b --- /dev/null +++ b/handlers/facebookapp/testdata/fba/notPage.json @@ -0,0 +1,6 @@ +{ + "object": "notpage", + "entry": [ + {} + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/optIn.json b/handlers/facebookapp/testdata/fba/optIn.json new file mode 100644 index 000000000..c04934368 --- /dev/null +++ b/handlers/facebookapp/testdata/fba/optIn.json @@ -0,0 +1,23 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "optin": { + "ref": "optin_ref" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/optInUserRef.json b/handlers/facebookapp/testdata/fba/optInUserRef.json new file mode 100644 index 000000000..fc82744ef --- /dev/null +++ b/handlers/facebookapp/testdata/fba/optInUserRef.json @@ -0,0 +1,24 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "optin": { + "ref": "optin_ref", + "user_ref": "optin_user_ref" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/postback.json b/handlers/facebookapp/testdata/fba/postback.json new file mode 100644 index 000000000..495707830 --- /dev/null +++ b/handlers/facebookapp/testdata/fba/postback.json @@ -0,0 +1,29 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "postback": { + "title": "postback title", + "payload": "postback payload", + "referral": { + "ref": "postback ref", + "source": "postback source", + "type": "postback type" + } + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/postbackGetStarted.json b/handlers/facebookapp/testdata/fba/postbackGetStarted.json new file mode 100644 index 000000000..7e8f92569 --- /dev/null +++ b/handlers/facebookapp/testdata/fba/postbackGetStarted.json @@ -0,0 +1,24 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "postback": { + "title": "postback title", + "payload": "get_started" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/postbackReferral.json b/handlers/facebookapp/testdata/fba/postbackReferral.json new file mode 100644 index 000000000..701e5ffa6 --- /dev/null +++ b/handlers/facebookapp/testdata/fba/postbackReferral.json @@ -0,0 +1,30 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "postback": { + "title": "postback title", + "payload": "get_started", + "referral": { + "ref": "postback ref", + "source": "postback source", + "type": "postback type", + "ad_id": "ad id" + } + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/referral.json b/handlers/facebookapp/testdata/fba/referral.json new file mode 100644 index 000000000..ad8e1e772 --- /dev/null +++ b/handlers/facebookapp/testdata/fba/referral.json @@ -0,0 +1,27 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "referral": { + "ref": "referral id", + "ad_id": "ad id", + "source": "referral source", + "type": "referral type" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678", + "user_ref": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/thumbsUp.json b/handlers/facebookapp/testdata/fba/thumbsUp.json new file mode 100644 index 000000000..ef137a99d --- /dev/null +++ b/handlers/facebookapp/testdata/fba/thumbsUp.json @@ -0,0 +1,32 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "time": 1459991487970, + "messaging": [ + { + "sender": { + "id": "5678" + }, + "recipient": { + "id": "12345" + }, + "timestamp": 1459991487970, + "message": { + "mid": "external_id", + "attachments": [ + { + "type": "image", + "payload": { + "sticker_id": 369239263222822, + "url": "https://scontent.xx.fbcdn.net/v/arst" + } + } + ] + } + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/fba/unknownMessagingEntryFBA.json b/handlers/facebookapp/testdata/fba/unknownMessagingEntryFBA.json new file mode 100644 index 000000000..e7ddb1292 --- /dev/null +++ b/handlers/facebookapp/testdata/fba/unknownMessagingEntryFBA.json @@ -0,0 +1,19 @@ +{ + "object": "page", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/attachmentIG.json b/handlers/facebookapp/testdata/ig/attachmentIG.json new file mode 100644 index 000000000..a60e4bc7b --- /dev/null +++ b/handlers/facebookapp/testdata/ig/attachmentIG.json @@ -0,0 +1,31 @@ +{ + "object": "instagram", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "mid": "external_id", + "attachments": [ + { + "type": "image", + "payload": { + "url": "https://image-url/foo.png" + } + } + ] + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/differentPageIG.json b/handlers/facebookapp/testdata/ig/differentPageIG.json new file mode 100644 index 000000000..e0a6bbde8 --- /dev/null +++ b/handlers/facebookapp/testdata/ig/differentPageIG.json @@ -0,0 +1,24 @@ +{ + "object": "instagram", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "1235" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/duplicateMsgIG.json b/handlers/facebookapp/testdata/ig/duplicateMsgIG.json new file mode 100644 index 000000000..3ba9e0f03 --- /dev/null +++ b/handlers/facebookapp/testdata/ig/duplicateMsgIG.json @@ -0,0 +1,43 @@ +{ + "object": "instagram", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + }, + { + "id": "12345", + "messaging": [ + { + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/echoIG.json b/handlers/facebookapp/testdata/ig/echoIG.json new file mode 100644 index 000000000..783c6eca1 --- /dev/null +++ b/handlers/facebookapp/testdata/ig/echoIG.json @@ -0,0 +1,23 @@ +{ + "object": "instagram", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970, + "message": { + "is_echo": true, + "mid": "qT7ywaK" + } + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/helloMsgIG.json b/handlers/facebookapp/testdata/ig/helloMsgIG.json new file mode 100644 index 000000000..416c75e35 --- /dev/null +++ b/handlers/facebookapp/testdata/ig/helloMsgIG.json @@ -0,0 +1,24 @@ +{ + "object": "instagram", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/icebreakerGetStarted.json b/handlers/facebookapp/testdata/ig/icebreakerGetStarted.json new file mode 100644 index 000000000..d8655dc3a --- /dev/null +++ b/handlers/facebookapp/testdata/ig/icebreakerGetStarted.json @@ -0,0 +1,24 @@ +{ + "object": "instagram", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "postback": { + "title": "icebreaker question", + "payload": "get_started" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/invalidURNIG.json b/handlers/facebookapp/testdata/ig/invalidURNIG.json new file mode 100644 index 000000000..a95b8dd35 --- /dev/null +++ b/handlers/facebookapp/testdata/ig/invalidURNIG.json @@ -0,0 +1,24 @@ +{ + "object": "instagram", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "text": "Hello World", + "mid": "external_id" + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "abc5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/like_heart.json b/handlers/facebookapp/testdata/ig/like_heart.json new file mode 100644 index 000000000..cbcd4c31b --- /dev/null +++ b/handlers/facebookapp/testdata/ig/like_heart.json @@ -0,0 +1,28 @@ +{ + "object": "instagram", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "sender": { + "id": "5678" + }, + "recipient": { + "id": "12345" + }, + "timestamp": 1459991487970, + "message": { + "mid": "external_id", + "attachments": [ + { + "type": "like_heart" + } + ] + } + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/noEntriesIG.json b/handlers/facebookapp/testdata/ig/noEntriesIG.json new file mode 100644 index 000000000..1e07223fe --- /dev/null +++ b/handlers/facebookapp/testdata/ig/noEntriesIG.json @@ -0,0 +1,4 @@ +{ + "object": "instagram", + "entry": [] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/noMessagingEntriesIG.json b/handlers/facebookapp/testdata/ig/noMessagingEntriesIG.json new file mode 100644 index 000000000..182b08a2c --- /dev/null +++ b/handlers/facebookapp/testdata/ig/noMessagingEntriesIG.json @@ -0,0 +1,8 @@ +{ + "object": "instagram", + "entry": [ + { + "id": "12345" + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/notInstagram.json b/handlers/facebookapp/testdata/ig/notInstagram.json new file mode 100644 index 000000000..39dead49f --- /dev/null +++ b/handlers/facebookapp/testdata/ig/notInstagram.json @@ -0,0 +1,6 @@ +{ + "object": "notinstagram", + "entry": [ + {} + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/storyMentionIG.json b/handlers/facebookapp/testdata/ig/storyMentionIG.json new file mode 100644 index 000000000..6ed908e4d --- /dev/null +++ b/handlers/facebookapp/testdata/ig/storyMentionIG.json @@ -0,0 +1,31 @@ +{ + "object": "instagram", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "message": { + "mid": "external_id", + "attachments": [ + { + "type": "story_mention", + "payload": { + "url": "https://story-url" + } + } + ] + }, + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ], + "time": 1459991487970 + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/unknownMessagingEntryIG.json b/handlers/facebookapp/testdata/ig/unknownMessagingEntryIG.json new file mode 100644 index 000000000..f912b6522 --- /dev/null +++ b/handlers/facebookapp/testdata/ig/unknownMessagingEntryIG.json @@ -0,0 +1,19 @@ +{ + "object": "instagram", + "entry": [ + { + "id": "12345", + "messaging": [ + { + "recipient": { + "id": "12345" + }, + "sender": { + "id": "5678" + }, + "timestamp": 1459991487970 + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/ig/unsentMsgIG.json b/handlers/facebookapp/testdata/ig/unsentMsgIG.json new file mode 100644 index 000000000..379fc44b6 --- /dev/null +++ b/handlers/facebookapp/testdata/ig/unsentMsgIG.json @@ -0,0 +1,24 @@ +{ + "object":"instagram", + "entry":[ + { + "id":"12345", + "time":1459991487970, + "messaging":[ + { + "sender":{ + "id":"5678" + }, + "recipient":{ + "id":"12345" + }, + "timestamp":1569262485349, + "message":{ + "mid":"external_id", + "is_deleted": true + } + } + ] + } + ] +} diff --git a/test.go b/test.go index 69a634862..86240b5df 100644 --- a/test.go +++ b/test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "log" + "os" "strconv" "strings" "sync" @@ -726,3 +727,11 @@ type mockContact struct { } func (c *mockContact) UUID() ContactUUID { return c.uuid } + +func ReadFile(path string) []byte { + d, err := os.ReadFile(path) + if err != nil { + panic(err) + } + return d +} From e223a2eb93487fb32b9bb8ad7fb13dc1670e8a42 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 8 Apr 2022 19:24:46 +0200 Subject: [PATCH 094/146] Add support for WA Cloud API --- handlers/facebookapp/facebookapp.go | 781 +++++++++++++++++- handlers/facebookapp/facebookapp_test.go | 310 ++++++- .../facebookapp/testdata/cwa/audioCWA.json | 43 + .../facebookapp/testdata/cwa/buttonCWA.json | 44 + .../facebookapp/testdata/cwa/documentCWA.json | 43 + .../testdata/cwa/duplicateCWA.json | 48 ++ .../facebookapp/testdata/cwa/helloCWA.json | 39 + .../testdata/cwa/ignoreStatusCWA.json | 49 ++ .../facebookapp/testdata/cwa/imageCWA.json | 43 + .../facebookapp/testdata/cwa/invalidFrom.json | 39 + .../testdata/cwa/invalidStatusCWA.json | 49 ++ .../testdata/cwa/invalidTimestamp.json | 39 + .../facebookapp/testdata/cwa/locationCWA.json | 43 + .../testdata/cwa/validStatusCWA.json | 49 ++ .../facebookapp/testdata/cwa/videoCWA.json | 43 + .../facebookapp/testdata/cwa/voiceCWA.json | 42 + 16 files changed, 1684 insertions(+), 20 deletions(-) create mode 100644 handlers/facebookapp/testdata/cwa/audioCWA.json create mode 100644 handlers/facebookapp/testdata/cwa/buttonCWA.json create mode 100644 handlers/facebookapp/testdata/cwa/documentCWA.json create mode 100644 handlers/facebookapp/testdata/cwa/duplicateCWA.json create mode 100644 handlers/facebookapp/testdata/cwa/helloCWA.json create mode 100644 handlers/facebookapp/testdata/cwa/ignoreStatusCWA.json create mode 100644 handlers/facebookapp/testdata/cwa/imageCWA.json create mode 100644 handlers/facebookapp/testdata/cwa/invalidFrom.json create mode 100644 handlers/facebookapp/testdata/cwa/invalidStatusCWA.json create mode 100644 handlers/facebookapp/testdata/cwa/invalidTimestamp.json create mode 100644 handlers/facebookapp/testdata/cwa/locationCWA.json create mode 100644 handlers/facebookapp/testdata/cwa/validStatusCWA.json create mode 100644 handlers/facebookapp/testdata/cwa/videoCWA.json create mode 100644 handlers/facebookapp/testdata/cwa/voiceCWA.json diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index b4c879a0b..b81b4aba8 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -10,6 +10,7 @@ import ( "fmt" "net/http" "net/url" + "strconv" "strings" "time" @@ -28,6 +29,8 @@ var ( signatureHeader = "X-Hub-Signature" + configCWAPhoneNumberID = "cwa_phone_number_id" + // max for the body maxMsgLength = 1000 @@ -56,6 +59,17 @@ const ( payloadKey = "payload" ) +var waStatusMapping = map[string]courier.MsgStatusValue{ + "sent": courier.MsgSent, + "delivered": courier.MsgDelivered, + "read": courier.MsgDelivered, + "failed": courier.MsgFailed, +} + +var waIgnoreStatuses = map[string]bool{ + "deleted": true, +} + func newHandler(channelType courier.ChannelType, name string, useUUIDRoutes bool) courier.ChannelHandler { return &handler{handlers.NewBaseHandlerWithParams(channelType, name, useUUIDRoutes)} } @@ -63,6 +77,7 @@ func newHandler(channelType courier.ChannelType, name string, useUUIDRoutes bool func init() { courier.RegisterHandler(newHandler("IG", "Instagram", false)) courier.RegisterHandler(newHandler("FBA", "Facebook", false)) + courier.RegisterHandler(newHandler("CWA", "Cloud API WhatsApp", false)) } @@ -104,11 +119,88 @@ type User struct { // }] // }] // } + +type cwaMedia struct { + Caption string `json:"caption"` + Filename string `json:"filename"` + ID string `json:"id"` + Mimetype string `json:"mime_type"` + SHA256 string `json:"sha256"` +} type moPayload struct { Object string `json:"object"` Entry []struct { - ID string `json:"id"` - Time int64 `json:"time"` + ID string `json:"id"` + Time int64 `json:"time"` + Changes []struct { + Field string `json:"field"` + Value struct { + MessagingProduct string `json:"messaging_product"` + Metadata *struct { + DisplayPhoneNumber string `json:"display_phone_number"` + PhoneNumberID string `json:"phone_number_id"` + } `json:"metadata"` + Contacts []struct { + Profile struct { + Name string `json:"name"` + } `json:"profile"` + WaID string `json:"wa_id"` + } `json:"contacts"` + Messages []struct { + ID string `json:"id"` + From string `json:"from"` + Timestamp string `json:"timestamp"` + Type string `json:"type"` + Context *struct { + Forwarded bool `json:"forwarded"` + FrequentlyForwarded bool `json:"frequently_forwarded"` + From string `json:"from"` + ID string `json:"id"` + } `json:"context"` + Text struct { + Body string `json:"body"` + } `json:"text"` + Image *cwaMedia `json:"image"` + Audio *cwaMedia `json:"audio"` + Video *cwaMedia `json:"video"` + Document *cwaMedia `json:"document"` + Voice *cwaMedia `json:"voice"` + Location *struct { + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Name string `json:"name"` + Address string `json:"address"` + } `json:"location"` + Button *struct { + Text string `json:"text"` + Payload string `json:"payload"` + } `json:"button"` + } `json:"messages"` + Statuses []struct { + ID string `json:"id"` + RecipientID string `json:"recipient_id"` + Status string `json:"status"` + Timestamp string `json:"timestamp"` + Type string `json:"type"` + Conversation *struct { + ID string `json:"id"` + Origin *struct { + Type string `json:"type"` + } `json:"origin"` + ExpirationTimestamp int64 `json:"expiration_timestamp"` + } `json:"conversation"` + Pricing *struct { + PricingModel string `json:"pricing_model"` + Billable bool `json:"billable"` + Category string `json:"category"` + } `json:"pricing"` + } `json:"statuses"` + Errors []struct { + Code int `json:"code"` + Title string `json:"title"` + } `json:"errors"` + } `json:"value"` + } `json:"changes"` Messaging []struct { Sender Sender `json:"sender"` Recipient User `json:"recipient"` @@ -177,8 +269,8 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan } // is not a 'page' and 'instagram' object? ignore it - if payload.Object != "page" && payload.Object != "instagram" { - return nil, fmt.Errorf("object expected 'page' or 'instagram', found %s", payload.Object) + if payload.Object != "page" && payload.Object != "instagram" && payload.Object != "whatsapp_business_account" { + return nil, fmt.Errorf("object expected 'page', 'instagram' or 'whatsapp_business_account', found %s", payload.Object) } // no entries? ignore this request @@ -186,13 +278,25 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan return nil, fmt.Errorf("no entries found") } - entryID := payload.Entry[0].ID + var channelAddress string //if object is 'page' returns type FBA, if object is 'instagram' returns type IG if payload.Object == "page" { - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(entryID)) + channelAddress = payload.Entry[0].ID + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("FBA"), courier.ChannelAddress(channelAddress)) + } else if payload.Object == "instagram" { + channelAddress = payload.Entry[0].ID + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(channelAddress)) } else { - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("IG"), courier.ChannelAddress(entryID)) + if len(payload.Entry[0].Changes) == 0 { + return nil, fmt.Errorf("no changes found") + } + + channelAddress = payload.Entry[0].Changes[0].Value.Metadata.DisplayPhoneNumber + if channelAddress == "" { + return nil, fmt.Errorf("no channel adress found") + } + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("CWA"), courier.ChannelAddress(channelAddress)) } } @@ -215,6 +319,30 @@ func (h *handler) receiveVerify(ctx context.Context, channel courier.Channel, w return nil, err } +func resolveMediaURL(channel courier.Channel, mediaID string) (string, error) { + token := channel.StringConfigForKey(courier.ConfigAuthToken, "") + if token == "" { + return "", fmt.Errorf("missing token for WA channel") + } + + base, _ := url.Parse(graphURL) + path, _ := url.Parse(fmt.Sprintf("/%s", mediaID)) + retreiveURL := base.ResolveReference(path) + + // set the access token as the authorization header + req, _ := http.NewRequest(http.MethodGet, retreiveURL.String(), nil) + //req.Header.Set("User-Agent", utils.HTTPUserAgent) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + resp, err := utils.MakeHTTPRequest(req) + if err != nil { + return "", err + } + + mediaURL, err := jsonparser.GetString(resp.Body, "url") + return mediaURL, err +} + // receiveEvent is our HTTP handler function for incoming messages and status updates func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w http.ResponseWriter, r *http.Request) ([]courier.Event, error) { err := h.validateSignature(r) @@ -229,7 +357,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h } // is not a 'page' and 'instagram' object? ignore it - if payload.Object != "page" && payload.Object != "instagram" { + if payload.Object != "page" && payload.Object != "instagram" && payload.Object != "whatsapp_business_account" { return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request") } @@ -238,6 +366,150 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h return nil, handlers.WriteAndLogRequestIgnored(ctx, h, channel, w, r, "ignoring request, no entries") } + var events []courier.Event + var data []interface{} + + if channel.ChannelType() == "FBA" || channel.ChannelType() == "IG" { + events, data, err = h.processFacebookInstagramPayload(ctx, channel, payload, w, r) + } else { + events, data, err = h.processCloudWhatsAppPayload(ctx, channel, payload, w, r) + + } + + if err != nil { + return nil, err + } + + return events, courier.WriteDataResponse(ctx, w, http.StatusOK, "Events Handled", data) +} + +func (h *handler) processCloudWhatsAppPayload(ctx context.Context, channel courier.Channel, payload *moPayload, w http.ResponseWriter, r *http.Request) ([]courier.Event, []interface{}, error) { + // the list of events we deal with + events := make([]courier.Event, 0, 2) + + // the list of data we will return in our response + data := make([]interface{}, 0, 2) + + var contactNames = make(map[string]string) + + // for each entry + for _, entry := range payload.Entry { + if len(entry.Changes) == 0 { + continue + } + + for _, change := range entry.Changes { + + for _, contact := range change.Value.Contacts { + contactNames[contact.WaID] = contact.Profile.Name + } + + for _, msg := range change.Value.Messages { + // create our date from the timestamp + ts, err := strconv.ParseInt(msg.Timestamp, 10, 64) + if err != nil { + return nil, nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("invalid timestamp: %s", msg.Timestamp)) + } + date := time.Unix(ts, 0).UTC() + + urn, err := urns.NewWhatsAppURN(msg.From) + if err != nil { + return nil, nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + + text := "" + mediaURL := "" + + if msg.Type == "text" { + text = msg.Text.Body + } else if msg.Type == "audio" && msg.Audio != nil { + text = msg.Audio.Caption + mediaURL, err = resolveMediaURL(channel, msg.Audio.ID) + } else if msg.Type == "voice" && msg.Voice != nil { + text = msg.Voice.Caption + mediaURL, err = resolveMediaURL(channel, msg.Voice.ID) + } else if msg.Type == "button" && msg.Button != nil { + text = msg.Button.Text + } else if msg.Type == "document" && msg.Document != nil { + text = msg.Document.Caption + mediaURL, err = resolveMediaURL(channel, msg.Document.ID) + } else if msg.Type == "image" && msg.Image != nil { + text = msg.Image.Caption + mediaURL, err = resolveMediaURL(channel, msg.Image.ID) + } else if msg.Type == "video" && msg.Video != nil { + text = msg.Video.Caption + mediaURL, err = resolveMediaURL(channel, msg.Video.ID) + } else if msg.Type == "location" && msg.Location != nil { + mediaURL = fmt.Sprintf("geo:%f,%f", msg.Location.Latitude, msg.Location.Longitude) + } else { + // we received a message type we do not support. + courier.LogRequestError(r, channel, fmt.Errorf("unsupported message type %s", msg.Type)) + } + + // create our message + ev := h.Backend().NewIncomingMsg(channel, urn, text).WithReceivedOn(date).WithExternalID(msg.ID).WithContactName(contactNames[msg.From]) + event := h.Backend().CheckExternalIDSeen(ev) + + // we had an error downloading media + if err != nil { + courier.LogRequestError(r, channel, err) + } + + if mediaURL != "" { + event.WithAttachment(mediaURL) + } + + err = h.Backend().WriteMsg(ctx, event) + if err != nil { + return nil, nil, err + } + + h.Backend().WriteExternalIDSeen(event) + + events = append(events, event) + data = append(data, courier.NewMsgReceiveData(event)) + + } + + for _, status := range change.Value.Statuses { + + msgStatus, found := waStatusMapping[status.Status] + if !found { + if waIgnoreStatuses[status.Status] { + data = append(data, courier.NewInfoData(fmt.Sprintf("ignoring status: %s", status.Status))) + } else { + handlers.WriteAndLogRequestError(ctx, h, channel, w, r, fmt.Errorf("unknown status: %s", status.Status)) + } + continue + } + + event := h.Backend().NewMsgStatusForExternalID(channel, status.ID, msgStatus) + err := h.Backend().WriteMsgStatus(ctx, event) + + // we don't know about this message, just tell them we ignored it + if err == courier.ErrMsgNotFound { + data = append(data, courier.NewInfoData(fmt.Sprintf("message id: %s not found, ignored", status.ID))) + continue + } + + if err != nil { + return nil, nil, err + } + + events = append(events, event) + data = append(data, courier.NewStatusData(event)) + + } + + } + + } + return events, data, nil +} + +func (h *handler) processFacebookInstagramPayload(ctx context.Context, channel courier.Channel, payload *moPayload, w http.ResponseWriter, r *http.Request) ([]courier.Event, []interface{}, error) { + var err error + // the list of events we deal with events := make([]courier.Event, 0, 2) @@ -273,12 +545,12 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h if payload.Object == "instagram" { urn, err = urns.NewInstagramURN(sender) if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + return nil, nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } } else { urn, err = urns.NewFacebookURN(sender) if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + return nil, nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } } @@ -292,7 +564,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h if msg.OptIn.UserRef != "" { urn, err = urns.NewFacebookURN(urns.FacebookRefPrefix + msg.OptIn.UserRef) if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + return nil, nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) } } @@ -306,7 +578,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h err := h.Backend().WriteChannelEvent(ctx, event) if err != nil { - return nil, err + return nil, nil, err } events = append(events, event) @@ -341,7 +613,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h err := h.Backend().WriteChannelEvent(ctx, event) if err != nil { - return nil, err + return nil, nil, err } events = append(events, event) @@ -370,7 +642,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h err := h.Backend().WriteChannelEvent(ctx, event) if err != nil { - return nil, err + return nil, nil, err } events = append(events, event) @@ -435,7 +707,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h err := h.Backend().WriteMsg(ctx, event) if err != nil { - return nil, err + return nil, nil, err } h.Backend().WriteExternalIDSeen(event) @@ -456,7 +728,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h } if err != nil { - return nil, err + return nil, nil, err } events = append(events, event) @@ -468,7 +740,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h } } - return events, courier.WriteDataResponse(ctx, w, http.StatusOK, "Events Handled", data) + return events, data, nil } // { @@ -516,6 +788,16 @@ type mtQuickReply struct { } func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { + if msg.Channel().ChannelType() == "FBA" || msg.Channel().ChannelType() == "IG" { + return h.sendFacebookInstagramMsg(ctx, msg) + } else if msg.Channel().ChannelType() == "CWA" { + return h.sendCloudAPIWhatsappMsg(ctx, msg) + } + + return nil, fmt.Errorf("unssuported channel type") +} + +func (h *handler) sendFacebookInstagramMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { // can't do anything without an access token accessToken := msg.Channel().StringConfigForKey(courier.ConfigAuthToken, "") if accessToken == "" { @@ -661,8 +943,346 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat return status, nil } +type cwaMTMedia struct { + ID string `json:"id,omitempty"` + Link string `json:"link,omitempty"` + Caption string `json:"caption,omitempty"` + Filename string `json:"filename,omitempty"` +} + +type cwaMTSection struct { + Title string `json:"title,omitempty"` + Rows []cwaMTSectionRow `json:"rows" validate:"required"` +} + +type cwaMTSectionRow struct { + ID string `json:"id" validate:"required"` + Title string `json:"title,omitempty"` + Description string `json:"description,omitempty"` +} + +type cwaMTButton struct { + Type string `json:"type" validate:"required"` + Reply struct { + ID string `json:"id" validate:"required"` + Title string `json:"title" validate:"required"` + } `json:"reply" validate:"required"` +} + +type cwaParam struct { + Type string `json:"type"` + Text string `json:"text"` +} + +type cwaComponent struct { + Type string `json:"type"` + SubType string `json:"sub_type"` + Index string `json:"index"` + Params []*cwaParam `json:"parameters"` +} + +type cwaText struct { + Body string `json:"body"` +} + +type cwaLanguage struct { + Policy string `json:"policy"` + Code string `json:"code"` +} + +type cwaTemplate struct { + Name string `json:"name"` + Language *cwaLanguage `json:"language"` + Components []*cwaComponent `json:"components"` +} + +type cwaInteractive struct { + Type string `json:"type"` + Header *struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + Video string `json:"video,omitempty"` + Image string `json:"image,omitempty"` + Document string `json:"document,omitempty"` + } `json:"header,omitempty"` + Body struct { + Text string `json:"text"` + } `json:"body" validate:"required"` + Footer *struct { + Text string `json:"text"` + } `json:"footer,omitempty"` + Action *struct { + Button string `json:"button,omitempty"` + Sections []cwaMTSection `json:"sections,omitempty"` + Buttons []cwaMTButton `json:"buttons,omitempty"` + } `json:"action,omitempty"` +} + +type cwaMTPayload struct { + MessagingProduct string `json:"messaging_product"` + PreviewURL bool `json:"preview_url"` + RecipientType string `json:"recipient_type"` + To string `json:"to"` + Type string `json:"type"` + + Text *cwaText `json:"text,omitempty"` + + Document *cwaMTMedia `json:"document,omitempty"` + Image *cwaMTMedia `json:"image,omitempty"` + Audio *cwaMTMedia `json:"audio,omitempty"` + Video *cwaMTMedia `json:"video,omitempty"` + + Interactive *cwaInteractive `json:"interactive,omitempty"` + + Template *cwaTemplate `json:"template,omitempty"` +} + +type cwaMTResponse struct { + Messages []*struct { + ID string `json:"id"` + } `json:"messages"` +} + +func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { + // can't do anything without an access token + accessToken := msg.Channel().StringConfigForKey(courier.ConfigAuthToken, "") + if accessToken == "" { + return nil, fmt.Errorf("missing access token") + } + + phoneNumberId := msg.Channel().StringConfigForKey(configCWAPhoneNumberID, "") + if phoneNumberId == "" { + return nil, fmt.Errorf("missing CWA phone number ID") + } + + base, _ := url.Parse(graphURL) + path, _ := url.Parse(fmt.Sprintf("/%s/messages", phoneNumberId)) + cwaPhoneURL := base.ResolveReference(path) + + status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) + + msgParts := make([]string, 0) + if msg.Text() != "" { + msgParts = handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) + } + qrs := msg.QuickReplies() + + for i := 0; i < len(msgParts)+len(msg.Attachments()); i++ { + payload := cwaMTPayload{MessagingProduct: "whatsapp", RecipientType: "individual", To: msg.URN().Path()} + + if len(msg.Attachments()) == 0 { + // do we have a template? + var templating *MsgTemplating + templating, err := h.getTemplate(msg) + if err != nil { + return nil, errors.Wrapf(err, "unable to decode template: %s for channel: %s", string(msg.Metadata()), msg.Channel().UUID()) + } + if templating != nil { + + payload.Type = "template" + + template := cwaTemplate{Name: templating.Template.Name, Language: &cwaLanguage{Policy: "deterministic", Code: templating.Language}} + payload.Template = &template + + component := &cwaComponent{Type: "body"} + + for _, v := range templating.Variables { + component.Params = append(component.Params, &cwaParam{Type: "text", Text: v}) + } + template.Components = append(payload.Template.Components, component) + + } else { + if i < (len(msgParts) + len(msg.Attachments()) - 1) { + // this is still a msg part + payload.Type = "text" + payload.Text = &cwaText{Body: msgParts[i-len(msg.Attachments())]} + } else { + if len(qrs) > 0 { + payload.Type = "interactive" + // We can use buttons + if len(qrs) <= 3 { + interactive := cwaInteractive{Type: "button", Body: struct { + Text string "json:\"text\"" + }{Text: msgParts[i-len(msg.Attachments())]}} + + btns := make([]cwaMTButton, len(qrs)) + for i, qr := range qrs { + btns[i] = cwaMTButton{ + Type: "reply", + } + btns[i].Reply.ID = fmt.Sprint(i) + btns[i].Reply.Title = qr + } + interactive.Action = &struct { + Button string "json:\"button,omitempty\"" + Sections []cwaMTSection "json:\"sections,omitempty\"" + Buttons []cwaMTButton "json:\"buttons,omitempty\"" + }{Buttons: btns} + payload.Interactive = &interactive + } else if len(qrs) <= 10 { + interactive := cwaInteractive{Type: "list", Body: struct { + Text string "json:\"text\"" + }{Text: msgParts[i-len(msg.Attachments())]}} + + section := cwaMTSection{ + Rows: make([]cwaMTSectionRow, len(qrs)), + } + for i, qr := range qrs { + section.Rows[i] = cwaMTSectionRow{ + ID: fmt.Sprint(i), + Title: qr, + } + } + + interactive.Action = &struct { + Button string "json:\"button,omitempty\"" + Sections []cwaMTSection "json:\"sections,omitempty\"" + Buttons []cwaMTButton "json:\"buttons,omitempty\"" + }{Button: "Menu", Sections: []cwaMTSection{ + section, + }} + + payload.Interactive = &interactive + } else { + return nil, fmt.Errorf("too many quick replies CWA supports only up to 10 quick replies") + } + } else { + // this is still a msg part + payload.Type = "text" + payload.Text = &cwaText{Body: msgParts[i-len(msg.Attachments())]} + } + } + } + + } else if i < len(msg.Attachments()) { + attType, attURL := handlers.SplitAttachment(msg.Attachments()[i]) + attType = strings.Split(attType, "/")[0] + if attType == "application" { + attType = "document" + } + payload.Type = attType + media := cwaMTMedia{Link: attURL} + + if attType == "image" { + payload.Image = &media + } else if attType == "audio" { + payload.Audio = &media + } else if attType == "video" { + payload.Video = &media + } else if attType == "document" { + payload.Document = &media + } + } else { + if i < (len(msgParts) + len(msg.Attachments()) - 1) { + // this is still a msg part + payload.Type = "text" + payload.Text = &cwaText{Body: msgParts[i-len(msg.Attachments())]} + } else { + if len(qrs) > 0 { + payload.Type = "interactive" + // We can use buttons + if len(qrs) <= 3 { + interactive := cwaInteractive{Type: "button", Body: struct { + Text string "json:\"text\"" + }{Text: msgParts[i-len(msg.Attachments())]}} + + btns := make([]cwaMTButton, len(qrs)) + for i, qr := range qrs { + btns[i] = cwaMTButton{ + Type: "reply", + } + btns[i].Reply.ID = fmt.Sprint(i) + btns[i].Reply.Title = qr + } + interactive.Action = &struct { + Button string "json:\"button,omitempty\"" + Sections []cwaMTSection "json:\"sections,omitempty\"" + Buttons []cwaMTButton "json:\"buttons,omitempty\"" + }{Buttons: btns} + payload.Interactive = &interactive + + } else if len(qrs) <= 10 { + interactive := cwaInteractive{Type: "list", Body: struct { + Text string "json:\"text\"" + }{Text: msgParts[i-len(msg.Attachments())]}} + + section := cwaMTSection{ + Rows: make([]cwaMTSectionRow, len(qrs)), + } + for i, qr := range qrs { + section.Rows[i] = cwaMTSectionRow{ + ID: fmt.Sprint(i), + Title: qr, + } + } + + interactive.Action = &struct { + Button string "json:\"button,omitempty\"" + Sections []cwaMTSection "json:\"sections,omitempty\"" + Buttons []cwaMTButton "json:\"buttons,omitempty\"" + }{Button: "Menu", Sections: []cwaMTSection{ + section, + }} + + payload.Interactive = &interactive + } else { + return nil, fmt.Errorf("too many quick replies CWA supports only up to 10 quick replies") + } + } else { + // this is still a msg part + payload.Type = "text" + payload.Text = &cwaText{Body: msgParts[i-len(msg.Attachments())]} + } + } + + } + + jsonBody, err := json.Marshal(payload) + if err != nil { + return status, err + } + + req, err := http.NewRequest(http.MethodPost, cwaPhoneURL.String(), bytes.NewReader(jsonBody)) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", accessToken)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + rr, err := utils.MakeHTTPRequest(req) + + // record our status and log + log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) + status.AddLog(log) + if err != nil { + return status, nil + } + + respPayload := &cwaMTResponse{} + err = json.Unmarshal(rr.Body, respPayload) + if err != nil { + log.WithError("Message Send Error", errors.Errorf("unable to unmarshal response body")) + return status, nil + } + externalID := respPayload.Messages[0].ID + if i == 0 && externalID != "" { + status.SetExternalID(externalID) + } + // this was wired successfully + status.SetStatus(courier.MsgWired) + + } + return status, nil +} + // DescribeURN looks up URN metadata for new contacts func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn urns.URN) (map[string]string, error) { + if channel.ChannelType() == "CWA" { + return map[string]string{}, nil + + } + // can't do anything with facebook refs, ignore them if urn.IsFacebookRef() { return map[string]string{}, nil @@ -746,3 +1366,130 @@ func fbCalculateSignature(appSecret string, body []byte) (string, error) { return hex.EncodeToString(mac.Sum(nil)), nil } + +func (h *handler) getTemplate(msg courier.Msg) (*MsgTemplating, error) { + mdJSON := msg.Metadata() + if len(mdJSON) == 0 { + return nil, nil + } + metadata := &TemplateMetadata{} + err := json.Unmarshal(mdJSON, metadata) + if err != nil { + return nil, err + } + templating := metadata.Templating + if templating == nil { + return nil, nil + } + + // check our template is valid + err = handlers.Validate(templating) + if err != nil { + return nil, errors.Wrapf(err, "invalid templating definition") + } + // check country + if templating.Country != "" { + templating.Language = fmt.Sprintf("%s_%s", templating.Language, templating.Country) + } + + // map our language from iso639-3_iso3166-2 to the WA country / iso638-2 pair + language, found := languageMap[templating.Language] + if !found { + return nil, fmt.Errorf("unable to find mapping for language: %s", templating.Language) + } + templating.Language = language + + return templating, err +} + +type TemplateMetadata struct { + Templating *MsgTemplating `json:"templating"` +} + +type MsgTemplating struct { + Template struct { + Name string `json:"name" validate:"required"` + UUID string `json:"uuid" validate:"required"` + } `json:"template" validate:"required,dive"` + Language string `json:"language" validate:"required"` + Country string `json:"country"` + Namespace string `json:"namespace"` + Variables []string `json:"variables"` +} + +// mapping from iso639-3_iso3166-2 to WA language code +var languageMap = map[string]string{ + "afr": "af", // Afrikaans + "sqi": "sq", // Albanian + "ara": "ar", // Arabic + "aze": "az", // Azerbaijani + "ben": "bn", // Bengali + "bul": "bg", // Bulgarian + "cat": "ca", // Catalan + "zho": "zh_CN", // Chinese + "zho_CN": "zh_CN", // Chinese (CHN) + "zho_HK": "zh_HK", // Chinese (HKG) + "zho_TW": "zh_TW", // Chinese (TAI) + "hrv": "hr", // Croatian + "ces": "cs", // Czech + "dah": "da", // Danish + "nld": "nl", // Dutch + "eng": "en", // English + "eng_GB": "en_GB", // English (UK) + "eng_US": "en_US", // English (US) + "est": "et", // Estonian + "fil": "fil", // Filipino + "fin": "fi", // Finnish + "fra": "fr", // French + "kat": "ka", // Georgian + "deu": "de", // German + "ell": "el", // Greek + "guj": "gu", // Gujarati + "hau": "ha", // Hausa + "enb": "he", // Hebrew + "hin": "hi", // Hindi + "hun": "hu", // Hungarian + "ind": "id", // Indonesian + "gle": "ga", // Irish + "ita": "it", // Italian + "jpn": "ja", // Japanese + "kan": "kn", // Kannada + "kaz": "kk", // Kazakh + "kin": "rw_RW", // Kinyarwanda + "kor": "ko", // Korean + "kir": "ky_KG", // Kyrgyzstan + "lao": "lo", // Lao + "lav": "lv", // Latvian + "lit": "lt", // Lithuanian + "mal": "ml", // Malayalam + "mkd": "mk", // Macedonian + "msa": "ms", // Malay + "mar": "mr", // Marathi + "nob": "nb", // Norwegian + "fas": "fa", // Persian + "pol": "pl", // Polish + "por": "pt_PT", // Portuguese + "por_BR": "pt_BR", // Portuguese (BR) + "por_PT": "pt_PT", // Portuguese (POR) + "pan": "pa", // Punjabi + "ron": "ro", // Romanian + "rus": "ru", // Russian + "srp": "sr", // Serbian + "slk": "sk", // Slovak + "slv": "sl", // Slovenian + "spa": "es", // Spanish + "spa_AR": "es_AR", // Spanish (ARG) + "spa_ES": "es_ES", // Spanish (SPA) + "spa_MX": "es_MX", // Spanish (MEX) + "swa": "sw", // Swahili + "swe": "sv", // Swedish + "tam": "ta", // Tamil + "tel": "te", // Telugu + "tha": "th", // Thai + "tur": "tr", // Turkish + "ukr": "uk", // Ukrainian + "urd": "ur", // Urdu + "uzb": "uz", // Uzbek + "vie": "vi", // Vietnamese + "zul": "zu", // Zulu +} diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 9ec8576c0..c5a526ebe 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -2,6 +2,7 @@ package facebookapp import ( "context" + "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -24,6 +25,10 @@ var testChannelsIG = []courier.Channel{ courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), } +var testChannelsCWA = []courier.Channel{ + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "CWA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), +} + var testCasesFBA = []ChannelHandleTestCase{ {Label: "Receive Message FBA", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/helloMsgFBA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Hello World"), URN: Sp("facebook:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 4, 7, 1, 11, 27, 970000000, time.UTC)), @@ -77,7 +82,7 @@ var testCasesFBA = []ChannelHandleTestCase{ {Label: "Different Page", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/differentPageFBA.json")), Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, {Label: "Echo", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/echoFBA.json")), Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, - {Label: "Not Page", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/notPage.json")), Status: 400, Response: "object expected 'page' or 'instagram', found notpage", PrepRequest: addValidSignature}, + {Label: "Not Page", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/notPage.json")), Status: 400, Response: "object expected 'page', 'instagram' or 'whatsapp_business_account', found notpage", PrepRequest: addValidSignature}, {Label: "No Entries", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/noEntriesFBA.json")), Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, {Label: "No Messaging Entries", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/noMessagingEntriesFBA.json")), Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Unknown Messaging Entry", URL: "/c/fba/receive", Data: string(courier.ReadFile("./testdata/fba/unknownMessagingEntryFBA.json")), Status: 200, Response: "Handled", PrepRequest: addValidSignature}, @@ -110,7 +115,7 @@ var testCasesIG = []ChannelHandleTestCase{ {Label: "Different Page", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/differentPageIG.json")), Status: 200, Response: `"data":[]`, PrepRequest: addValidSignature}, {Label: "Echo", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/echoIG.json")), Status: 200, Response: `ignoring echo`, PrepRequest: addValidSignature}, {Label: "No Entries", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/noEntriesIG.json")), Status: 400, Response: "no entries found", PrepRequest: addValidSignature}, - {Label: "Not Instagram", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/notInstagram.json")), Status: 400, Response: "object expected 'page' or 'instagram', found notinstagram", PrepRequest: addValidSignature}, + {Label: "Not Instagram", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/notInstagram.json")), Status: 400, Response: "object expected 'page', 'instagram' or 'whatsapp_business_account', found notinstagram", PrepRequest: addValidSignature}, {Label: "No Messaging Entries", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/noMessagingEntriesIG.json")), Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Unknown Messaging Entry", URL: "/c/ig/receive", Data: string(courier.ReadFile("./testdata/ig/unknownMessagingEntryIG.json")), Status: 200, Response: "Handled", PrepRequest: addValidSignature}, {Label: "Not JSON", URL: "/c/ig/receive", Data: "not JSON", Status: 400, Response: "Error", PrepRequest: addValidSignature}, @@ -213,10 +218,111 @@ func TestDescribeIG(t *testing.T) { } } +func TestDescribeCWA(t *testing.T) { + handler := newHandler("CWA", "Cloud API WhatsApp", false).(courier.URNDescriber) + + tcs := []struct { + urn urns.URN + metadata map[string]string + }{{"whatsapp:1337", map[string]string{}}, + {"whatsapp:4567", map[string]string{}}} + + for _, tc := range tcs { + metadata, _ := handler.DescribeURN(context.Background(), testChannelsCWA[0], tc.urn) + assert.Equal(t, metadata, tc.metadata) + } +} + +var cwaReceiveURL = "/c/cwa/receive" + +var testCasesCWA = []ChannelHandleTestCase{ + {Label: "Receive Message CWA", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/helloCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("Hello World"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), + PrepRequest: addValidSignature}, + {Label: "Receive Duplicate Valid Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/duplicateCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("Hello World"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Valid Voice Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/voiceCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp(""), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Attachment: Sp("https://foo.bar/attachmentURL_Voice"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Valid Button Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/buttonCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("No"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Valid Document Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/documentCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("80skaraokesonglistartist"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Attachment: Sp("https://foo.bar/attachmentURL_Document"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), + PrepRequest: addValidSignature}, + {Label: "Receive Valid Image Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/imageCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("Check out my new phone!"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Attachment: Sp("https://foo.bar/attachmentURL_Image"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), + PrepRequest: addValidSignature}, + {Label: "Receive Valid Video Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/videoCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("Check out my new phone!"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Attachment: Sp("https://foo.bar/attachmentURL_Video"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), + PrepRequest: addValidSignature}, + {Label: "Receive Valid Audio Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/audioCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("Check out my new phone!"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Attachment: Sp("https://foo.bar/attachmentURL_Audio"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), + PrepRequest: addValidSignature}, + {Label: "Receive Valid Location Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/locationCWA.json")), Status: 200, Response: `"type":"msg"`, + Text: Sp(""), Attachment: Sp("geo:0.000000,1.000000"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), + PrepRequest: addValidSignature}, + + {Label: "Receive Invalid JSON", URL: cwaReceiveURL, Data: "not json", Status: 400, Response: "unable to parse", PrepRequest: addValidSignature}, + {Label: "Receive Invalid JSON", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/invalidFrom.json")), Status: 400, Response: "invalid whatsapp id", PrepRequest: addValidSignature}, + {Label: "Receive Invalid JSON", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/invalidTimestamp.json")), Status: 400, Response: "invalid timestamp", PrepRequest: addValidSignature}, + + {Label: "Receive Valid Status", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/validStatusCWA.json")), Status: 200, Response: `"type":"status"`, + MsgStatus: Sp("S"), ExternalID: Sp("external_id"), PrepRequest: addValidSignature}, + {Label: "Receive Invalid Status", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/invalidStatusCWA.json")), Status: 400, Response: `"unknown status: in_orbit"`, PrepRequest: addValidSignature}, + {Label: "Receive Ignore Status", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/ignoreStatusCWA.json")), Status: 200, Response: `"ignoring status: deleted"`, PrepRequest: addValidSignature}, +} + func TestHandler(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + accessToken := r.Header.Get("Authorization") + defer r.Body.Close() + + // invalid auth token + if accessToken != "Bearer a123" { + fmt.Printf("Access token: %s\n", accessToken) + http.Error(w, "invalid auth token", 403) + return + } + + if strings.HasSuffix(r.URL.Path, "image") { + w.Write([]byte(`{"url": "https://foo.bar/attachmentURL_Image"}`)) + return + } + + if strings.HasSuffix(r.URL.Path, "audio") { + w.Write([]byte(`{"url": "https://foo.bar/attachmentURL_Audio"}`)) + return + } + + if strings.HasSuffix(r.URL.Path, "voice") { + w.Write([]byte(`{"url": "https://foo.bar/attachmentURL_Voice"}`)) + return + } + + if strings.HasSuffix(r.URL.Path, "video") { + w.Write([]byte(`{"url": "https://foo.bar/attachmentURL_Video"}`)) + return + } + + if strings.HasSuffix(r.URL.Path, "document") { + w.Write([]byte(`{"url": "https://foo.bar/attachmentURL_Document"}`)) + return + } + + // valid token + w.Write([]byte(`{"url": "https://foo.bar/attachmentURL"}`)) + + })) + graphURL = server.URL + + RunChannelTestCases(t, testChannelsCWA, newHandler("CWA", "Cloud API WhatsApp", false), testCasesCWA) RunChannelTestCases(t, testChannelsFBA, newHandler("FBA", "Facebook", false), testCasesFBA) RunChannelTestCases(t, testChannelsIG, newHandler("IG", "Instagram", false), testCasesIG) - } func BenchmarkHandler(b *testing.B) { @@ -256,6 +362,7 @@ func TestVerify(t *testing.T) { // setSendURL takes care of setting the send_url to our test server host func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, m courier.Msg) { sendURL = s.URL + graphURL = s.URL } var SendTestCasesFBA = []ChannelSendTestCase{ @@ -387,13 +494,210 @@ var SendTestCasesIG = []ChannelSendTestCase{ SendPrep: setSendURL}, } +var SendTestCasesCWA = []ChannelSendTestCase{ + {Label: "Plain Send", + Text: "Simple Message", URN: "whatsapp:250788123123", Path: "/12345_ID/messages", + Status: "W", ExternalID: "157b5e14568e8", + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + RequestBody: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"text","text":{"body":"Simple Message"}}`, + SendPrep: setSendURL}, + {Label: "Unicode Send", + Text: "☺", URN: "whatsapp:250788123123", Path: "/12345_ID/messages", + Status: "W", ExternalID: "157b5e14568e8", + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + RequestBody: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"text","text":{"body":"☺"}}`, + SendPrep: setSendURL}, + {Label: "Audio Send", + Text: "audio caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"audio/mpeg:https://foo.bar/audio.mp3"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"audio","audio":{"link":"https://foo.bar/audio.mp3"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"text","text":{"body":"audio caption"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, + {Label: "Document Send", + Text: "document caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"application/pdf:https://foo.bar/document.pdf"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"document","document":{"link":"https://foo.bar/document.pdf"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"text","text":{"body":"document caption"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, + + {Label: "Image Send", + Text: "document caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"image","image":{"link":"https://foo.bar/image.jpg"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"text","text":{"body":"document caption"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, + {Label: "Video Send", + Text: "video caption", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"video/mp4:https://foo.bar/video.mp4"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"video","video":{"link":"https://foo.bar/video.mp4"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"text","text":{"body":"video caption"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, + + {Label: "Template Send", + Text: "templated message", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Metadata: json.RawMessage(`{ "templating": { "template": { "name": "revive_issue", "uuid": "171f8a4d-f725-46d7-85a6-11aceff0bfe3" }, "language": "eng", "variables": ["Chef", "tomorrow"]}}`), + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, + RequestBody: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"template","template":{"name":"revive_issue","language":{"policy":"deterministic","code":"en"},"components":[{"type":"body","sub_type":"","index":"","parameters":[{"type":"text","text":"Chef"},{"type":"text","text":"tomorrow"}]}]}}`, + SendPrep: setSendURL, + }, + + {Label: "Template Country Language", + Text: "templated message", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Metadata: json.RawMessage(`{ "templating": { "template": { "name": "revive_issue", "uuid": "171f8a4d-f725-46d7-85a6-11aceff0bfe3" }, "language": "eng", "country": "US", "variables": ["Chef", "tomorrow"]}}`), + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, + RequestBody: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"template","template":{"name":"revive_issue","language":{"policy":"deterministic","code":"en_US"},"components":[{"type":"body","sub_type":"","index":"","parameters":[{"type":"text","text":"Chef"},{"type":"text","text":"tomorrow"}]}]}}`, + SendPrep: setSendURL, + }, + {Label: "Template Invalid Language", + Text: "templated message", URN: "whatsapp:250788123123", + Error: `unable to decode template: {"templating": { "template": { "name": "revive_issue", "uuid": "8ca114b4-bee2-4d3b-aaf1-9aa6b48d41e8" }, "language": "bnt", "variables": ["Chef", "tomorrow"]}} for channel: 8eb23e93-5ecb-45ba-b726-3b064e0c56ab: unable to find mapping for language: bnt`, + Metadata: json.RawMessage(`{"templating": { "template": { "name": "revive_issue", "uuid": "8ca114b4-bee2-4d3b-aaf1-9aa6b48d41e8" }, "language": "bnt", "variables": ["Chef", "tomorrow"]}}`), + }, + {Label: "Interactive Button Message Send", + Text: "Interactive Button Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"BUTTON1"}, + Status: "W", ExternalID: "157b5e14568e8", + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + RequestBody: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"interactive","interactive":{"type":"button","body":{"text":"Interactive Button Msg"},"action":{"buttons":[{"type":"reply","reply":{"id":"0","title":"BUTTON1"}}]}}}`, + SendPrep: setSendURL}, + {Label: "Interactive List Message Send", + Text: "Interactive List Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"ROW1", "ROW2", "ROW3", "ROW4"}, + Status: "W", ExternalID: "157b5e14568e8", + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + RequestBody: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"interactive","interactive":{"type":"list","body":{"text":"Interactive List Msg"},"action":{"button":"Menu","sections":[{"rows":[{"id":"0","title":"ROW1"},{"id":"1","title":"ROW2"},{"id":"2","title":"ROW3"},{"id":"3","title":"ROW4"}]}]}}}`, + SendPrep: setSendURL}, + {Label: "Interactive Button Message Send with attachment", + Text: "Interactive Button Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"BUTTON1"}, + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"image","image":{"link":"https://foo.bar/image.jpg"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"interactive","interactive":{"type":"button","body":{"text":"Interactive Button Msg"},"action":{"buttons":[{"type":"reply","reply":{"id":"0","title":"BUTTON1"}}]}}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, + {Label: "Interactive List Message Send with attachment", + Text: "Interactive List Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"ROW1", "ROW2", "ROW3", "ROW4"}, + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"image","image":{"link":"https://foo.bar/image.jpg"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/12345_ID/messages", + Body: `{"messaging_product":"whatsapp","preview_url":false,"recipient_type":"individual","to":"250788123123","type":"interactive","interactive":{"type":"list","body":{"text":"Interactive List Msg"},"action":{"button":"Menu","sections":[{"rows":[{"id":"0","title":"ROW1"},{"id":"1","title":"ROW2"},{"id":"2","title":"ROW3"},{"id":"3","title":"ROW4"}]}]}}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL}, +} + func TestSending(t *testing.T) { // shorter max msg length for testing maxMsgLength = 100 var ChannelFBA = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) var ChannelIG = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) + var ChannelCWA = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "CWA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123", configCWAPhoneNumberID: "12345_ID"}) RunChannelSendTestCases(t, ChannelFBA, newHandler("FBA", "Facebook", false), SendTestCasesFBA, nil) RunChannelSendTestCases(t, ChannelIG, newHandler("IG", "Instagram", false), SendTestCasesIG, nil) + RunChannelSendTestCases(t, ChannelCWA, newHandler("CWA", "Cloud API WhatsApp", false), SendTestCasesCWA, nil) } func TestSigning(t *testing.T) { diff --git a/handlers/facebookapp/testdata/cwa/audioCWA.json b/handlers/facebookapp/testdata/cwa/audioCWA.json new file mode 100644 index 000000000..47ce575db --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/audioCWA.json @@ -0,0 +1,43 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "messages": [ + { + "from": "5678", + "id": "external_id", + "audio": { + "file": "/usr/local/wamedia/shared/b1cf38-8734-4ad3-b4a1-ef0c10d0d683", + "id": "id_audio", + "mime_type": "image/jpeg", + "sha256": "29ed500fa64eb55fc19dc4124acb300e5dcc54a0f822a301ae99944db", + "caption": "Check out my new phone!" + }, + "timestamp": "1454119029", + "type": "audio" + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/buttonCWA.json b/handlers/facebookapp/testdata/cwa/buttonCWA.json new file mode 100644 index 000000000..36efcca6e --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/buttonCWA.json @@ -0,0 +1,44 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "messages": [ + { + "button": { + "payload": "No-Button-Payload", + "text": "No" + }, + "context": { + "from": "5678", + "id": "gBGGFmkiWVVPAgkgQkwi7IORac0" + }, + "from": "5678", + "id": "external_id", + "timestamp": "1454119029", + "type": "button" + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/documentCWA.json b/handlers/facebookapp/testdata/cwa/documentCWA.json new file mode 100644 index 000000000..d65921ef4 --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/documentCWA.json @@ -0,0 +1,43 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "messages": [ + { + "from": "5678", + "id": "external_id", + "timestamp": "1454119029", + "type": "document", + "document": { + "caption": "80skaraokesonglistartist", + "file": "/usr/local/wamedia/shared/fc233119-733f-49c-bcbd-b2f68f798e33", + "id": "id_document", + "mime_type": "application/pdf", + "sha256": "3b11fa6ef2bde1dd14726e09d3edaf782120919d06f6484f32d5d5caa4b8e" + } + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/duplicateCWA.json b/handlers/facebookapp/testdata/cwa/duplicateCWA.json new file mode 100644 index 000000000..f857c63f7 --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/duplicateCWA.json @@ -0,0 +1,48 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "messages": [ + { + "from": "5678", + "id": "external_id", + "timestamp": "1454119029", + "text": { + "body": "Hello World" + }, + "type": "text" + }, + { + "from": "5678", + "id": "external_id", + "timestamp": "1454119029", + "text": { + "body": "Hello World" + }, + "type": "text" + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/helloCWA.json b/handlers/facebookapp/testdata/cwa/helloCWA.json new file mode 100644 index 000000000..f49303d2f --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/helloCWA.json @@ -0,0 +1,39 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "messages": [ + { + "from": "5678", + "id": "external_id", + "timestamp": "1454119029", + "text": { + "body": "Hello World" + }, + "type": "text" + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/ignoreStatusCWA.json b/handlers/facebookapp/testdata/cwa/ignoreStatusCWA.json new file mode 100644 index 000000000..bf9f69714 --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/ignoreStatusCWA.json @@ -0,0 +1,49 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "statuses": [ + { + "id": "external_id", + "recipient_id": "5678", + "status": "deleted", + "timestamp": "1454119029", + "type": "message", + "conversation": { + "id": "CONVERSATION_ID", + "expiration_timestamp": 1454119029, + "origin": { + "type": "referral_conversion" + } + }, + "pricing": { + "pricing_model": "CBP", + "billable": false, + "category": "referral_conversion" + } + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/imageCWA.json b/handlers/facebookapp/testdata/cwa/imageCWA.json new file mode 100644 index 000000000..f06b631fc --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/imageCWA.json @@ -0,0 +1,43 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "messages": [ + { + "from": "5678", + "id": "external_id", + "image": { + "file": "/usr/local/wamedia/shared/b1cf38-8734-4ad3-b4a1-ef0c10d0d683", + "id": "id_image", + "mime_type": "image/jpeg", + "sha256": "29ed500fa64eb55fc19dc4124acb300e5dcc54a0f822a301ae99944db", + "caption": "Check out my new phone!" + }, + "timestamp": "1454119029", + "type": "image" + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/invalidFrom.json b/handlers/facebookapp/testdata/cwa/invalidFrom.json new file mode 100644 index 000000000..052db4a38 --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/invalidFrom.json @@ -0,0 +1,39 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "bla" + } + ], + "messages": [ + { + "from": "bla", + "id": "external_id", + "timestamp": "1454119029", + "text": { + "body": "Hello World" + }, + "type": "text" + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/invalidStatusCWA.json b/handlers/facebookapp/testdata/cwa/invalidStatusCWA.json new file mode 100644 index 000000000..60676257f --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/invalidStatusCWA.json @@ -0,0 +1,49 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "statuses": [ + { + "id": "external_id", + "recipient_id": "5678", + "status": "in_orbit", + "timestamp": "1454119029", + "type": "message", + "conversation": { + "id": "CONVERSATION_ID", + "expiration_timestamp": 1454119029, + "origin": { + "type": "referral_conversion" + } + }, + "pricing": { + "pricing_model": "CBP", + "billable": false, + "category": "referral_conversion" + } + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/invalidTimestamp.json b/handlers/facebookapp/testdata/cwa/invalidTimestamp.json new file mode 100644 index 000000000..e9f301ce9 --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/invalidTimestamp.json @@ -0,0 +1,39 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "bla" + } + ], + "messages": [ + { + "from": "bla", + "id": "external_id", + "timestamp": "asdf", + "text": { + "body": "Hello World" + }, + "type": "text" + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/locationCWA.json b/handlers/facebookapp/testdata/cwa/locationCWA.json new file mode 100644 index 000000000..15cedaa1f --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/locationCWA.json @@ -0,0 +1,43 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "messages": [ + { + "from": "5678", + "id": "external_id", + "location": { + "address": "Main Street Beach, Santa Cruz, CA", + "latitude": 0.000000, + "longitude": 1.000000, + "name": "Main Street Beach", + "url": "https://foursquare.com/v/4d7031d35b5df7744" + }, + "timestamp": "1454119029", + "type": "location" + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/validStatusCWA.json b/handlers/facebookapp/testdata/cwa/validStatusCWA.json new file mode 100644 index 000000000..0aaf2edb7 --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/validStatusCWA.json @@ -0,0 +1,49 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "statuses": [ + { + "id": "external_id", + "recipient_id": "5678", + "status": "sent", + "timestamp": "1454119029", + "type": "message", + "conversation": { + "id": "CONVERSATION_ID", + "expiration_timestamp": 1454119029, + "origin": { + "type": "referral_conversion" + } + }, + "pricing": { + "pricing_model": "CBP", + "billable": false, + "category": "referral_conversion" + } + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/videoCWA.json b/handlers/facebookapp/testdata/cwa/videoCWA.json new file mode 100644 index 000000000..210dd9a81 --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/videoCWA.json @@ -0,0 +1,43 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "messages": [ + { + "from": "5678", + "id": "external_id", + "video": { + "file": "/usr/local/wamedia/shared/b1cf38-8734-4ad3-b4a1-ef0c10d0d683", + "id": "id_video", + "mime_type": "image/jpeg", + "sha256": "29ed500fa64eb55fc19dc4124acb300e5dcc54a0f822a301ae99944db", + "caption": "Check out my new phone!" + }, + "timestamp": "1454119029", + "type": "video" + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file diff --git a/handlers/facebookapp/testdata/cwa/voiceCWA.json b/handlers/facebookapp/testdata/cwa/voiceCWA.json new file mode 100644 index 000000000..3e2022eff --- /dev/null +++ b/handlers/facebookapp/testdata/cwa/voiceCWA.json @@ -0,0 +1,42 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "12345", + "phone_number_id": "27681414235104944" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ], + "messages": [ + { + "from": "5678", + "id": "external_id", + "timestamp": "1454119029", + "type": "voice", + "voice": { + "file": "/usr/local/wamedia/shared/463e/b7ec/ff4e4d9bb1101879cbd411b2", + "id": "id_voice", + "mime_type": "audio/ogg; codecs=opus", + "sha256": "fa9e1807d936b7cebe63654ea3a7912b1fa9479220258d823590521ef53b0710" + } + } + ] + }, + "field": "messages" + } + ] + } + ] +} \ No newline at end of file From ed7597db6f82f6d2b83b293fc50656c26db65834 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 30 Mar 2022 16:27:14 -0300 Subject: [PATCH 095/146] Add link preview attribute --- handlers/whatsapp/whatsapp.go | 18 +++++++++++++++--- handlers/whatsapp/whatsapp_test.go | 6 ++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index bdd568f2c..f075805f2 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -359,6 +359,7 @@ var waIgnoreStatuses = map[string]bool{ type mtTextPayload struct { To string `json:"to" validate:"required"` Type string `json:"type" validate:"required"` + Preview_URL bool `json:"preview_url,omitempty"` Text struct { Body string `json:"body" validate:"required"` } `json:"text"` @@ -798,9 +799,20 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann } } else { for _, part := range parts { - payload := mtTextPayload{ - To: msg.URN().Path(), - Type: "text", + + //check if you have a link + var payload mtTextPayload + if strings.Contains(part, "https://") || strings.Contains(part, "http://") { + payload = mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + Preview_URL: true, + } + } else { + payload = mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + } } payload.Text.Body = part payloads = append(payloads, payload) diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index e125e0247..2dad38b5e 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -370,6 +370,12 @@ func setSendURL(s *httptest.Server, h courier.ChannelHandler, c courier.Channel, } var defaultSendTestCases = []ChannelSendTestCase{ + {Label: "Link Sending", + Text: "Link Sending https://link.com", URN: "whatsapp:250788123123", Path: "/v1/messages", + Status: "W", ExternalID: "157b5e14568e8", + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + RequestBody: `{"to":"250788123123","type":"text","preview_url":true,"text":{"body":"Link Sending https://link.com"}}`, + SendPrep: setSendURL}, {Label: "Plain Send", Text: "Simple Message", URN: "whatsapp:250788123123", Path: "/v1/messages", Status: "W", ExternalID: "157b5e14568e8", From c15c3d663d0c5614acfbf5c781cc8d13630cdd88 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 31 Mar 2022 11:10:33 -0300 Subject: [PATCH 096/146] Rename preview attribute --- handlers/whatsapp/whatsapp.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index f075805f2..d27144309 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -359,7 +359,7 @@ var waIgnoreStatuses = map[string]bool{ type mtTextPayload struct { To string `json:"to" validate:"required"` Type string `json:"type" validate:"required"` - Preview_URL bool `json:"preview_url,omitempty"` + PreviewURL bool `json:"preview_url,omitempty"` Text struct { Body string `json:"body" validate:"required"` } `json:"text"` @@ -806,7 +806,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann payload = mtTextPayload{ To: msg.URN().Path(), Type: "text", - Preview_URL: true, + PreviewURL: true, } } else { payload = mtTextPayload{ From 06c9728e1ae9964df0f0d98fbd372b87b5dcaa9a Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 5 Apr 2022 17:07:14 -0300 Subject: [PATCH 097/146] Add library with greater support for detection of mime types --- go.mod | 1 + go.sum | 2 ++ handlers/whatsapp/whatsapp.go | 10 +++++++++- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index c5a8de616..d8c857a5d 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/structs v1.0.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.0 // indirect github.com/golang/protobuf v1.3.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect diff --git a/go.sum b/go.sum index 85e0c7856..5f85e91d7 100644 --- a/go.sum +++ b/go.sum @@ -18,6 +18,8 @@ github.com/evalphobia/logrus_sentry v0.4.6 h1:825MLGu+SW5H8hMXGeBI7TwX7vgJLd9hz0 github.com/evalphobia/logrus_sentry v0.4.6/go.mod h1:pKcp+vriitUqu9KiWj/VRFbRfFNUwz95/UkgG8a6MNc= github.com/fatih/structs v1.0.0 h1:BrX964Rv5uQ3wwS+KRUAJCBBw5PQmgJfJ6v4yly5QwU= github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/gabriel-vasile/mimetype v1.4.0 h1:Cn9dkdYsMIu56tGho+fqzh7XmvY2YyGU0FnbhiOsEro= +github.com/gabriel-vasile/mimetype v1.4.0/go.mod h1:fA8fi6KUiG7MgQQ+mEWotXoEOvmxRtOJlERCzSmRvr8= github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 h1:YO10pIIBftO/kkTFdWhctH96grJ7qiy7bMdiZcIvPKs= github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index bdd568f2c..29f7a2df7 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -12,6 +12,7 @@ import ( "time" "github.com/buger/jsonparser" + "github.com/gabriel-vasile/mimetype" "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" @@ -863,7 +864,14 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri return "", logs, errors.Wrapf(err, "error building request to media endpoint") } setWhatsAppAuthHeader(&req.Header, msg.Channel()) - req.Header.Add("Content-Type", http.DetectContentType(rr.Body)) + mtype := http.DetectContentType(rr.Body) + + if mtype != mimeType || mtype == "application/octet-stream" || mtype == "application/zip" { + mimeT := mimetype.Detect(rr.Body) + req.Header.Add("Content-Type", mimeT.String()) + } else { + req.Header.Add("Content-Type", mtype) + } rr, err = utils.MakeHTTPRequest(req) log = courier.NewChannelLogFromRR("Uploading media to WhatsApp", msg.Channel(), msg.ID(), rr).WithError("Error uploading media to WhatsApp", err) logs = append(logs, log) From f21056c977a65d7fe1947434777d79935816880d Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 12 Apr 2022 10:17:10 -0300 Subject: [PATCH 098/146] Remove http library usage for mime types in Whatsapp --- handlers/whatsapp/whatsapp.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 29f7a2df7..f983a42ab 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -864,14 +864,8 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri return "", logs, errors.Wrapf(err, "error building request to media endpoint") } setWhatsAppAuthHeader(&req.Header, msg.Channel()) - mtype := http.DetectContentType(rr.Body) - - if mtype != mimeType || mtype == "application/octet-stream" || mtype == "application/zip" { - mimeT := mimetype.Detect(rr.Body) - req.Header.Add("Content-Type", mimeT.String()) - } else { - req.Header.Add("Content-Type", mtype) - } + mimeT := mimetype.Detect(rr.Body) + req.Header.Add("Content-Type", mimeT.String()) rr, err = utils.MakeHTTPRequest(req) log = courier.NewChannelLogFromRR("Uploading media to WhatsApp", msg.Channel(), msg.ID(), rr).WithError("Error uploading media to WhatsApp", err) logs = append(logs, log) From c33da6adccc90615fdce92ab8d35c9a912f2cb84 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 13 Apr 2022 11:21:49 -0300 Subject: [PATCH 099/146] Update to latest gocommon and start using httpx.DetectContentType --- channel_log.go | 4 ++-- go.mod | 2 +- go.sum | 3 +++ handlers/whatsapp/whatsapp.go | 5 ++--- 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/channel_log.go b/channel_log.go index d0c0b7237..8a3bba57a 100644 --- a/channel_log.go +++ b/channel_log.go @@ -2,11 +2,11 @@ package courier import ( "fmt" - "net/http" "strings" "time" "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/httpx" ) // NilStatusCode is used when we have an error before even sending anything @@ -42,7 +42,7 @@ func sanitizeBody(body string) string { return body } - ct := http.DetectContentType([]byte(parts[1])) + ct := httpx.DetectContentType([]byte(parts[1])) // if this isn't text, replace with placeholder if !strings.HasPrefix(ct, "text") { diff --git a/go.mod b/go.mod index d8c857a5d..02e8906de 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.17.1 + github.com/nyaruka/gocommon v1.18.0 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/nyaruka/redisx v0.2.1 diff --git a/go.sum b/go.sum index 5f85e91d7..bb9ebebab 100644 --- a/go.sum +++ b/go.sum @@ -72,6 +72,8 @@ github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDY github.com/nyaruka/gocommon v1.5.3/go.mod h1:2ZeBZF9yt20IaAJ4aC1ujojAsFhJBk2IuDvSl7KuQDw= github.com/nyaruka/gocommon v1.17.1 h1:4bbNp+0/BIbne4VDiKOxh3kcbdvEu/WsrsZiG/VyRZ8= github.com/nyaruka/gocommon v1.17.1/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0= +github.com/nyaruka/gocommon v1.18.0 h1:pRSV63w449FVHrjeXJNNIyoD0k47ruqPq3fROuHBjD0= +github.com/nyaruka/gocommon v1.18.0/go.mod h1:+jVWR2FB6XTqFz7fjvVIZMFwp2B6uoaCk7NVLGnfPFM= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= @@ -108,6 +110,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index f983a42ab..647e2e826 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -12,11 +12,11 @@ import ( "time" "github.com/buger/jsonparser" - "github.com/gabriel-vasile/mimetype" "github.com/gomodule/redigo/redis" "github.com/nyaruka/courier" "github.com/nyaruka/courier/handlers" "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/redisx" "github.com/patrickmn/go-cache" @@ -864,8 +864,7 @@ func (h *handler) fetchMediaID(msg courier.Msg, mimeType, mediaURL string) (stri return "", logs, errors.Wrapf(err, "error building request to media endpoint") } setWhatsAppAuthHeader(&req.Header, msg.Channel()) - mimeT := mimetype.Detect(rr.Body) - req.Header.Add("Content-Type", mimeT.String()) + req.Header.Add("Content-Type", httpx.DetectContentType(rr.Body)) rr, err = utils.MakeHTTPRequest(req) log = courier.NewChannelLogFromRR("Uploading media to WhatsApp", msg.Channel(), msg.ID(), rr).WithError("Error uploading media to WhatsApp", err) logs = append(logs, log) From 65a09d27e1ab8b1b5833ccba9d828c8975cd00ba Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 13 Apr 2022 19:31:19 +0200 Subject: [PATCH 100/146] Rename to use WAC and WhatsApp Cloud --- handlers/facebookapp/facebookapp.go | 154 +++++++++--------- handlers/facebookapp/facebookapp_test.go | 52 +++--- .../{cwa/audioCWA.json => wac/audioWAC.json} | 0 .../buttonCWA.json => wac/buttonWAC.json} | 0 .../documentCWA.json => wac/documentWAC.json} | 0 .../duplicateWAC.json} | 0 .../{cwa/helloCWA.json => wac/helloWAC.json} | 0 .../ignoreStatusWAC.json} | 0 .../{cwa/imageCWA.json => wac/imageWAC.json} | 0 .../testdata/{cwa => wac}/invalidFrom.json | 0 .../invalidStatusWAC.json} | 0 .../{cwa => wac}/invalidTimestamp.json | 0 .../locationCWA.json => wac/locationWAC.json} | 0 .../validStatusWAC.json} | 0 .../{cwa/videoCWA.json => wac/videoWAC.json} | 0 .../{cwa/voiceCWA.json => wac/voiceWAC.json} | 0 16 files changed, 103 insertions(+), 103 deletions(-) rename handlers/facebookapp/testdata/{cwa/audioCWA.json => wac/audioWAC.json} (100%) rename handlers/facebookapp/testdata/{cwa/buttonCWA.json => wac/buttonWAC.json} (100%) rename handlers/facebookapp/testdata/{cwa/documentCWA.json => wac/documentWAC.json} (100%) rename handlers/facebookapp/testdata/{cwa/duplicateCWA.json => wac/duplicateWAC.json} (100%) rename handlers/facebookapp/testdata/{cwa/helloCWA.json => wac/helloWAC.json} (100%) rename handlers/facebookapp/testdata/{cwa/ignoreStatusCWA.json => wac/ignoreStatusWAC.json} (100%) rename handlers/facebookapp/testdata/{cwa/imageCWA.json => wac/imageWAC.json} (100%) rename handlers/facebookapp/testdata/{cwa => wac}/invalidFrom.json (100%) rename handlers/facebookapp/testdata/{cwa/invalidStatusCWA.json => wac/invalidStatusWAC.json} (100%) rename handlers/facebookapp/testdata/{cwa => wac}/invalidTimestamp.json (100%) rename handlers/facebookapp/testdata/{cwa/locationCWA.json => wac/locationWAC.json} (100%) rename handlers/facebookapp/testdata/{cwa/validStatusCWA.json => wac/validStatusWAC.json} (100%) rename handlers/facebookapp/testdata/{cwa/videoCWA.json => wac/videoWAC.json} (100%) rename handlers/facebookapp/testdata/{cwa/voiceCWA.json => wac/voiceWAC.json} (100%) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index b81b4aba8..c5e0bf11b 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -29,7 +29,7 @@ var ( signatureHeader = "X-Hub-Signature" - configCWAPhoneNumberID = "cwa_phone_number_id" + configWACPhoneNumberID = "wac_phone_number_id" // max for the body maxMsgLength = 1000 @@ -77,7 +77,7 @@ func newHandler(channelType courier.ChannelType, name string, useUUIDRoutes bool func init() { courier.RegisterHandler(newHandler("IG", "Instagram", false)) courier.RegisterHandler(newHandler("FBA", "Facebook", false)) - courier.RegisterHandler(newHandler("CWA", "Cloud API WhatsApp", false)) + courier.RegisterHandler(newHandler("WAC", "WhatsApp Cloud", false)) } @@ -120,7 +120,7 @@ type User struct { // }] // } -type cwaMedia struct { +type wacMedia struct { Caption string `json:"caption"` Filename string `json:"filename"` ID string `json:"id"` @@ -160,11 +160,11 @@ type moPayload struct { Text struct { Body string `json:"body"` } `json:"text"` - Image *cwaMedia `json:"image"` - Audio *cwaMedia `json:"audio"` - Video *cwaMedia `json:"video"` - Document *cwaMedia `json:"document"` - Voice *cwaMedia `json:"voice"` + Image *wacMedia `json:"image"` + Audio *wacMedia `json:"audio"` + Video *wacMedia `json:"video"` + Document *wacMedia `json:"document"` + Voice *wacMedia `json:"voice"` Location *struct { Latitude float64 `json:"latitude"` Longitude float64 `json:"longitude"` @@ -294,9 +294,9 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan channelAddress = payload.Entry[0].Changes[0].Value.Metadata.DisplayPhoneNumber if channelAddress == "" { - return nil, fmt.Errorf("no channel adress found") + return nil, fmt.Errorf("no channel address found") } - return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("CWA"), courier.ChannelAddress(channelAddress)) + return h.Backend().GetChannelByAddress(ctx, courier.ChannelType("WAC"), courier.ChannelAddress(channelAddress)) } } @@ -790,7 +790,7 @@ type mtQuickReply struct { func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { if msg.Channel().ChannelType() == "FBA" || msg.Channel().ChannelType() == "IG" { return h.sendFacebookInstagramMsg(ctx, msg) - } else if msg.Channel().ChannelType() == "CWA" { + } else if msg.Channel().ChannelType() == "WAC" { return h.sendCloudAPIWhatsappMsg(ctx, msg) } @@ -943,25 +943,25 @@ func (h *handler) sendFacebookInstagramMsg(ctx context.Context, msg courier.Msg) return status, nil } -type cwaMTMedia struct { +type wacMTMedia struct { ID string `json:"id,omitempty"` Link string `json:"link,omitempty"` Caption string `json:"caption,omitempty"` Filename string `json:"filename,omitempty"` } -type cwaMTSection struct { +type wacMTSection struct { Title string `json:"title,omitempty"` - Rows []cwaMTSectionRow `json:"rows" validate:"required"` + Rows []wacMTSectionRow `json:"rows" validate:"required"` } -type cwaMTSectionRow struct { +type wacMTSectionRow struct { ID string `json:"id" validate:"required"` Title string `json:"title,omitempty"` Description string `json:"description,omitempty"` } -type cwaMTButton struct { +type wacMTButton struct { Type string `json:"type" validate:"required"` Reply struct { ID string `json:"id" validate:"required"` @@ -969,34 +969,34 @@ type cwaMTButton struct { } `json:"reply" validate:"required"` } -type cwaParam struct { +type wacParam struct { Type string `json:"type"` Text string `json:"text"` } -type cwaComponent struct { +type wacComponent struct { Type string `json:"type"` SubType string `json:"sub_type"` Index string `json:"index"` - Params []*cwaParam `json:"parameters"` + Params []*wacParam `json:"parameters"` } -type cwaText struct { +type wacText struct { Body string `json:"body"` } -type cwaLanguage struct { +type wacLanguage struct { Policy string `json:"policy"` Code string `json:"code"` } -type cwaTemplate struct { +type wacTemplate struct { Name string `json:"name"` - Language *cwaLanguage `json:"language"` - Components []*cwaComponent `json:"components"` + Language *wacLanguage `json:"language"` + Components []*wacComponent `json:"components"` } -type cwaInteractive struct { +type wacInteractive struct { Type string `json:"type"` Header *struct { Type string `json:"type"` @@ -1013,31 +1013,31 @@ type cwaInteractive struct { } `json:"footer,omitempty"` Action *struct { Button string `json:"button,omitempty"` - Sections []cwaMTSection `json:"sections,omitempty"` - Buttons []cwaMTButton `json:"buttons,omitempty"` + Sections []wacMTSection `json:"sections,omitempty"` + Buttons []wacMTButton `json:"buttons,omitempty"` } `json:"action,omitempty"` } -type cwaMTPayload struct { +type wacMTPayload struct { MessagingProduct string `json:"messaging_product"` PreviewURL bool `json:"preview_url"` RecipientType string `json:"recipient_type"` To string `json:"to"` Type string `json:"type"` - Text *cwaText `json:"text,omitempty"` + Text *wacText `json:"text,omitempty"` - Document *cwaMTMedia `json:"document,omitempty"` - Image *cwaMTMedia `json:"image,omitempty"` - Audio *cwaMTMedia `json:"audio,omitempty"` - Video *cwaMTMedia `json:"video,omitempty"` + Document *wacMTMedia `json:"document,omitempty"` + Image *wacMTMedia `json:"image,omitempty"` + Audio *wacMTMedia `json:"audio,omitempty"` + Video *wacMTMedia `json:"video,omitempty"` - Interactive *cwaInteractive `json:"interactive,omitempty"` + Interactive *wacInteractive `json:"interactive,omitempty"` - Template *cwaTemplate `json:"template,omitempty"` + Template *wacTemplate `json:"template,omitempty"` } -type cwaMTResponse struct { +type wacMTResponse struct { Messages []*struct { ID string `json:"id"` } `json:"messages"` @@ -1050,14 +1050,14 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) return nil, fmt.Errorf("missing access token") } - phoneNumberId := msg.Channel().StringConfigForKey(configCWAPhoneNumberID, "") + phoneNumberId := msg.Channel().StringConfigForKey(configWACPhoneNumberID, "") if phoneNumberId == "" { - return nil, fmt.Errorf("missing CWA phone number ID") + return nil, fmt.Errorf("missing WAC phone number ID") } base, _ := url.Parse(graphURL) path, _ := url.Parse(fmt.Sprintf("/%s/messages", phoneNumberId)) - cwaPhoneURL := base.ResolveReference(path) + wacPhoneURL := base.ResolveReference(path) status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) @@ -1068,7 +1068,7 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) qrs := msg.QuickReplies() for i := 0; i < len(msgParts)+len(msg.Attachments()); i++ { - payload := cwaMTPayload{MessagingProduct: "whatsapp", RecipientType: "individual", To: msg.URN().Path()} + payload := wacMTPayload{MessagingProduct: "whatsapp", RecipientType: "individual", To: msg.URN().Path()} if len(msg.Attachments()) == 0 { // do we have a template? @@ -1081,13 +1081,13 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) payload.Type = "template" - template := cwaTemplate{Name: templating.Template.Name, Language: &cwaLanguage{Policy: "deterministic", Code: templating.Language}} + template := wacTemplate{Name: templating.Template.Name, Language: &wacLanguage{Policy: "deterministic", Code: templating.Language}} payload.Template = &template - component := &cwaComponent{Type: "body"} + component := &wacComponent{Type: "body"} for _, v := range templating.Variables { - component.Params = append(component.Params, &cwaParam{Type: "text", Text: v}) + component.Params = append(component.Params, &wacParam{Type: "text", Text: v}) } template.Components = append(payload.Template.Components, component) @@ -1095,19 +1095,19 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) if i < (len(msgParts) + len(msg.Attachments()) - 1) { // this is still a msg part payload.Type = "text" - payload.Text = &cwaText{Body: msgParts[i-len(msg.Attachments())]} + payload.Text = &wacText{Body: msgParts[i-len(msg.Attachments())]} } else { if len(qrs) > 0 { payload.Type = "interactive" // We can use buttons if len(qrs) <= 3 { - interactive := cwaInteractive{Type: "button", Body: struct { + interactive := wacInteractive{Type: "button", Body: struct { Text string "json:\"text\"" }{Text: msgParts[i-len(msg.Attachments())]}} - btns := make([]cwaMTButton, len(qrs)) + btns := make([]wacMTButton, len(qrs)) for i, qr := range qrs { - btns[i] = cwaMTButton{ + btns[i] = wacMTButton{ Type: "reply", } btns[i].Reply.ID = fmt.Sprint(i) @@ -1115,20 +1115,20 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) } interactive.Action = &struct { Button string "json:\"button,omitempty\"" - Sections []cwaMTSection "json:\"sections,omitempty\"" - Buttons []cwaMTButton "json:\"buttons,omitempty\"" + Sections []wacMTSection "json:\"sections,omitempty\"" + Buttons []wacMTButton "json:\"buttons,omitempty\"" }{Buttons: btns} payload.Interactive = &interactive } else if len(qrs) <= 10 { - interactive := cwaInteractive{Type: "list", Body: struct { + interactive := wacInteractive{Type: "list", Body: struct { Text string "json:\"text\"" }{Text: msgParts[i-len(msg.Attachments())]}} - section := cwaMTSection{ - Rows: make([]cwaMTSectionRow, len(qrs)), + section := wacMTSection{ + Rows: make([]wacMTSectionRow, len(qrs)), } for i, qr := range qrs { - section.Rows[i] = cwaMTSectionRow{ + section.Rows[i] = wacMTSectionRow{ ID: fmt.Sprint(i), Title: qr, } @@ -1136,20 +1136,20 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) interactive.Action = &struct { Button string "json:\"button,omitempty\"" - Sections []cwaMTSection "json:\"sections,omitempty\"" - Buttons []cwaMTButton "json:\"buttons,omitempty\"" - }{Button: "Menu", Sections: []cwaMTSection{ + Sections []wacMTSection "json:\"sections,omitempty\"" + Buttons []wacMTButton "json:\"buttons,omitempty\"" + }{Button: "Menu", Sections: []wacMTSection{ section, }} payload.Interactive = &interactive } else { - return nil, fmt.Errorf("too many quick replies CWA supports only up to 10 quick replies") + return nil, fmt.Errorf("too many quick replies WAC supports only up to 10 quick replies") } } else { // this is still a msg part payload.Type = "text" - payload.Text = &cwaText{Body: msgParts[i-len(msg.Attachments())]} + payload.Text = &wacText{Body: msgParts[i-len(msg.Attachments())]} } } } @@ -1161,7 +1161,7 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) attType = "document" } payload.Type = attType - media := cwaMTMedia{Link: attURL} + media := wacMTMedia{Link: attURL} if attType == "image" { payload.Image = &media @@ -1176,19 +1176,19 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) if i < (len(msgParts) + len(msg.Attachments()) - 1) { // this is still a msg part payload.Type = "text" - payload.Text = &cwaText{Body: msgParts[i-len(msg.Attachments())]} + payload.Text = &wacText{Body: msgParts[i-len(msg.Attachments())]} } else { if len(qrs) > 0 { payload.Type = "interactive" // We can use buttons if len(qrs) <= 3 { - interactive := cwaInteractive{Type: "button", Body: struct { + interactive := wacInteractive{Type: "button", Body: struct { Text string "json:\"text\"" }{Text: msgParts[i-len(msg.Attachments())]}} - btns := make([]cwaMTButton, len(qrs)) + btns := make([]wacMTButton, len(qrs)) for i, qr := range qrs { - btns[i] = cwaMTButton{ + btns[i] = wacMTButton{ Type: "reply", } btns[i].Reply.ID = fmt.Sprint(i) @@ -1196,21 +1196,21 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) } interactive.Action = &struct { Button string "json:\"button,omitempty\"" - Sections []cwaMTSection "json:\"sections,omitempty\"" - Buttons []cwaMTButton "json:\"buttons,omitempty\"" + Sections []wacMTSection "json:\"sections,omitempty\"" + Buttons []wacMTButton "json:\"buttons,omitempty\"" }{Buttons: btns} payload.Interactive = &interactive } else if len(qrs) <= 10 { - interactive := cwaInteractive{Type: "list", Body: struct { + interactive := wacInteractive{Type: "list", Body: struct { Text string "json:\"text\"" }{Text: msgParts[i-len(msg.Attachments())]}} - section := cwaMTSection{ - Rows: make([]cwaMTSectionRow, len(qrs)), + section := wacMTSection{ + Rows: make([]wacMTSectionRow, len(qrs)), } for i, qr := range qrs { - section.Rows[i] = cwaMTSectionRow{ + section.Rows[i] = wacMTSectionRow{ ID: fmt.Sprint(i), Title: qr, } @@ -1218,20 +1218,20 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) interactive.Action = &struct { Button string "json:\"button,omitempty\"" - Sections []cwaMTSection "json:\"sections,omitempty\"" - Buttons []cwaMTButton "json:\"buttons,omitempty\"" - }{Button: "Menu", Sections: []cwaMTSection{ + Sections []wacMTSection "json:\"sections,omitempty\"" + Buttons []wacMTButton "json:\"buttons,omitempty\"" + }{Button: "Menu", Sections: []wacMTSection{ section, }} payload.Interactive = &interactive } else { - return nil, fmt.Errorf("too many quick replies CWA supports only up to 10 quick replies") + return nil, fmt.Errorf("too many quick replies WAC supports only up to 10 quick replies") } } else { // this is still a msg part payload.Type = "text" - payload.Text = &cwaText{Body: msgParts[i-len(msg.Attachments())]} + payload.Text = &wacText{Body: msgParts[i-len(msg.Attachments())]} } } @@ -1242,7 +1242,7 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) return status, err } - req, err := http.NewRequest(http.MethodPost, cwaPhoneURL.String(), bytes.NewReader(jsonBody)) + req, err := http.NewRequest(http.MethodPost, wacPhoneURL.String(), bytes.NewReader(jsonBody)) if err != nil { return nil, err } @@ -1259,7 +1259,7 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) return status, nil } - respPayload := &cwaMTResponse{} + respPayload := &wacMTResponse{} err = json.Unmarshal(rr.Body, respPayload) if err != nil { log.WithError("Message Send Error", errors.Errorf("unable to unmarshal response body")) @@ -1278,7 +1278,7 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) // DescribeURN looks up URN metadata for new contacts func (h *handler) DescribeURN(ctx context.Context, channel courier.Channel, urn urns.URN) (map[string]string, error) { - if channel.ChannelType() == "CWA" { + if channel.ChannelType() == "WAC" { return map[string]string{}, nil } diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index c5a526ebe..506d0dfd8 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -25,8 +25,8 @@ var testChannelsIG = []courier.Channel{ courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), } -var testChannelsCWA = []courier.Channel{ - courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "CWA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), +var testChannelsWAC = []courier.Channel{ + courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c568c", "WAC", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}), } var testCasesFBA = []ChannelHandleTestCase{ @@ -218,8 +218,8 @@ func TestDescribeIG(t *testing.T) { } } -func TestDescribeCWA(t *testing.T) { - handler := newHandler("CWA", "Cloud API WhatsApp", false).(courier.URNDescriber) +func TestDescribeWAC(t *testing.T) { + handler := newHandler("WAC", "Cloud API WhatsApp", false).(courier.URNDescriber) tcs := []struct { urn urns.URN @@ -228,53 +228,53 @@ func TestDescribeCWA(t *testing.T) { {"whatsapp:4567", map[string]string{}}} for _, tc := range tcs { - metadata, _ := handler.DescribeURN(context.Background(), testChannelsCWA[0], tc.urn) + metadata, _ := handler.DescribeURN(context.Background(), testChannelsWAC[0], tc.urn) assert.Equal(t, metadata, tc.metadata) } } -var cwaReceiveURL = "/c/cwa/receive" +var wacReceiveURL = "/c/wac/receive" -var testCasesCWA = []ChannelHandleTestCase{ - {Label: "Receive Message CWA", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/helloCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, +var testCasesWAC = []ChannelHandleTestCase{ + {Label: "Receive Message WAC", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/helloWAC.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Hello World"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Duplicate Valid Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/duplicateCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + {Label: "Receive Duplicate Valid Message", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/duplicateWAC.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Hello World"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Valid Voice Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/voiceCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + {Label: "Receive Valid Voice Message", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/voiceWAC.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp(""), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Attachment: Sp("https://foo.bar/attachmentURL_Voice"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Valid Button Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/buttonCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + {Label: "Receive Valid Button Message", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/buttonWAC.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("No"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Valid Document Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/documentCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + {Label: "Receive Valid Document Message", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/documentWAC.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("80skaraokesonglistartist"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Attachment: Sp("https://foo.bar/attachmentURL_Document"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Valid Image Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/imageCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + {Label: "Receive Valid Image Message", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/imageWAC.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Check out my new phone!"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Attachment: Sp("https://foo.bar/attachmentURL_Image"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Valid Video Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/videoCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + {Label: "Receive Valid Video Message", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/videoWAC.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Check out my new phone!"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Attachment: Sp("https://foo.bar/attachmentURL_Video"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Valid Audio Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/audioCWA.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + {Label: "Receive Valid Audio Message", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/audioWAC.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Check out my new phone!"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Attachment: Sp("https://foo.bar/attachmentURL_Audio"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Valid Location Message", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/locationCWA.json")), Status: 200, Response: `"type":"msg"`, + {Label: "Receive Valid Location Message", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/locationWAC.json")), Status: 200, Response: `"type":"msg"`, Text: Sp(""), Attachment: Sp("geo:0.000000,1.000000"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), PrepRequest: addValidSignature}, - {Label: "Receive Invalid JSON", URL: cwaReceiveURL, Data: "not json", Status: 400, Response: "unable to parse", PrepRequest: addValidSignature}, - {Label: "Receive Invalid JSON", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/invalidFrom.json")), Status: 400, Response: "invalid whatsapp id", PrepRequest: addValidSignature}, - {Label: "Receive Invalid JSON", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/invalidTimestamp.json")), Status: 400, Response: "invalid timestamp", PrepRequest: addValidSignature}, + {Label: "Receive Invalid JSON", URL: wacReceiveURL, Data: "not json", Status: 400, Response: "unable to parse", PrepRequest: addValidSignature}, + {Label: "Receive Invalid JSON", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/invalidFrom.json")), Status: 400, Response: "invalid whatsapp id", PrepRequest: addValidSignature}, + {Label: "Receive Invalid JSON", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/invalidTimestamp.json")), Status: 400, Response: "invalid timestamp", PrepRequest: addValidSignature}, - {Label: "Receive Valid Status", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/validStatusCWA.json")), Status: 200, Response: `"type":"status"`, + {Label: "Receive Valid Status", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/validStatusWAC.json")), Status: 200, Response: `"type":"status"`, MsgStatus: Sp("S"), ExternalID: Sp("external_id"), PrepRequest: addValidSignature}, - {Label: "Receive Invalid Status", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/invalidStatusCWA.json")), Status: 400, Response: `"unknown status: in_orbit"`, PrepRequest: addValidSignature}, - {Label: "Receive Ignore Status", URL: cwaReceiveURL, Data: string(courier.ReadFile("./testdata/cwa/ignoreStatusCWA.json")), Status: 200, Response: `"ignoring status: deleted"`, PrepRequest: addValidSignature}, + {Label: "Receive Invalid Status", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/invalidStatusWAC.json")), Status: 400, Response: `"unknown status: in_orbit"`, PrepRequest: addValidSignature}, + {Label: "Receive Ignore Status", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/ignoreStatusWAC.json")), Status: 200, Response: `"ignoring status: deleted"`, PrepRequest: addValidSignature}, } func TestHandler(t *testing.T) { @@ -320,7 +320,7 @@ func TestHandler(t *testing.T) { })) graphURL = server.URL - RunChannelTestCases(t, testChannelsCWA, newHandler("CWA", "Cloud API WhatsApp", false), testCasesCWA) + RunChannelTestCases(t, testChannelsWAC, newHandler("WAC", "Cloud API WhatsApp", false), testCasesWAC) RunChannelTestCases(t, testChannelsFBA, newHandler("FBA", "Facebook", false), testCasesFBA) RunChannelTestCases(t, testChannelsIG, newHandler("IG", "Instagram", false), testCasesIG) } @@ -494,7 +494,7 @@ var SendTestCasesIG = []ChannelSendTestCase{ SendPrep: setSendURL}, } -var SendTestCasesCWA = []ChannelSendTestCase{ +var SendTestCasesWAC = []ChannelSendTestCase{ {Label: "Plain Send", Text: "Simple Message", URN: "whatsapp:250788123123", Path: "/12345_ID/messages", Status: "W", ExternalID: "157b5e14568e8", @@ -694,10 +694,10 @@ func TestSending(t *testing.T) { maxMsgLength = 100 var ChannelFBA = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) var ChannelIG = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) - var ChannelCWA = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "CWA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123", configCWAPhoneNumberID: "12345_ID"}) + var ChannelWAC = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "WAC", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123", configWACPhoneNumberID: "12345_ID"}) RunChannelSendTestCases(t, ChannelFBA, newHandler("FBA", "Facebook", false), SendTestCasesFBA, nil) RunChannelSendTestCases(t, ChannelIG, newHandler("IG", "Instagram", false), SendTestCasesIG, nil) - RunChannelSendTestCases(t, ChannelCWA, newHandler("CWA", "Cloud API WhatsApp", false), SendTestCasesCWA, nil) + RunChannelSendTestCases(t, ChannelWAC, newHandler("WAC", "Cloud API WhatsApp", false), SendTestCasesWAC, nil) } func TestSigning(t *testing.T) { diff --git a/handlers/facebookapp/testdata/cwa/audioCWA.json b/handlers/facebookapp/testdata/wac/audioWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/audioCWA.json rename to handlers/facebookapp/testdata/wac/audioWAC.json diff --git a/handlers/facebookapp/testdata/cwa/buttonCWA.json b/handlers/facebookapp/testdata/wac/buttonWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/buttonCWA.json rename to handlers/facebookapp/testdata/wac/buttonWAC.json diff --git a/handlers/facebookapp/testdata/cwa/documentCWA.json b/handlers/facebookapp/testdata/wac/documentWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/documentCWA.json rename to handlers/facebookapp/testdata/wac/documentWAC.json diff --git a/handlers/facebookapp/testdata/cwa/duplicateCWA.json b/handlers/facebookapp/testdata/wac/duplicateWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/duplicateCWA.json rename to handlers/facebookapp/testdata/wac/duplicateWAC.json diff --git a/handlers/facebookapp/testdata/cwa/helloCWA.json b/handlers/facebookapp/testdata/wac/helloWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/helloCWA.json rename to handlers/facebookapp/testdata/wac/helloWAC.json diff --git a/handlers/facebookapp/testdata/cwa/ignoreStatusCWA.json b/handlers/facebookapp/testdata/wac/ignoreStatusWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/ignoreStatusCWA.json rename to handlers/facebookapp/testdata/wac/ignoreStatusWAC.json diff --git a/handlers/facebookapp/testdata/cwa/imageCWA.json b/handlers/facebookapp/testdata/wac/imageWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/imageCWA.json rename to handlers/facebookapp/testdata/wac/imageWAC.json diff --git a/handlers/facebookapp/testdata/cwa/invalidFrom.json b/handlers/facebookapp/testdata/wac/invalidFrom.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/invalidFrom.json rename to handlers/facebookapp/testdata/wac/invalidFrom.json diff --git a/handlers/facebookapp/testdata/cwa/invalidStatusCWA.json b/handlers/facebookapp/testdata/wac/invalidStatusWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/invalidStatusCWA.json rename to handlers/facebookapp/testdata/wac/invalidStatusWAC.json diff --git a/handlers/facebookapp/testdata/cwa/invalidTimestamp.json b/handlers/facebookapp/testdata/wac/invalidTimestamp.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/invalidTimestamp.json rename to handlers/facebookapp/testdata/wac/invalidTimestamp.json diff --git a/handlers/facebookapp/testdata/cwa/locationCWA.json b/handlers/facebookapp/testdata/wac/locationWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/locationCWA.json rename to handlers/facebookapp/testdata/wac/locationWAC.json diff --git a/handlers/facebookapp/testdata/cwa/validStatusCWA.json b/handlers/facebookapp/testdata/wac/validStatusWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/validStatusCWA.json rename to handlers/facebookapp/testdata/wac/validStatusWAC.json diff --git a/handlers/facebookapp/testdata/cwa/videoCWA.json b/handlers/facebookapp/testdata/wac/videoWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/videoCWA.json rename to handlers/facebookapp/testdata/wac/videoWAC.json diff --git a/handlers/facebookapp/testdata/cwa/voiceCWA.json b/handlers/facebookapp/testdata/wac/voiceWAC.json similarity index 100% rename from handlers/facebookapp/testdata/cwa/voiceCWA.json rename to handlers/facebookapp/testdata/wac/voiceWAC.json From 4be945fb5fc031bfa511c3e03807b3a04f09b52c Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 14 Apr 2022 10:19:23 -0500 Subject: [PATCH 101/146] Update CHANGELOG.md for v7.3.2 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e71877ff7..366bdd427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +v7.3.2 +---------- + * Update to latest gocommon and start using httpx.DetectContentType + * Add link preview attribute for sending whatsapp + * Update golang.org/x/sys + v7.3.1 ---------- * Fix handling stops via status callbacks on Twilio From 0692235884e7c59519c743c797070806a8168eb7 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 14 Apr 2022 10:38:44 -0500 Subject: [PATCH 102/146] Update to codecov action v2 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d713fc6e..5cfff763b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,7 +37,7 @@ jobs: - name: Upload coverage if: success() - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v2 with: fail_ci_if_error: true From 8e1e8f77772dd3792f843edeb0f33b01027ebebd Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 20 Apr 2022 16:29:18 +0200 Subject: [PATCH 103/146] Quick fix to allow JSON content to not be omitted in logs --- channel_log.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/channel_log.go b/channel_log.go index 8a3bba57a..67c1cc3c0 100644 --- a/channel_log.go +++ b/channel_log.go @@ -45,7 +45,7 @@ func sanitizeBody(body string) string { ct := httpx.DetectContentType([]byte(parts[1])) // if this isn't text, replace with placeholder - if !strings.HasPrefix(ct, "text") { + if !strings.HasPrefix(ct, "text") && !strings.HasPrefix(ct, "application/json") { return fmt.Sprintf("%s\r\n\r\nOmitting non text body of type: %s", parts[0], ct) } From 5834bc2fb9789326d201b25ec373ab9e599bfb02 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 20 Apr 2022 09:42:18 -0500 Subject: [PATCH 104/146] Update CHANGELOG.md for v7.3.3 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 366bdd427..56f2e39e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.3.3 +---------- + * Quick fix to stop JSON content being omitted in logs + v7.3.2 ---------- * Update to latest gocommon and start using httpx.DetectContentType From 5ac9dc76b7a6d4dd0a315a8c392c44f8a6bbccb2 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 20 Apr 2022 18:08:59 +0200 Subject: [PATCH 105/146] Stop Telegram contact that have blocked the channel bot --- handlers/telegram/telegram.go | 77 +++++++++++++++++++++++------- handlers/telegram/telegram_test.go | 8 ++++ 2 files changed, 68 insertions(+), 17 deletions(-) diff --git a/handlers/telegram/telegram.go b/handlers/telegram/telegram.go index 3be0d3889..71ed80349 100644 --- a/handlers/telegram/telegram.go +++ b/handlers/telegram/telegram.go @@ -2,6 +2,7 @@ package telegram import ( "context" + "encoding/json" "fmt" "net/http" "net/url" @@ -131,7 +132,16 @@ func (h *handler) receiveMessage(ctx context.Context, channel courier.Channel, w return handlers.WriteMsgsAndResponse(ctx, h, []courier.Msg{msg}, w, r) } -func (h *handler) sendMsgPart(msg courier.Msg, token string, path string, form url.Values, keyboard *ReplyKeyboardMarkup) (string, *courier.ChannelLog, error) { +type mtResponse struct { + Ok bool `json:"ok" validate:"required"` + ErrorCode int `json:"error_code"` + Description string `json:"description"` + Result struct { + MessageID int64 `json:"message_id"` + } `json:"result"` +} + +func (h *handler) sendMsgPart(msg courier.Msg, token string, path string, form url.Values, keyboard *ReplyKeyboardMarkup) (string, *courier.ChannelLog, bool, error) { // either include or remove our keyboard if keyboard == nil { form.Add("reply_markup", `{"remove_keyboard":true}`) @@ -142,7 +152,7 @@ func (h *handler) sendMsgPart(msg courier.Msg, token string, path string, form u sendURL := fmt.Sprintf("%s/bot%s/%s", apiURL, token, path) req, err := http.NewRequest(http.MethodPost, sendURL, strings.NewReader(form.Encode())) if err != nil { - return "", nil, err + return "", nil, false, err } req.Header.Add("Content-Type", "application/x-www-form-urlencoded") @@ -151,19 +161,22 @@ func (h *handler) sendMsgPart(msg courier.Msg, token string, path string, form u // build our channel log log := courier.NewChannelLogFromRR("Message Sent", msg.Channel(), msg.ID(), rr).WithError("Message Send Error", err) - // was this request successful? - ok, err := jsonparser.GetBoolean([]byte(rr.Body), "ok") - if err != nil || !ok { - return "", log, errors.Errorf("response not 'ok'") - } + response := &mtResponse{} + err = json.Unmarshal(rr.Body, response) + + if err != nil || !response.Ok { + if response.ErrorCode == 403 && response.Description == "Forbidden: bot was blocked by the user" { + return "", log, true, errors.Errorf("response not 'ok'") + + } + return "", log, false, errors.Errorf("response not 'ok'") - // grab our message id - externalID, err := jsonparser.GetInt([]byte(rr.Body), "result", "message_id") - if err != nil { - return "", log, errors.Errorf("no 'result.message_id' in response") } - return strconv.FormatInt(externalID, 10), log, nil + if response.Result.MessageID > 0 { + return strconv.FormatInt(response.Result.MessageID, 10), log, false, nil + } + return "", log, true, errors.Errorf("no 'result.message_id' in response") } // SendMsg sends the passed in message, returning any error @@ -205,7 +218,13 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "text": []string{msg.Text()}, } - externalID, log, err := h.sendMsgPart(msg, authToken, "sendMessage", form, msgKeyBoard) + externalID, log, botBlocked, err := h.sendMsgPart(msg, authToken, "sendMessage", form, msgKeyBoard) + if botBlocked { + status.SetStatus(courier.MsgFailed) + channelEvent := h.Backend().NewChannelEvent(msg.Channel(), courier.StopContact, msg.URN()) + err = h.Backend().WriteChannelEvent(ctx, channelEvent) + return status, err + } status.SetExternalID(externalID) hasError = err != nil status.AddLog(log) @@ -227,7 +246,13 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "photo": []string{mediaURL}, "caption": []string{caption}, } - externalID, log, err := h.sendMsgPart(msg, authToken, "sendPhoto", form, attachmentKeyBoard) + externalID, log, botBlocked, err := h.sendMsgPart(msg, authToken, "sendPhoto", form, attachmentKeyBoard) + if botBlocked { + status.SetStatus(courier.MsgFailed) + channelEvent := h.Backend().NewChannelEvent(msg.Channel(), courier.StopContact, msg.URN()) + err = h.Backend().WriteChannelEvent(ctx, channelEvent) + return status, err + } status.SetExternalID(externalID) hasError = err != nil status.AddLog(log) @@ -238,7 +263,13 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "video": []string{mediaURL}, "caption": []string{caption}, } - externalID, log, err := h.sendMsgPart(msg, authToken, "sendVideo", form, attachmentKeyBoard) + externalID, log, botBlocked, err := h.sendMsgPart(msg, authToken, "sendVideo", form, attachmentKeyBoard) + if botBlocked { + status.SetStatus(courier.MsgFailed) + channelEvent := h.Backend().NewChannelEvent(msg.Channel(), courier.StopContact, msg.URN()) + err = h.Backend().WriteChannelEvent(ctx, channelEvent) + return status, err + } status.SetExternalID(externalID) hasError = err != nil status.AddLog(log) @@ -249,7 +280,13 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "audio": []string{mediaURL}, "caption": []string{caption}, } - externalID, log, err := h.sendMsgPart(msg, authToken, "sendAudio", form, attachmentKeyBoard) + externalID, log, botBlocked, err := h.sendMsgPart(msg, authToken, "sendAudio", form, attachmentKeyBoard) + if botBlocked { + status.SetStatus(courier.MsgFailed) + channelEvent := h.Backend().NewChannelEvent(msg.Channel(), courier.StopContact, msg.URN()) + err = h.Backend().WriteChannelEvent(ctx, channelEvent) + return status, err + } status.SetExternalID(externalID) hasError = err != nil status.AddLog(log) @@ -260,7 +297,13 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "document": []string{mediaURL}, "caption": []string{caption}, } - externalID, log, err := h.sendMsgPart(msg, authToken, "sendDocument", form, attachmentKeyBoard) + externalID, log, botBlocked, err := h.sendMsgPart(msg, authToken, "sendDocument", form, attachmentKeyBoard) + if botBlocked { + status.SetStatus(courier.MsgFailed) + channelEvent := h.Backend().NewChannelEvent(msg.Channel(), courier.StopContact, msg.URN()) + err = h.Backend().WriteChannelEvent(ctx, channelEvent) + return status, err + } status.SetExternalID(externalID) hasError = err != nil status.AddLog(log) diff --git a/handlers/telegram/telegram_test.go b/handlers/telegram/telegram_test.go index f992541f5..d8c1c343d 100644 --- a/handlers/telegram/telegram_test.go +++ b/handlers/telegram/telegram_test.go @@ -608,6 +608,14 @@ var defaultSendTestCases = []ChannelSendTestCase{ ResponseBody: `{ "ok": false }`, ResponseStatus: 403, PostParams: map[string]string{"text": `Error`, "chat_id": "12345"}, SendPrep: setSendURL}, + {Label: "Stopped Contact Code", + Text: "Stopped Contact", URN: "telegram:12345", + Status: "F", + ResponseBody: `{ "ok": false, "error_code":403, "description":"Forbidden: bot was blocked by the user"}`, ResponseStatus: 403, + PostParams: map[string]string{"text": `Stopped Contact`, "chat_id": "12345"}, + SendPrep: setSendURL, + Stopped: true}, + {Label: "Send Photo", Text: "My pic!", URN: "telegram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, Status: "W", From b0732ef42225595617f0e8070e454247ff1b40b6 Mon Sep 17 00:00:00 2001 From: Rafael Soares Date: Wed, 20 Apr 2022 19:59:39 -0300 Subject: [PATCH 106/146] tweak in whatsapp handler on sendWhatsAppMsg returned wppID --- handlers/test.go | 6 ++++ handlers/whatsapp/whatsapp.go | 21 ++++++++------ handlers/whatsapp/whatsapp_test.go | 45 +++++++++++++++++------------- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/handlers/test.go b/handlers/test.go index 9d245c0d2..03550f68e 100644 --- a/handlers/test.go +++ b/handlers/test.go @@ -110,6 +110,7 @@ type ChannelSendTestCase struct { ContactURNs map[string]bool SendPrep SendPrepFunc + NewURN string } // Sp is a utility method to get the pointer to the passed in string @@ -349,6 +350,11 @@ func RunChannelSendTestCases(t *testing.T, channel courier.Channel, handler cour } } + if testCase.NewURN != "" { + old, new := status.UpdatedURN() + require.Equal(urns.URN(testCase.URN), old) + require.Equal(urns.URN(testCase.NewURN), new) + } }) } diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 7e2cf4ec5..563f03913 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -358,10 +358,10 @@ var waIgnoreStatuses = map[string]bool{ // } type mtTextPayload struct { - To string `json:"to" validate:"required"` - Type string `json:"type" validate:"required"` - PreviewURL bool `json:"preview_url,omitempty"` - Text struct { + To string `json:"to" validate:"required"` + Type string `json:"type" validate:"required"` + PreviewURL bool `json:"preview_url,omitempty"` + Text struct { Body string `json:"body" validate:"required"` } `json:"text"` } @@ -548,6 +548,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat // we are wired it there were no errors if err == nil { + // so update contact URN if wppID != "" if wppID != "" { newURN, _ := urns.NewWhatsAppURN(wppID) err = status.SetUpdatedURN(msg.URN(), newURN) @@ -805,14 +806,14 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann var payload mtTextPayload if strings.Contains(part, "https://") || strings.Contains(part, "http://") { payload = mtTextPayload{ - To: msg.URN().Path(), - Type: "text", + To: msg.URN().Path(), + Type: "text", PreviewURL: true, } } else { payload = mtTextPayload{ - To: msg.URN().Path(), - Type: "text", + To: msg.URN().Path(), + Type: "text", } } payload.Text.Body = part @@ -1025,6 +1026,10 @@ func sendWhatsAppMsg(rc redis.Conn, msg courier.Msg, sendPath *url.URL, payload return wppID, externalID, []*courier.ChannelLog{log, checkLog, retryLog}, err } externalID, err := getSendWhatsAppMsgId(rr) + wppID, err := jsonparser.GetString(rr.Body, "contacts", "[0]", "wa_id") + if wppID != msg.URN().Path() { + return wppID, externalID, []*courier.ChannelLog{log}, err + } return "", externalID, []*courier.ChannelLog{log}, err } diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 2dad38b5e..27b24c92e 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -373,19 +373,19 @@ var defaultSendTestCases = []ChannelSendTestCase{ {Label: "Link Sending", Text: "Link Sending https://link.com", URN: "whatsapp:250788123123", Path: "/v1/messages", Status: "W", ExternalID: "157b5e14568e8", - ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"text","preview_url":true,"text":{"body":"Link Sending https://link.com"}}`, SendPrep: setSendURL}, {Label: "Plain Send", Text: "Simple Message", URN: "whatsapp:250788123123", Path: "/v1/messages", Status: "W", ExternalID: "157b5e14568e8", - ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"text","text":{"body":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Unicode Send", Text: "☺", URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", - ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"text","text":{"body":"☺"}}`, SendPrep: setSendURL}, {Label: "Error", @@ -424,7 +424,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"audio","audio":{"link":"https://foo.bar/audio.mp3"}}`, }: MockedResponse{ Status: 201, - Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -441,7 +441,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"document","document":{"link":"https://foo.bar/document.pdf","caption":"document caption","filename":"document.pdf"}}`, }: MockedResponse{ Status: 201, - Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -458,7 +458,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"image","image":{"link":"https://foo.bar/image.jpg","caption":"document caption"}}`, }: MockedResponse{ Status: 201, - Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -475,7 +475,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"video","video":{"link":"https://foo.bar/video.mp4","caption":"video caption"}}`, }: MockedResponse{ Status: 201, - Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -485,7 +485,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", Metadata: json.RawMessage(`{ "templating": { "template": { "name": "revive_issue", "uuid": "171f8a4d-f725-46d7-85a6-11aceff0bfe3" }, "language": "eng", "variables": ["Chef", "tomorrow"]}}`), - ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, + ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, RequestBody: `{"to":"250788123123","type":"template","template":{"namespace":"waba_namespace","name":"revive_issue","language":{"policy":"deterministic","code":"en"},"components":[{"type":"body","parameters":[{"type":"text","text":"Chef"},{"type":"text","text":"tomorrow"}]}]}}`, SendPrep: setSendURL, }, @@ -494,7 +494,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", Metadata: json.RawMessage(`{ "templating": { "template": { "name": "revive_issue", "uuid": "171f8a4d-f725-46d7-85a6-11aceff0bfe3" }, "language": "eng", "country": "US", "variables": ["Chef", "tomorrow"]}}`), - ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, + ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, RequestBody: `{"to":"250788123123","type":"template","template":{"namespace":"waba_namespace","name":"revive_issue","language":{"policy":"deterministic","code":"en_US"},"components":[{"type":"body","parameters":[{"type":"text","text":"Chef"},{"type":"text","text":"tomorrow"}]}]}}`, SendPrep: setSendURL, }, @@ -503,7 +503,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", Metadata: json.RawMessage(`{ "templating": { "template": { "name": "revive_issue", "uuid": "171f8a4d-f725-46d7-85a6-11aceff0bfe3" }, "namespace": "wa_template_namespace", "language": "eng", "country": "US", "variables": ["Chef", "tomorrow"]}}`), - ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, + ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, RequestBody: `{"to":"250788123123","type":"template","template":{"namespace":"wa_template_namespace","name":"revive_issue","language":{"policy":"deterministic","code":"en_US"},"components":[{"type":"body","parameters":[{"type":"text","text":"Chef"},{"type":"text","text":"tomorrow"}]}]}}`, SendPrep: setSendURL, }, @@ -634,13 +634,13 @@ var defaultSendTestCases = []ChannelSendTestCase{ {Label: "Interactive Button Message Send", Text: "Interactive Button Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"BUTTON1"}, Status: "W", ExternalID: "157b5e14568e8", - ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"interactive","interactive":{"type":"button","body":{"text":"Interactive Button Msg"},"action":{"buttons":[{"type":"reply","reply":{"id":"0","title":"BUTTON1"}}]}}}`, SendPrep: setSendURL}, {Label: "Interactive List Message Send", Text: "Interactive List Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"ROW1", "ROW2", "ROW3", "ROW4"}, Status: "W", ExternalID: "157b5e14568e8", - ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"interactive","interactive":{"type":"list","body":{"text":"Interactive List Msg"},"action":{"button":"Menu","sections":[{"rows":[{"id":"0","title":"ROW1"},{"id":"1","title":"ROW2"},{"id":"2","title":"ROW3"},{"id":"3","title":"ROW4"}]}]}}}`, SendPrep: setSendURL}, {Label: "Interactive Button Message Send with attachment", @@ -689,6 +689,13 @@ var defaultSendTestCases = []ChannelSendTestCase{ }, }, SendPrep: setSendURL}, + {Label: "Update URN with wa_id returned", + Text: "Simple Message", URN: "whatsapp:5511987654321", Path: "/v1/messages", + Status: "W", ExternalID: "157b5e14568e8", + ResponseBody: `{ "contacts":[{"input":"5511987654321","wa_id":"551187654321"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + RequestBody: `{"to":"5511987654321","type":"text","text":{"body":"Simple Message"}}`, + SendPrep: setSendURL, + NewURN: "whatsapp:551187654321"}, } var mediaCacheSendTestCases = []ChannelSendTestCase{ @@ -712,7 +719,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ BodyContains: `/document.pdf`, }: MockedResponse{ Status: 201, - Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -729,7 +736,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ BodyContains: `/document.pdf`, }: MockedResponse{ Status: 201, - Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -754,7 +761,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"video","video":{"id":"36c484d1-1283-4b94-988d-7276bdec4de2","caption":"video caption"}}`, }: MockedResponse{ Status: 201, - Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -771,7 +778,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"video","video":{"id":"36c484d1-1283-4b94-988d-7276bdec4de2","caption":"video caption"}}`, }: MockedResponse{ Status: 201, - Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -797,7 +804,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"document","document":{"id":"25c484d1-1283-4b94-988d-7276bdec4ef3","caption":"document caption","filename":"document2.pdf"}}`, }: MockedResponse{ Status: 201, - Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -814,7 +821,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"document","document":{"id":"25c484d1-1283-4b94-988d-7276bdec4ef3","caption":"document caption","filename":"document2.pdf"}}`, }: MockedResponse{ Status: 201, - Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -827,7 +834,7 @@ var hsmSupportSendTestCases = []ChannelSendTestCase{ URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", Metadata: json.RawMessage(`{ "templating": { "template": { "name": "revive_issue", "uuid": "171f8a4d-f725-46d7-85a6-11aceff0bfe3" }, "language": "eng", "variables": ["Chef", "tomorrow"]}}`), - ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, + ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, RequestBody: `{"to":"250788123123","type":"hsm","hsm":{"namespace":"waba_namespace","element_name":"revive_issue","language":{"policy":"deterministic","code":"en"},"localizable_params":[{"default":"Chef"},{"default":"tomorrow"}]}}`, SendPrep: setSendURL, }, From b6ec19f64136be44ff99bb915b19d8167bba3e15 Mon Sep 17 00:00:00 2001 From: Rafael Soares Date: Fri, 22 Apr 2022 15:04:36 -0300 Subject: [PATCH 107/146] fix error handling in get externalID on sendWhatsAppMsg --- handlers/whatsapp/whatsapp.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 563f03913..8973bd820 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -1026,11 +1026,14 @@ func sendWhatsAppMsg(rc redis.Conn, msg courier.Msg, sendPath *url.URL, payload return wppID, externalID, []*courier.ChannelLog{log, checkLog, retryLog}, err } externalID, err := getSendWhatsAppMsgId(rr) + if err != nil { + return "", "", []*courier.ChannelLog{log}, err + } wppID, err := jsonparser.GetString(rr.Body, "contacts", "[0]", "wa_id") - if wppID != msg.URN().Path() { + if wppID != "" && wppID != msg.URN().Path() { return wppID, externalID, []*courier.ChannelLog{log}, err } - return "", externalID, []*courier.ChannelLog{log}, err + return "", externalID, []*courier.ChannelLog{log}, nil } func setWhatsAppAuthHeader(header *http.Header, channel courier.Channel) { From 373fd0081a36b3f6bbc1482ef448d8fc1776b449 Mon Sep 17 00:00:00 2001 From: Rafael Soares Date: Mon, 25 Apr 2022 12:14:29 -0300 Subject: [PATCH 108/146] tweak whatsapp test --- handlers/whatsapp/whatsapp_test.go | 38 +++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 27b24c92e..07bdb2c4d 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -373,19 +373,19 @@ var defaultSendTestCases = []ChannelSendTestCase{ {Label: "Link Sending", Text: "Link Sending https://link.com", URN: "whatsapp:250788123123", Path: "/v1/messages", Status: "W", ExternalID: "157b5e14568e8", - ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"text","preview_url":true,"text":{"body":"Link Sending https://link.com"}}`, SendPrep: setSendURL}, {Label: "Plain Send", Text: "Simple Message", URN: "whatsapp:250788123123", Path: "/v1/messages", Status: "W", ExternalID: "157b5e14568e8", - ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"text","text":{"body":"Simple Message"}}`, SendPrep: setSendURL}, {Label: "Unicode Send", Text: "☺", URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", - ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"text","text":{"body":"☺"}}`, SendPrep: setSendURL}, {Label: "Error", @@ -424,7 +424,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"audio","audio":{"link":"https://foo.bar/audio.mp3"}}`, }: MockedResponse{ Status: 201, - Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -441,7 +441,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"document","document":{"link":"https://foo.bar/document.pdf","caption":"document caption","filename":"document.pdf"}}`, }: MockedResponse{ Status: 201, - Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -458,7 +458,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"image","image":{"link":"https://foo.bar/image.jpg","caption":"document caption"}}`, }: MockedResponse{ Status: 201, - Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -475,7 +475,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"video","video":{"link":"https://foo.bar/video.mp4","caption":"video caption"}}`, }: MockedResponse{ Status: 201, - Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -485,7 +485,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", Metadata: json.RawMessage(`{ "templating": { "template": { "name": "revive_issue", "uuid": "171f8a4d-f725-46d7-85a6-11aceff0bfe3" }, "language": "eng", "variables": ["Chef", "tomorrow"]}}`), - ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, RequestBody: `{"to":"250788123123","type":"template","template":{"namespace":"waba_namespace","name":"revive_issue","language":{"policy":"deterministic","code":"en"},"components":[{"type":"body","parameters":[{"type":"text","text":"Chef"},{"type":"text","text":"tomorrow"}]}]}}`, SendPrep: setSendURL, }, @@ -494,7 +494,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", Metadata: json.RawMessage(`{ "templating": { "template": { "name": "revive_issue", "uuid": "171f8a4d-f725-46d7-85a6-11aceff0bfe3" }, "language": "eng", "country": "US", "variables": ["Chef", "tomorrow"]}}`), - ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, RequestBody: `{"to":"250788123123","type":"template","template":{"namespace":"waba_namespace","name":"revive_issue","language":{"policy":"deterministic","code":"en_US"},"components":[{"type":"body","parameters":[{"type":"text","text":"Chef"},{"type":"text","text":"tomorrow"}]}]}}`, SendPrep: setSendURL, }, @@ -503,7 +503,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", Metadata: json.RawMessage(`{ "templating": { "template": { "name": "revive_issue", "uuid": "171f8a4d-f725-46d7-85a6-11aceff0bfe3" }, "namespace": "wa_template_namespace", "language": "eng", "country": "US", "variables": ["Chef", "tomorrow"]}}`), - ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, RequestBody: `{"to":"250788123123","type":"template","template":{"namespace":"wa_template_namespace","name":"revive_issue","language":{"policy":"deterministic","code":"en_US"},"components":[{"type":"body","parameters":[{"type":"text","text":"Chef"},{"type":"text","text":"tomorrow"}]}]}}`, SendPrep: setSendURL, }, @@ -634,13 +634,13 @@ var defaultSendTestCases = []ChannelSendTestCase{ {Label: "Interactive Button Message Send", Text: "Interactive Button Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"BUTTON1"}, Status: "W", ExternalID: "157b5e14568e8", - ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"interactive","interactive":{"type":"button","body":{"text":"Interactive Button Msg"},"action":{"buttons":[{"type":"reply","reply":{"id":"0","title":"BUTTON1"}}]}}}`, SendPrep: setSendURL}, {Label: "Interactive List Message Send", Text: "Interactive List Msg", URN: "whatsapp:250788123123", QuickReplies: []string{"ROW1", "ROW2", "ROW3", "ROW4"}, Status: "W", ExternalID: "157b5e14568e8", - ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 201, RequestBody: `{"to":"250788123123","type":"interactive","interactive":{"type":"list","body":{"text":"Interactive List Msg"},"action":{"button":"Menu","sections":[{"rows":[{"id":"0","title":"ROW1"},{"id":"1","title":"ROW2"},{"id":"2","title":"ROW3"},{"id":"3","title":"ROW4"}]}]}}}`, SendPrep: setSendURL}, {Label: "Interactive Button Message Send with attachment", @@ -719,7 +719,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ BodyContains: `/document.pdf`, }: MockedResponse{ Status: 201, - Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -736,7 +736,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ BodyContains: `/document.pdf`, }: MockedResponse{ Status: 201, - Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -761,7 +761,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"video","video":{"id":"36c484d1-1283-4b94-988d-7276bdec4de2","caption":"video caption"}}`, }: MockedResponse{ Status: 201, - Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -778,7 +778,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"video","video":{"id":"36c484d1-1283-4b94-988d-7276bdec4de2","caption":"video caption"}}`, }: MockedResponse{ Status: 201, - Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -804,7 +804,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"document","document":{"id":"25c484d1-1283-4b94-988d-7276bdec4ef3","caption":"document caption","filename":"document2.pdf"}}`, }: MockedResponse{ Status: 201, - Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -821,7 +821,7 @@ var mediaCacheSendTestCases = []ChannelSendTestCase{ Body: `{"to":"250788123123","type":"document","document":{"id":"25c484d1-1283-4b94-988d-7276bdec4ef3","caption":"document caption","filename":"document2.pdf"}}`, }: MockedResponse{ Status: 201, - Body: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, }, }, SendPrep: setSendURL, @@ -834,7 +834,7 @@ var hsmSupportSendTestCases = []ChannelSendTestCase{ URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", Metadata: json.RawMessage(`{ "templating": { "template": { "name": "revive_issue", "uuid": "171f8a4d-f725-46d7-85a6-11aceff0bfe3" }, "language": "eng", "variables": ["Chef", "tomorrow"]}}`), - ResponseBody: `{ "contacts":[{"input":"250788123123","wa_id":"250788123123"}], "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, + ResponseBody: `{ "messages": [{"id": "157b5e14568e8"}] }`, ResponseStatus: 200, RequestBody: `{"to":"250788123123","type":"hsm","hsm":{"namespace":"waba_namespace","element_name":"revive_issue","language":{"policy":"deterministic","code":"en"},"localizable_params":[{"default":"Chef"},{"default":"tomorrow"}]}}`, SendPrep: setSendURL, }, From 2fe2766fd91e477c0a447db2ae941d43ddf8851b Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 27 Apr 2022 14:38:09 -0500 Subject: [PATCH 109/146] Update CHANGELOG.md for v7.3.4 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56f2e39e9..a5d627bce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.3.4 +---------- + * Update WhatsApp handler so that we update the URN if the returned ID doesn't match + * Stop Telegram contact that have blocked the channel bot + v7.3.3 ---------- * Quick fix to stop JSON content being omitted in logs From 097b939444b1c9927dd39ba67c5c25c11bb52ff4 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 28 Apr 2022 18:40:33 +0200 Subject: [PATCH 110/146] Adjust to use phone ID stored as channel address --- handlers/facebookapp/facebookapp.go | 9 ++------- handlers/facebookapp/facebookapp_test.go | 2 +- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index c5e0bf11b..4c3c45908 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -292,7 +292,7 @@ func (h *handler) GetChannel(ctx context.Context, r *http.Request) (courier.Chan return nil, fmt.Errorf("no changes found") } - channelAddress = payload.Entry[0].Changes[0].Value.Metadata.DisplayPhoneNumber + channelAddress = payload.Entry[0].Changes[0].Value.Metadata.PhoneNumberID if channelAddress == "" { return nil, fmt.Errorf("no channel address found") } @@ -1050,13 +1050,8 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) return nil, fmt.Errorf("missing access token") } - phoneNumberId := msg.Channel().StringConfigForKey(configWACPhoneNumberID, "") - if phoneNumberId == "" { - return nil, fmt.Errorf("missing WAC phone number ID") - } - base, _ := url.Parse(graphURL) - path, _ := url.Parse(fmt.Sprintf("/%s/messages", phoneNumberId)) + path, _ := url.Parse(fmt.Sprintf("/%s/messages", msg.Channel().Address())) wacPhoneURL := base.ResolveReference(path) status := h.Backend().NewMsgStatusForID(msg.Channel(), msg.ID(), courier.MsgErrored) diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 506d0dfd8..aaba35fd3 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -694,7 +694,7 @@ func TestSending(t *testing.T) { maxMsgLength = 100 var ChannelFBA = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) var ChannelIG = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) - var ChannelWAC = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "WAC", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123", configWACPhoneNumberID: "12345_ID"}) + var ChannelWAC = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "WAC", "12345_ID", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) RunChannelSendTestCases(t, ChannelFBA, newHandler("FBA", "Facebook", false), SendTestCasesFBA, nil) RunChannelSendTestCases(t, ChannelIG, newHandler("IG", "Instagram", false), SendTestCasesIG, nil) RunChannelSendTestCases(t, ChannelWAC, newHandler("WAC", "Cloud API WhatsApp", false), SendTestCasesWAC, nil) From bd84397b84a44342ea201e091b2bd7a07b614306 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 28 Apr 2022 20:05:36 +0200 Subject: [PATCH 111/146] Fix tests --- handlers/facebookapp/testdata/wac/audioWAC.json | 4 ++-- handlers/facebookapp/testdata/wac/buttonWAC.json | 4 ++-- handlers/facebookapp/testdata/wac/documentWAC.json | 4 ++-- handlers/facebookapp/testdata/wac/duplicateWAC.json | 4 ++-- handlers/facebookapp/testdata/wac/helloWAC.json | 4 ++-- handlers/facebookapp/testdata/wac/ignoreStatusWAC.json | 4 ++-- handlers/facebookapp/testdata/wac/imageWAC.json | 4 ++-- handlers/facebookapp/testdata/wac/invalidFrom.json | 4 ++-- handlers/facebookapp/testdata/wac/invalidStatusWAC.json | 4 ++-- handlers/facebookapp/testdata/wac/invalidTimestamp.json | 4 ++-- handlers/facebookapp/testdata/wac/locationWAC.json | 4 ++-- handlers/facebookapp/testdata/wac/validStatusWAC.json | 4 ++-- handlers/facebookapp/testdata/wac/videoWAC.json | 4 ++-- handlers/facebookapp/testdata/wac/voiceWAC.json | 4 ++-- 14 files changed, 28 insertions(+), 28 deletions(-) diff --git a/handlers/facebookapp/testdata/wac/audioWAC.json b/handlers/facebookapp/testdata/wac/audioWAC.json index 47ce575db..f578e5fc9 100644 --- a/handlers/facebookapp/testdata/wac/audioWAC.json +++ b/handlers/facebookapp/testdata/wac/audioWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/buttonWAC.json b/handlers/facebookapp/testdata/wac/buttonWAC.json index 36efcca6e..10f592773 100644 --- a/handlers/facebookapp/testdata/wac/buttonWAC.json +++ b/handlers/facebookapp/testdata/wac/buttonWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/documentWAC.json b/handlers/facebookapp/testdata/wac/documentWAC.json index d65921ef4..1c5f08eab 100644 --- a/handlers/facebookapp/testdata/wac/documentWAC.json +++ b/handlers/facebookapp/testdata/wac/documentWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/duplicateWAC.json b/handlers/facebookapp/testdata/wac/duplicateWAC.json index f857c63f7..69463fb0f 100644 --- a/handlers/facebookapp/testdata/wac/duplicateWAC.json +++ b/handlers/facebookapp/testdata/wac/duplicateWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/helloWAC.json b/handlers/facebookapp/testdata/wac/helloWAC.json index f49303d2f..d7cf38ee8 100644 --- a/handlers/facebookapp/testdata/wac/helloWAC.json +++ b/handlers/facebookapp/testdata/wac/helloWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/ignoreStatusWAC.json b/handlers/facebookapp/testdata/wac/ignoreStatusWAC.json index bf9f69714..2b2e583a1 100644 --- a/handlers/facebookapp/testdata/wac/ignoreStatusWAC.json +++ b/handlers/facebookapp/testdata/wac/ignoreStatusWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/imageWAC.json b/handlers/facebookapp/testdata/wac/imageWAC.json index f06b631fc..7d3728e5b 100644 --- a/handlers/facebookapp/testdata/wac/imageWAC.json +++ b/handlers/facebookapp/testdata/wac/imageWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/invalidFrom.json b/handlers/facebookapp/testdata/wac/invalidFrom.json index 052db4a38..12a28cc54 100644 --- a/handlers/facebookapp/testdata/wac/invalidFrom.json +++ b/handlers/facebookapp/testdata/wac/invalidFrom.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/invalidStatusWAC.json b/handlers/facebookapp/testdata/wac/invalidStatusWAC.json index 60676257f..6a3a4fbcc 100644 --- a/handlers/facebookapp/testdata/wac/invalidStatusWAC.json +++ b/handlers/facebookapp/testdata/wac/invalidStatusWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/invalidTimestamp.json b/handlers/facebookapp/testdata/wac/invalidTimestamp.json index e9f301ce9..dc0dd66d5 100644 --- a/handlers/facebookapp/testdata/wac/invalidTimestamp.json +++ b/handlers/facebookapp/testdata/wac/invalidTimestamp.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/locationWAC.json b/handlers/facebookapp/testdata/wac/locationWAC.json index 15cedaa1f..09a721c8d 100644 --- a/handlers/facebookapp/testdata/wac/locationWAC.json +++ b/handlers/facebookapp/testdata/wac/locationWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/validStatusWAC.json b/handlers/facebookapp/testdata/wac/validStatusWAC.json index 0aaf2edb7..8a3360787 100644 --- a/handlers/facebookapp/testdata/wac/validStatusWAC.json +++ b/handlers/facebookapp/testdata/wac/validStatusWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/videoWAC.json b/handlers/facebookapp/testdata/wac/videoWAC.json index 210dd9a81..234422efe 100644 --- a/handlers/facebookapp/testdata/wac/videoWAC.json +++ b/handlers/facebookapp/testdata/wac/videoWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { diff --git a/handlers/facebookapp/testdata/wac/voiceWAC.json b/handlers/facebookapp/testdata/wac/voiceWAC.json index 3e2022eff..03e03375f 100644 --- a/handlers/facebookapp/testdata/wac/voiceWAC.json +++ b/handlers/facebookapp/testdata/wac/voiceWAC.json @@ -8,8 +8,8 @@ "value": { "messaging_product": "whatsapp", "metadata": { - "display_phone_number": "12345", - "phone_number_id": "27681414235104944" + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" }, "contacts": [ { From aab4afa908148ac615681698840cad60bede5d10 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 29 Apr 2022 12:35:00 +0200 Subject: [PATCH 112/146] Fix token to send WA messages --- config.go | 43 +++++++++++++++-------------- handlers/facebookapp/facebookapp.go | 5 +--- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/config.go b/config.go index 5e6447690..fb9d59a5b 100644 --- a/config.go +++ b/config.go @@ -30,6 +30,8 @@ type Config struct { LogLevel string `help:"the logging level courier should use"` Version string `help:"the version that will be used in request and response headers"` + WhatsappAdminSystemUserToken string `help:"the token of the admin system user for WhatsApp"` + // IncludeChannels is the list of channels to enable, empty means include all IncludeChannels []string @@ -40,26 +42,27 @@ type Config struct { // NewConfig returns a new default configuration object func NewConfig() *Config { return &Config{ - Backend: "rapidpro", - Domain: "localhost", - Address: "", - Port: 8080, - DB: "postgres://temba:temba@localhost/temba?sslmode=disable", - Redis: "redis://localhost:6379/15", - SpoolDir: "/var/spool/courier", - S3Endpoint: "https://s3.amazonaws.com", - S3Region: "us-east-1", - S3MediaBucket: "courier-media", - S3MediaPrefix: "/media/", - S3DisableSSL: false, - S3ForcePathStyle: false, - AWSAccessKeyID: "", - AWSSecretAccessKey: "", - FacebookApplicationSecret: "missing_facebook_app_secret", - FacebookWebhookSecret: "missing_facebook_webhook_secret", - MaxWorkers: 32, - LogLevel: "error", - Version: "Dev", + Backend: "rapidpro", + Domain: "localhost", + Address: "", + Port: 8080, + DB: "postgres://temba:temba@localhost/temba?sslmode=disable", + Redis: "redis://localhost:6379/15", + SpoolDir: "/var/spool/courier", + S3Endpoint: "https://s3.amazonaws.com", + S3Region: "us-east-1", + S3MediaBucket: "courier-media", + S3MediaPrefix: "/media/", + S3DisableSSL: false, + S3ForcePathStyle: false, + AWSAccessKeyID: "", + AWSSecretAccessKey: "", + FacebookApplicationSecret: "missing_facebook_app_secret", + FacebookWebhookSecret: "missing_facebook_webhook_secret", + WhatsappAdminSystemUserToken: "missing_whatsapp_admin_system_user_token", + MaxWorkers: 32, + LogLevel: "error", + Version: "Dev", } } diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 4c3c45908..e3e2e29af 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -1045,10 +1045,7 @@ type wacMTResponse struct { func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) (courier.MsgStatus, error) { // can't do anything without an access token - accessToken := msg.Channel().StringConfigForKey(courier.ConfigAuthToken, "") - if accessToken == "" { - return nil, fmt.Errorf("missing access token") - } + accessToken := h.Server().Config().WhatsappAdminSystemUserToken base, _ := url.Parse(graphURL) path, _ := url.Parse(fmt.Sprintf("/%s/messages", msg.Channel().Address())) From 4dec84e8b72816e340e518e3bcc63451c22d874f Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Fri, 29 Apr 2022 17:36:05 +0200 Subject: [PATCH 113/146] Update Start Mobile send URL --- handlers/start/start.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handlers/start/start.go b/handlers/start/start.go index 3fde1351a..bf2cf254a 100644 --- a/handlers/start/start.go +++ b/handlers/start/start.go @@ -22,7 +22,7 @@ import ( var ( maxMsgLength = 1600 - sendURL = "http://bulk.startmobile.com.ua/clients.php" + sendURL = "https://bulk.startmobile.ua/clients.php" ) func init() { From 6905d4aa2ee068980bc99be9eebd15953968bbc1 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 29 Apr 2022 13:58:10 -0500 Subject: [PATCH 114/146] Update CHANGELOG.md for v7.3.5 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a5d627bce..8049ea6f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.3.5 +---------- + * Update Start Mobile send URL + v7.3.4 ---------- * Update WhatsApp handler so that we update the URN if the returned ID doesn't match From 08f4f8bb9d4ee13afae3f97aca32905c1ba2f3d5 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 2 May 2022 11:49:53 -0500 Subject: [PATCH 115/146] Update to go 1.18 and latest gocommon/phonenumbers --- .github/workflows/ci.yml | 4 ++-- go.mod | 31 ++++++++++++++++--------------- go.sum | 37 +++++++++++++++++++------------------ 3 files changed, 37 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5cfff763b..96328b1dc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,8 +1,8 @@ name: CI on: [push, pull_request] env: - go-version: "1.17.7" - redis-version: "3.2.4" + go-version: "1.18.x" + redis-version: "5.0.6" jobs: test: name: Test diff --git a/go.mod b/go.mod index 02e8906de..dd2843f97 100644 --- a/go.mod +++ b/go.mod @@ -1,57 +1,58 @@ module github.com/nyaruka/courier -go 1.17 +go 1.18 require ( github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 - github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect - github.com/aws/aws-sdk-go v1.40.56 // indirect github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456 - github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect github.com/dghubble/oauth1 v0.4.0 github.com/evalphobia/logrus_sentry v0.4.6 - github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 // indirect github.com/go-chi/chi v4.1.2+incompatible github.com/go-errors/errors v1.0.1 - github.com/go-playground/locales v0.14.0 // indirect - github.com/go-playground/universal-translator v0.18.0 // indirect github.com/gofrs/uuid v3.3.0+incompatible github.com/gomodule/redigo v1.8.8 github.com/gorilla/schema v1.0.2 github.com/jmoiron/sqlx v1.3.4 - github.com/kr/pretty v0.1.0 // indirect - github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/lib/pq v1.10.4 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.18.0 + github.com/nyaruka/gocommon v1.19.1 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/null v1.1.1 github.com/nyaruka/redisx v0.2.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.4.2 - github.com/stretchr/testify v1.7.0 + github.com/stretchr/testify v1.7.1 golang.org/x/mod v0.4.2 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/h2non/filetype.v1 v1.0.5 ) require ( + github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect + github.com/aws/aws-sdk-go v1.40.56 // indirect + github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/structs v1.0.0 // indirect github.com/gabriel-vasile/mimetype v1.4.0 // indirect - github.com/golang/protobuf v1.3.2 // indirect + github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 // indirect + github.com/go-playground/locales v0.14.0 // indirect + github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect + github.com/kr/pretty v0.1.0 // indirect + github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.1 // indirect - github.com/nyaruka/phonenumbers v1.0.71 // indirect + github.com/nyaruka/phonenumbers v1.0.75 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/shopspring/decimal v1.2.0 // indirect golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect - golang.org/x/text v0.3.6 // indirect + golang.org/x/text v0.3.7 // indirect + google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect ) diff --git a/go.sum b/go.sum index bb9ebebab..dfeb4d8ed 100644 --- a/go.sum +++ b/go.sum @@ -2,7 +2,6 @@ github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 h1:4EgP6xLAdrD/TR github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa h1:lL66YnJWy1tHlhjSx8fXnpgmv8kQVYnI4ilbYpNB6Zs= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= -github.com/aws/aws-sdk-go v1.34.31/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.40.56 h1:FM2yjR0UUYFzDTMx+mH9Vyw1k1EUUxsAFzk+BjkzANA= github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456 h1:SnUWpAH4lEUoS86woR12h21VMUbDe+DYp88V646wwMI= @@ -34,10 +33,14 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/gomodule/redigo v1.8.8 h1:f6cXq6RRfiyrOJEV7p3JhLDlmawGBVBBP1MggY8Mo4E= github.com/gomodule/redigo v1.8.8/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gorilla/schema v1.0.2 h1:sAgNfOcNYvdDSrzGHVy9nzCQahG+qmsg+nE8dK85QRA= github.com/gorilla/schema v1.0.2/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -69,18 +72,14 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.5.3/go.mod h1:2ZeBZF9yt20IaAJ4aC1ujojAsFhJBk2IuDvSl7KuQDw= -github.com/nyaruka/gocommon v1.17.1 h1:4bbNp+0/BIbne4VDiKOxh3kcbdvEu/WsrsZiG/VyRZ8= -github.com/nyaruka/gocommon v1.17.1/go.mod h1:nmYyb7MZDM0iW4DYJKiBzfKuE9nbnx+xSHZasuIBOT0= -github.com/nyaruka/gocommon v1.18.0 h1:pRSV63w449FVHrjeXJNNIyoD0k47ruqPq3fROuHBjD0= -github.com/nyaruka/gocommon v1.18.0/go.mod h1:+jVWR2FB6XTqFz7fjvVIZMFwp2B6uoaCk7NVLGnfPFM= +github.com/nyaruka/gocommon v1.19.1 h1:miVSFCqSEe1pKAID/PWqwRBtdvPnw6kQZC/Bz9tYVgQ= +github.com/nyaruka/gocommon v1.19.1/go.mod h1:JrQSLAPo9ezSy1AzsJ1zDr1HW0/eu+aipICJkN/+kpg= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= github.com/nyaruka/null v1.1.1/go.mod h1:HSAFbLNOaEhHnoU0VCveCPz0GDtJ3GEtFWhvnBNkhPE= -github.com/nyaruka/phonenumbers v1.0.58/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= -github.com/nyaruka/phonenumbers v1.0.71 h1:itkCGhxkQkHrJ6OyZSApdjQVlPmrWs88MF283pPvbFU= -github.com/nyaruka/phonenumbers v1.0.71/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= +github.com/nyaruka/phonenumbers v1.0.75 h1:OCwKXSjTi6IzuI4gVi8zfY+0s60DQUC6ks8Ll4j0eyU= +github.com/nyaruka/phonenumbers v1.0.75/go.mod h1:cGaEsOrLjIL0iKGqJR5Rfywy86dSkbApEpXuM9KySNA= github.com/nyaruka/redisx v0.2.1 h1:BavpQRCsK5xV2uxPdJJ26yVmjSo+q6bdjWqeNNf0s5w= github.com/nyaruka/redisx v0.2.1/go.mod h1:cdbAm4y/+oFWu7qFzH2ERPeqRXJC2CtgRhwcBacM4Oc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= @@ -99,17 +98,15 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200925080053-05aa5d4ee321/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -117,25 +114,29 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y= From d97f45cd1e9e7ef71bbf3b31c19d0ce51f87e927 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 2 May 2022 11:54:13 -0500 Subject: [PATCH 116/146] Update jsonparser --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dd2843f97..9f87fcee1 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 - github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456 + github.com/buger/jsonparser v1.1.1 github.com/dghubble/oauth1 v0.4.0 github.com/evalphobia/logrus_sentry v0.4.6 github.com/go-chi/chi v4.1.2+incompatible diff --git a/go.sum b/go.sum index dfeb4d8ed..38748ea1a 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa h1:lL66YnJWy1tHlhjSx github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= github.com/aws/aws-sdk-go v1.40.56 h1:FM2yjR0UUYFzDTMx+mH9Vyw1k1EUUxsAFzk+BjkzANA= github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456 h1:SnUWpAH4lEUoS86woR12h21VMUbDe+DYp88V646wwMI= -github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg= github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= From 470cd9d8179edf6f77125b85b5b152107aadab69 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 2 May 2022 12:54:36 -0500 Subject: [PATCH 117/146] Update CHANGELOG.md for v7.3.6 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8049ea6f7..a86d0bba6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.3.6 +---------- + * Update to go 1.18 and latest gocommon/phonenumbers/jsonparser + v7.3.5 ---------- * Update Start Mobile send URL From d2d537ca0c28ad0af8ee6d3642f2aff20c069452 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Wed, 4 May 2022 16:32:17 +0200 Subject: [PATCH 118/146] Fix to not stop contact for other errors --- handlers/telegram/telegram.go | 2 +- handlers/telegram/telegram_test.go | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/handlers/telegram/telegram.go b/handlers/telegram/telegram.go index 71ed80349..08bfa78e7 100644 --- a/handlers/telegram/telegram.go +++ b/handlers/telegram/telegram.go @@ -176,7 +176,7 @@ func (h *handler) sendMsgPart(msg courier.Msg, token string, path string, form u if response.Result.MessageID > 0 { return strconv.FormatInt(response.Result.MessageID, 10), log, false, nil } - return "", log, true, errors.Errorf("no 'result.message_id' in response") + return "", log, false, errors.Errorf("no 'result.message_id' in response") } // SendMsg sends the passed in message, returning any error diff --git a/handlers/telegram/telegram_test.go b/handlers/telegram/telegram_test.go index d8c1c343d..c2e11f53f 100644 --- a/handlers/telegram/telegram_test.go +++ b/handlers/telegram/telegram_test.go @@ -615,7 +615,17 @@ var defaultSendTestCases = []ChannelSendTestCase{ PostParams: map[string]string{"text": `Stopped Contact`, "chat_id": "12345"}, SendPrep: setSendURL, Stopped: true}, - + {Label: "Should not stop other error", + Text: "Simple Message", URN: "telegram:12345", + Status: "E", + ResponseBody: `{ "ok": true }`, ResponseStatus: 200, + PostParams: map[string]string{ + "text": "Simple Message", + "chat_id": "12345", + "reply_markup": `{"remove_keyboard":true}`, + }, + SendPrep: setSendURL, + Stopped: false}, {Label: "Send Photo", Text: "My pic!", URN: "telegram:12345", Attachments: []string{"image/jpeg:https://foo.bar/image.jpg"}, Status: "W", From 52bec90145ed535b4a834acdda4430500a769b76 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 4 May 2022 10:07:46 -0500 Subject: [PATCH 119/146] Update CHANGELOG.md for v7.3.7 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a86d0bba6..b06779311 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.3.7 +---------- + * Fix to not stop contact for other errors + v7.3.6 ---------- * Update to go 1.18 and latest gocommon/phonenumbers/jsonparser From bea7f185090def8ef165d0abe9afe88bce2fe40f Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Mon, 9 May 2022 10:48:50 +0200 Subject: [PATCH 120/146] Add log to status first --- handlers/telegram/telegram.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/handlers/telegram/telegram.go b/handlers/telegram/telegram.go index 08bfa78e7..499fe75f9 100644 --- a/handlers/telegram/telegram.go +++ b/handlers/telegram/telegram.go @@ -219,6 +219,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } externalID, log, botBlocked, err := h.sendMsgPart(msg, authToken, "sendMessage", form, msgKeyBoard) + status.AddLog(log) if botBlocked { status.SetStatus(courier.MsgFailed) channelEvent := h.Backend().NewChannelEvent(msg.Channel(), courier.StopContact, msg.URN()) @@ -227,7 +228,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } status.SetExternalID(externalID) hasError = err != nil - status.AddLog(log) } @@ -247,6 +247,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "caption": []string{caption}, } externalID, log, botBlocked, err := h.sendMsgPart(msg, authToken, "sendPhoto", form, attachmentKeyBoard) + status.AddLog(log) if botBlocked { status.SetStatus(courier.MsgFailed) channelEvent := h.Backend().NewChannelEvent(msg.Channel(), courier.StopContact, msg.URN()) @@ -255,7 +256,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } status.SetExternalID(externalID) hasError = err != nil - status.AddLog(log) case "video": form := url.Values{ @@ -264,6 +264,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "caption": []string{caption}, } externalID, log, botBlocked, err := h.sendMsgPart(msg, authToken, "sendVideo", form, attachmentKeyBoard) + status.AddLog(log) if botBlocked { status.SetStatus(courier.MsgFailed) channelEvent := h.Backend().NewChannelEvent(msg.Channel(), courier.StopContact, msg.URN()) @@ -272,7 +273,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } status.SetExternalID(externalID) hasError = err != nil - status.AddLog(log) case "audio": form := url.Values{ @@ -281,6 +281,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "caption": []string{caption}, } externalID, log, botBlocked, err := h.sendMsgPart(msg, authToken, "sendAudio", form, attachmentKeyBoard) + status.AddLog(log) if botBlocked { status.SetStatus(courier.MsgFailed) channelEvent := h.Backend().NewChannelEvent(msg.Channel(), courier.StopContact, msg.URN()) @@ -289,7 +290,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } status.SetExternalID(externalID) hasError = err != nil - status.AddLog(log) case "application": form := url.Values{ @@ -298,6 +298,7 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat "caption": []string{caption}, } externalID, log, botBlocked, err := h.sendMsgPart(msg, authToken, "sendDocument", form, attachmentKeyBoard) + status.AddLog(log) if botBlocked { status.SetStatus(courier.MsgFailed) channelEvent := h.Backend().NewChannelEvent(msg.Channel(), courier.StopContact, msg.URN()) @@ -306,7 +307,6 @@ func (h *handler) SendMsg(ctx context.Context, msg courier.Msg) (courier.MsgStat } status.SetExternalID(externalID) hasError = err != nil - status.AddLog(log) default: status.AddLog(courier.NewChannelLog("Unknown media type: "+mediaType, msg.Channel(), msg.ID(), "", "", courier.NilStatusCode, From 04543ee0b5a4541aab5b1fd0a2e19dab8bbda258 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 9 May 2022 09:24:25 -0500 Subject: [PATCH 121/146] Update CHANGELOG.md for v7.3.8 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b06779311..62d0861f8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.3.8 +---------- + * Add log to status first when handling telegram opt outs + v7.3.7 ---------- * Fix to not stop contact for other errors From 10f90ca9e7fa712b118a44afae3c49ee486b14c1 Mon Sep 17 00:00:00 2001 From: Morris Mukiri Date: Mon, 16 May 2022 11:28:36 +0300 Subject: [PATCH 122/146] Add arm64 as a build target Signed-off-by: Morris Mukiri --- goreleaser.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/goreleaser.yml b/goreleaser.yml index 01505738b..1e7275f34 100644 --- a/goreleaser.yml +++ b/goreleaser.yml @@ -7,6 +7,7 @@ build: - linux goarch: - amd64 + - arm64 archives: - files: From f68b9e05d5f26ea168d48375183cd84a9ca742d6 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Thu, 26 May 2022 13:25:39 -0500 Subject: [PATCH 123/146] Update CHANGELOG.md for v7.3.9 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 62d0861f8..2fe80d404 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +v7.3.9 +---------- + * Add arm64 as a build target + * Add support for WA Cloud API + * Refactor FBA tests + v7.3.8 ---------- * Add log to status first when handling telegram opt outs From 1b6866402c02b3208b55dfc57cc14b07fa6033d5 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Mon, 30 May 2022 13:25:15 +0200 Subject: [PATCH 124/146] Make sure text are sent after audio attachments for WA channels --- handlers/whatsapp/whatsapp.go | 27 +++++++++++++++++++++++++++ handlers/whatsapp/whatsapp_test.go | 10 +++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 8973bd820..ba93abe03 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -578,6 +578,8 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann isInteractiveMsgCompatible := semver.Compare(wppVersion, interactiveMsgMinSupVersion) isInteractiveMsg := (isInteractiveMsgCompatible >= 0) && (len(qrs) > 0) + textAsCaption := false + if len(msg.Attachments()) > 0 { for attachmentCount, attachment := range msg.Attachments() { @@ -608,6 +610,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann } if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() + textAsCaption = true } mediaPayload.Filename, err = utils.BasePathForURL(fileURL) @@ -624,6 +627,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann } if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() + textAsCaption = true } payload.Image = mediaPayload payloads = append(payloads, payload) @@ -634,6 +638,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann } if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() + textAsCaption = true } payload.Video = mediaPayload payloads = append(payloads, payload) @@ -646,6 +651,28 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann } } + if !textAsCaption && !isInteractiveMsg { + for _, part := range parts { + + //check if you have a link + var payload mtTextPayload + if strings.Contains(part, "https://") || strings.Contains(part, "http://") { + payload = mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + PreviewURL: true, + } + } else { + payload = mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + } + } + payload.Text.Body = part + payloads = append(payloads, payload) + } + } + if isInteractiveMsg { for i, part := range parts { if i < (len(parts) - 1) { //if split into more than one message, the first parts will be text and the last interactive diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 07bdb2c4d..8f9121f1d 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -413,7 +413,7 @@ var defaultSendTestCases = []ChannelSendTestCase{ RequestBody: `{"to":"250788123123","type":"text","text":{"body":"Error"}}`, SendPrep: setSendURL}, {Label: "Audio Send", - Text: "audio has no caption", + Text: "audio has no caption, sent as text", URN: "whatsapp:250788123123", Status: "W", ExternalID: "157b5e14568e8", Attachments: []string{"audio/mpeg:https://foo.bar/audio.mp3"}, @@ -426,6 +426,14 @@ var defaultSendTestCases = []ChannelSendTestCase{ Status: 201, Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"text","text":{"body":"audio has no caption, sent as text"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, }, SendPrep: setSendURL, }, From 2909fd18e41c2485393b94fa0bc8f04ea242f31d Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Mon, 30 May 2022 14:08:54 +0200 Subject: [PATCH 125/146] More coverage --- handlers/whatsapp/whatsapp_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/handlers/whatsapp/whatsapp_test.go b/handlers/whatsapp/whatsapp_test.go index 8f9121f1d..f9bd81632 100644 --- a/handlers/whatsapp/whatsapp_test.go +++ b/handlers/whatsapp/whatsapp_test.go @@ -437,6 +437,31 @@ var defaultSendTestCases = []ChannelSendTestCase{ }, SendPrep: setSendURL, }, + {Label: "Audio Send with link in text", + Text: "audio has no caption, sent as text with a https://example.com", + URN: "whatsapp:250788123123", + Status: "W", ExternalID: "157b5e14568e8", + Attachments: []string{"audio/mpeg:https://foo.bar/audio.mp3"}, + Responses: map[MockedRequest]MockedResponse{ + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"audio","audio":{"link":"https://foo.bar/audio.mp3"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + MockedRequest{ + Method: "POST", + Path: "/v1/messages", + Body: `{"to":"250788123123","type":"text","preview_url":true,"text":{"body":"audio has no caption, sent as text with a https://example.com"}}`, + }: MockedResponse{ + Status: 201, + Body: `{ "messages": [{"id": "157b5e14568e8"}] }`, + }, + }, + SendPrep: setSendURL, + }, {Label: "Document Send", Text: "document caption", URN: "whatsapp:250788123123", From d2e0c7be339e4b8d0d955fd345a0a9d0db88586a Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Mon, 30 May 2022 13:47:02 -0500 Subject: [PATCH 126/146] Update CHANGELOG.md for v7.3.10 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fe80d404..3dd1932bf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +v7.3.10 +---------- + * Make sure text are sent after audio attachments for WA channels + v7.3.9 ---------- * Add arm64 as a build target From 8356f93d2471f06c69d5626a5e0dfc01f6f43860 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Wed, 29 Jun 2022 09:33:05 +0100 Subject: [PATCH 127/146] Use analytics package from gocommon --- backends/rapidpro/backend.go | 18 ++++++------- backends/rapidpro/contact.go | 4 +-- go.mod | 25 +++++++++--------- go.sum | 51 ++++++++++++++++++++---------------- sender.go | 6 ++--- server.go | 20 +++++++------- 6 files changed, 65 insertions(+), 59 deletions(-) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index 654e84e39..62c83809f 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -18,10 +18,10 @@ import ( "github.com/nyaruka/courier/batch" "github.com/nyaruka/courier/queue" "github.com/nyaruka/courier/utils" + "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/gocommon/dbutil" "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/gocommon/urns" - "github.com/nyaruka/librato" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -473,14 +473,14 @@ func (b *backend) Heartbeat() error { b.redisWaitDuration = redisStats.WaitDuration b.redisWaitCount = redisStats.WaitCount - librato.Gauge("courier.db_busy", float64(dbStats.InUse)) - librato.Gauge("courier.db_idle", float64(dbStats.Idle)) - librato.Gauge("courier.db_wait_ms", float64(dbWaitDurationInPeriod/time.Millisecond)) - librato.Gauge("courier.db_wait_count", float64(dbWaitCountInPeriod)) - librato.Gauge("courier.redis_wait_ms", float64(redisWaitDurationInPeriod/time.Millisecond)) - librato.Gauge("courier.redis_wait_count", float64(redisWaitCountInPeriod)) - librato.Gauge("courier.bulk_queue", float64(bulkSize)) - librato.Gauge("courier.priority_queue", float64(prioritySize)) + analytics.Gauge("courier.db_busy", float64(dbStats.InUse)) + analytics.Gauge("courier.db_idle", float64(dbStats.Idle)) + analytics.Gauge("courier.db_wait_ms", float64(dbWaitDurationInPeriod/time.Millisecond)) + analytics.Gauge("courier.db_wait_count", float64(dbWaitCountInPeriod)) + analytics.Gauge("courier.redis_wait_ms", float64(redisWaitDurationInPeriod/time.Millisecond)) + analytics.Gauge("courier.redis_wait_count", float64(redisWaitCountInPeriod)) + analytics.Gauge("courier.bulk_queue", float64(bulkSize)) + analytics.Gauge("courier.priority_queue", float64(prioritySize)) logrus.WithFields(logrus.Fields{ "db_busy": dbStats.InUse, diff --git a/backends/rapidpro/contact.go b/backends/rapidpro/contact.go index 722ab4c76..831dedb28 100644 --- a/backends/rapidpro/contact.go +++ b/backends/rapidpro/contact.go @@ -9,10 +9,10 @@ import ( "unicode/utf8" "github.com/nyaruka/courier" + "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/gocommon/dbutil" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/gocommon/uuids" - "github.com/nyaruka/librato" "github.com/nyaruka/null" "github.com/pkg/errors" @@ -211,7 +211,7 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne contact.URNID_ = contactURN.ID // log that we created a new contact to librato - librato.Gauge("courier.new_contact", float64(1)) + analytics.Gauge("courier.new_contact", float64(1)) // and return it return contact, nil diff --git a/go.mod b/go.mod index 9f87fcee1..61456e6f8 100644 --- a/go.mod +++ b/go.mod @@ -9,20 +9,19 @@ require ( github.com/evalphobia/logrus_sentry v0.4.6 github.com/go-chi/chi v4.1.2+incompatible github.com/go-errors/errors v1.0.1 - github.com/gofrs/uuid v3.3.0+incompatible + github.com/gofrs/uuid v4.2.0+incompatible github.com/gomodule/redigo v1.8.8 github.com/gorilla/schema v1.0.2 - github.com/jmoiron/sqlx v1.3.4 - github.com/lib/pq v1.10.4 + github.com/jmoiron/sqlx v1.3.5 + github.com/lib/pq v1.10.6 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.19.1 - github.com/nyaruka/librato v1.0.0 + github.com/nyaruka/gocommon v1.22.2 github.com/nyaruka/null v1.1.1 github.com/nyaruka/redisx v0.2.1 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/sirupsen/logrus v1.4.2 - github.com/stretchr/testify v1.7.1 + github.com/sirupsen/logrus v1.8.1 + github.com/stretchr/testify v1.7.2 golang.org/x/mod v0.4.2 gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/h2non/filetype.v1 v1.0.5 @@ -30,7 +29,7 @@ require ( require ( github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect - github.com/aws/aws-sdk-go v1.40.56 // indirect + github.com/aws/aws-sdk-go v1.44.34 // indirect github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/structs v1.0.0 // indirect @@ -40,19 +39,19 @@ require ( github.com/go-playground/universal-translator v0.18.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect github.com/kr/pretty v0.1.0 // indirect github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.1 // indirect + github.com/nyaruka/librato v1.0.0 // indirect github.com/nyaruka/phonenumbers v1.0.75 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/shopspring/decimal v1.2.0 // indirect - golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect - golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect + github.com/shopspring/decimal v1.3.1 // indirect + golang.org/x/net v0.0.0-20220614195744-fb05da6f9022 // indirect + golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/protobuf v1.28.0 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 38748ea1a..27dc64210 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 h1:4EgP6xLAdrD/TR github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa h1:lL66YnJWy1tHlhjSx8fXnpgmv8kQVYnI4ilbYpNB6Zs= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa/go.mod h1:Yee4kTMuNiPYJ7nSNorELQMr1J33uOpXDMByNYhvtNk= -github.com/aws/aws-sdk-go v1.40.56 h1:FM2yjR0UUYFzDTMx+mH9Vyw1k1EUUxsAFzk+BjkzANA= -github.com/aws/aws-sdk-go v1.40.56/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= +github.com/aws/aws-sdk-go v1.44.34 h1:+ZtWIbtSGLNB99P8eBrxXfJZgiIouzUbpkf/MNxq2yQ= +github.com/aws/aws-sdk-go v1.44.34/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg= @@ -29,10 +29,10 @@ github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= -github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= +github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= @@ -47,9 +47,8 @@ github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9Y github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= -github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -62,8 +61,8 @@ github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= -github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= +github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= @@ -72,8 +71,8 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.19.1 h1:miVSFCqSEe1pKAID/PWqwRBtdvPnw6kQZC/Bz9tYVgQ= -github.com/nyaruka/gocommon v1.19.1/go.mod h1:JrQSLAPo9ezSy1AzsJ1zDr1HW0/eu+aipICJkN/+kpg= +github.com/nyaruka/gocommon v1.22.2 h1:iEusd0CijvYvhW+bEZ6LWYQuaXS4Oac5WH9W8iK6DEw= +github.com/nyaruka/gocommon v1.22.2/go.mod h1:g6/d9drZXDUrtRSPe2Kf8lTUS+baHt/0G0dwHq3qeIU= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= @@ -88,10 +87,11 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= -github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= +github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -99,8 +99,9 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= @@ -108,17 +109,22 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= -golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220614195744-fb05da6f9022 h1:0qjDla5xICC2suMtyRH/QqX3B1btXTfNsIt/i4LFgO0= +golang.org/x/net v0.0.0-20220614195744-fb05da6f9022/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 h1:PgOr27OhUx2IRqGJ2RxAWI4dJQ7bi9cSrB82uzFzfUA= +golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= @@ -144,5 +150,6 @@ gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4Dl gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/sender.go b/sender.go index dbed21aab..094e1b437 100644 --- a/sender.go +++ b/sender.go @@ -5,7 +5,7 @@ import ( "fmt" "time" - "github.com/nyaruka/librato" + "github.com/nyaruka/gocommon/analytics" "github.com/sirupsen/logrus" ) @@ -210,10 +210,10 @@ func (w *Sender) sendMessage(msg Msg) { // report to librato and log locally if status.Status() == MsgErrored || status.Status() == MsgFailed { log.WithField("elapsed", duration).Warning("msg errored") - librato.Gauge(fmt.Sprintf("courier.msg_send_error_%s", msg.Channel().ChannelType()), secondDuration) + analytics.Gauge(fmt.Sprintf("courier.msg_send_error_%s", msg.Channel().ChannelType()), secondDuration) } else { log.WithField("elapsed", duration).Info("msg sent") - librato.Gauge(fmt.Sprintf("courier.msg_send_%s", msg.Channel().ChannelType()), secondDuration) + analytics.Gauge(fmt.Sprintf("courier.msg_send_%s", msg.Channel().ChannelType()), secondDuration) } } diff --git a/server.go b/server.go index 9cd5661d5..4783667fd 100644 --- a/server.go +++ b/server.go @@ -20,7 +20,7 @@ import ( "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" "github.com/nyaruka/courier/utils" - "github.com/nyaruka/librato" + "github.com/nyaruka/gocommon/analytics" "github.com/sirupsen/logrus" ) @@ -90,10 +90,11 @@ func (s *server) Start() error { // configure librato if we have configuration options for it host, _ := os.Hostname() if s.config.LibratoUsername != "" { - librato.Configure(s.config.LibratoUsername, s.config.LibratoToken, host, time.Second, s.waitGroup) - librato.Start() + analytics.RegisterBackend(analytics.NewLibrato(s.config.LibratoUsername, s.config.LibratoToken, host, time.Second, s.waitGroup)) } + analytics.Start() + // start our backend err := s.backend.Start() if err != nil { @@ -189,8 +190,7 @@ func (s *server) Stop() error { return err } - // stop our librato sender - librato.Stop() + analytics.Stop() // wait for everything to stop s.waitGroup.Wait() @@ -322,10 +322,10 @@ func (s *server) channelHandleWrapper(handler ChannelHandler, handlerFunc Channe if channel != nil && len(events) == 0 { if err != nil { logs = append(logs, NewChannelLog("Channel Error", channel, NilMsgID, r.Method, url, ww.Status(), string(request), prependHeaders(response.String(), ww.Status(), w), duration, err)) - librato.Gauge(fmt.Sprintf("courier.channel_error_%s", channel.ChannelType()), secondDuration) + analytics.Gauge(fmt.Sprintf("courier.channel_error_%s", channel.ChannelType()), secondDuration) } else { logs = append(logs, NewChannelLog("Request Ignored", channel, NilMsgID, r.Method, url, ww.Status(), string(request), prependHeaders(response.String(), ww.Status(), w), duration, err)) - librato.Gauge(fmt.Sprintf("courier.channel_ignored_%s", channel.ChannelType()), secondDuration) + analytics.Gauge(fmt.Sprintf("courier.channel_ignored_%s", channel.ChannelType()), secondDuration) } } @@ -334,15 +334,15 @@ func (s *server) channelHandleWrapper(handler ChannelHandler, handlerFunc Channe switch e := event.(type) { case Msg: logs = append(logs, NewChannelLog("Message Received", channel, e.ID(), r.Method, url, ww.Status(), string(request), prependHeaders(response.String(), ww.Status(), w), duration, err)) - librato.Gauge(fmt.Sprintf("courier.msg_receive_%s", channel.ChannelType()), secondDuration) + analytics.Gauge(fmt.Sprintf("courier.msg_receive_%s", channel.ChannelType()), secondDuration) LogMsgReceived(r, e) case ChannelEvent: logs = append(logs, NewChannelLog("Event Received", channel, NilMsgID, r.Method, url, ww.Status(), string(request), prependHeaders(response.String(), ww.Status(), w), duration, err)) - librato.Gauge(fmt.Sprintf("courier.evt_receive_%s", channel.ChannelType()), secondDuration) + analytics.Gauge(fmt.Sprintf("courier.evt_receive_%s", channel.ChannelType()), secondDuration) LogChannelEventReceived(r, e) case MsgStatus: logs = append(logs, NewChannelLog("Status Updated", channel, e.ID(), r.Method, url, ww.Status(), string(request), response.String(), duration, err)) - librato.Gauge(fmt.Sprintf("courier.msg_status_%s", channel.ChannelType()), secondDuration) + analytics.Gauge(fmt.Sprintf("courier.msg_status_%s", channel.ChannelType()), secondDuration) LogMsgStatusReceived(r, e) } } From f48cb1d9fab8e64342eeac1a67b49d84330e9ffa Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 8 Jul 2022 13:40:22 +0100 Subject: [PATCH 128/146] Update README --- README.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 0e8d402a8..0d10f97ae 100644 --- a/README.md +++ b/README.md @@ -4,28 +4,26 @@ [![codecov](https://codecov.io/gh/nyaruka/courier/branch/main/graph/badge.svg)](https://codecov.io/gh/nyaruka/courier) [![Go Report Card](https://goreportcard.com/badge/github.com/nyaruka/courier)](https://goreportcard.com/report/github.com/nyaruka/courier) -# About - Courier is a messaging gateway for text-based messaging channels. It abstracts out various different texting mediums and providers, allowing applications to focus on the creation and processing of those messages. -Current courier supports over 50 different channel types, ranging for SMS aggregators like Twilio to +Currently it supports over 50 different channel types, ranging for SMS aggregators like Twilio to IP channels like Facebook and Telegram messenger. The goal is for Courier to support every popular messaging channels and aggregator and we are happy to accept pull requests to help accomplish that. Courier is currently used to power [RapidPro](https://rapidpro.io) and [TextIt](https://textit.com) but the backend is pluggable, so you can add your own backend to read and write messages. -# Deploying +## Deploying -As courier is a go application, it compiles to a binary and that binary along with the config file is all +As a Go application, it compiles to a binary and that binary along with the config file is all you need to run it on your server. You can find bundles for each platform in the -[releases directory](https://github.com/nyaruka/courier/releases). We recommend running Courier +[releases directory](https://github.com/nyaruka/courier/releases). We recommend running it behind a reverse proxy such as nginx or Elastic Load Balancer that provides HTTPs encryption. -# Configuration +## Configuration -Courier uses a tiered configuration system, each option takes precendence over the ones above it: +The service uses a tiered configuration system, each option takes precendence over the ones above it: 1. The configuration file 2. Environment variables starting with `COURIER_` @@ -35,7 +33,7 @@ We recommend running courier with no changes to the configuration and no paramet environment variables to configure it. You can use `% courier --help` to see a list of the environment variables and parameters and for more details on each option. -# RapidPro Configuration +### RapidPro For use with RapidPro, you will want to configure these settings: @@ -58,9 +56,9 @@ Recommended settings for error and performance monitoring: * `COURIER_LIBRATO_TOKEN`: The token to use for logging of events to Librato * `COURIER_SENTRY_DSN`: The DSN to use when logging errors to Sentry -# Development +## Development -Once you've checked out the code, you can build Courier with: +Once you've checked out the code, you can build it with: ``` go install github.com/nyaruka/courier/cmd/courier From 57732ae1de97e0166985cedcf0cca6eabc59fea0 Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 8 Jul 2022 13:41:30 +0100 Subject: [PATCH 129/146] Update CHANGELOG.md for v7.4.0 --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3dd1932bf..c0c898ff8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +v7.4.0 +---------- + * Update README + * Use analytics package from gocommon + v7.3.10 ---------- * Make sure text are sent after audio attachments for WA channels From f80c305ffadcb12d41cef7d098fb6636e74a03bb Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 3 Nov 2022 16:17:02 -0300 Subject: [PATCH 130/146] Update WENI-CHANGELOG.md for 1.5.0-courier-7.2.0 --- WENI-CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/WENI-CHANGELOG.md b/WENI-CHANGELOG.md index a199e6045..478f6be84 100644 --- a/WENI-CHANGELOG.md +++ b/WENI-CHANGELOG.md @@ -1,3 +1,7 @@ +1.5.0-courier-7.2.0 +---------- + * Merge tag v7.2.0 from nyaruka into our 1.4.5-courier-7.1.0 + 1.4.5-courier-7.1.0 ---------- * Remove expiration_timestamp from moPayload in WAC #133 From 335f5b54f236ce0aa363b8c61a1076e62f7cf70a Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 30 Nov 2022 12:11:56 -0300 Subject: [PATCH 131/146] Add caption verification on attachments --- go.mod | 17 ++++++----------- go.sum | 19 ++++++++++--------- handlers/whatsapp/whatsapp.go | 28 ++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 2ba3084ad..4dfa955e6 100644 --- a/go.mod +++ b/go.mod @@ -9,27 +9,29 @@ require ( github.com/evalphobia/logrus_sentry v0.4.6 github.com/go-chi/chi v4.1.2+incompatible github.com/go-errors/errors v1.0.1 - github.com/mattn/go-sqlite3 v1.14.10 // indirect github.com/gofrs/uuid v4.2.0+incompatible github.com/gomodule/redigo v1.8.8 github.com/gorilla/schema v1.0.2 github.com/jmoiron/sqlx v1.3.5 github.com/lib/pq v1.10.6 + github.com/mattn/go-sqlite3 v1.14.10 // indirect github.com/nyaruka/ezconf v0.2.1 github.com/nyaruka/gocommon v1.22.2 github.com/nyaruka/null v1.1.1 github.com/pkg/errors v0.9.1 - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.2 golang.org/x/mod v0.4.2 + gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/h2non/filetype.v1 v1.0.5 ) require ( + github.com/gabriel-vasile/mimetype v1.4.0 github.com/golang-jwt/jwt/v4 v4.4.1 github.com/lestrrat-go/jwx v1.2.25 + github.com/nyaruka/librato v1.0.0 github.com/nyaruka/redisx v0.2.2 github.com/patrickmn/go-cache v2.1.0+incompatible gopkg.in/go-playground/assert.v1 v1.2.1 @@ -42,11 +44,10 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d // indirect github.com/fatih/structs v1.0.0 // indirect - github.com/goccy/go-json v0.9.7 // indirect - github.com/gabriel-vasile/mimetype v1.4.0 // indirect github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 // indirect github.com/go-playground/locales v0.14.0 // indirect github.com/go-playground/universal-translator v0.18.0 // indirect + github.com/goccy/go-json v0.9.7 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kr/pretty v0.1.0 // indirect @@ -59,19 +60,13 @@ require ( github.com/lestrrat-go/option v1.0.0 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.1 // indirect - github.com/nyaruka/librato v1.0.0 // indirect github.com/nyaruka/phonenumbers v1.0.75 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect github.com/shopspring/decimal v1.3.1 // indirect + golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect golang.org/x/net v0.0.0-20220614195744-fb05da6f9022 // indirect golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/protobuf v1.28.0 // indirect - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - -replace github.com/nyaruka/gocommon => github.com/Ilhasoft/gocommon v1.16.2-weni - -replace github.com/gomodule/redigo => github.com/gomodule/redigo v1.8.8 diff --git a/go.sum b/go.sum index ca599c406..54dadc46d 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,3 @@ -github.com/Ilhasoft/gocommon v1.16.2-weni h1:IDDxPVNIVDMwSErQmTrAiziLMvEi6rbeRb3GG8D+XmA= -github.com/Ilhasoft/gocommon v1.16.2-weni/go.mod h1:pk8L9T79VoKO8OWTiZbtUutFPI3sGGKB5u8nNWDKuGE= github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 h1:4EgP6xLAdrD/TRlbSw4n2W6h68K2P3+R7lKqFoL5U9Q= github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa h1:lL66YnJWy1tHlhjSx8fXnpgmv8kQVYnI4ilbYpNB6Zs= @@ -34,14 +32,14 @@ github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= -github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= -github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= -github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM= +github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ= +github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= @@ -84,7 +82,6 @@ github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk= github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= @@ -102,8 +99,8 @@ github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= github.com/nyaruka/null v1.1.1/go.mod h1:HSAFbLNOaEhHnoU0VCveCPz0GDtJ3GEtFWhvnBNkhPE= github.com/nyaruka/phonenumbers v1.0.75 h1:OCwKXSjTi6IzuI4gVi8zfY+0s60DQUC6ks8Ll4j0eyU= github.com/nyaruka/phonenumbers v1.0.75/go.mod h1:cGaEsOrLjIL0iKGqJR5Rfywy86dSkbApEpXuM9KySNA= -github.com/nyaruka/redisx v0.2.1 h1:BavpQRCsK5xV2uxPdJJ26yVmjSo+q6bdjWqeNNf0s5w= -github.com/nyaruka/redisx v0.2.1/go.mod h1:cdbAm4y/+oFWu7qFzH2ERPeqRXJC2CtgRhwcBacM4Oc= +github.com/nyaruka/redisx v0.2.2 h1:OAJ4g1So2opn6O5akDWEWiDWgEOvPMKU10EUCG/Nv9Y= +github.com/nyaruka/redisx v0.2.2/go.mod h1:cdbAm4y/+oFWu7qFzH2ERPeqRXJC2CtgRhwcBacM4Oc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -128,11 +125,14 @@ github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f h1:OeJjE6G4dgCY4PIXvIRQbE8+RX+uXZyGhUy/ksMGJoc= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220614195744-fb05da6f9022 h1:0qjDla5xICC2suMtyRH/QqX3B1btXTfNsIt/i4LFgO0= golang.org/x/net v0.0.0-20220614195744-fb05da6f9022/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -167,6 +167,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y= diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index 4c2ecc491..dc41fd06e 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -641,6 +641,8 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann isInteractiveMsgCompatible := semver.Compare(wppVersion, interactiveMsgMinSupVersion) isInteractiveMsg := (isInteractiveMsgCompatible >= 0) && (len(qrs) > 0) + textAsCaption := false + // do we have a template? templating, err := h.getTemplate(msg) if templating != nil || len(msg.Attachments()) == 0 { @@ -847,6 +849,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann } if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() + textAsCaption = true } mediaPayload.Filename, err = utils.BasePathForURL(fileURL) // Logging error @@ -862,6 +865,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann } if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() + textAsCaption = true } payload.Image = mediaPayload payloads = append(payloads, payload) @@ -872,6 +876,7 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann } if attachmentCount == 0 && !isInteractiveMsg { mediaPayload.Caption = msg.Text() + textAsCaption = true } payload.Video = mediaPayload payloads = append(payloads, payload) @@ -883,6 +888,29 @@ func buildPayloads(msg courier.Msg, h *handler) ([]interface{}, []*courier.Chann break } } + + if !textAsCaption && !isInteractiveMsg { + for _, part := range parts { + + //check if you have a link + var payload mtTextPayload + if strings.Contains(part, "https://") || strings.Contains(part, "http://") { + payload = mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + PreviewURL: true, + } + } else { + payload = mtTextPayload{ + To: msg.URN().Path(), + Type: "text", + } + } + payload.Text.Body = part + payloads = append(payloads, payload) + } + } + if isInteractiveMsg { for i, part := range parts { if i < (len(parts) - 1) { //if split into more than one message, the first parts will be text and the last interactive From f3b19db203bddfc42ec6185353e27ade797918db Mon Sep 17 00:00:00 2001 From: Robi9 Date: Wed, 30 Nov 2022 12:40:00 -0300 Subject: [PATCH 132/146] Change gocommon version to v1.22.2-weni --- go.mod | 2 ++ go.sum | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 4dfa955e6..2a1c98214 100644 --- a/go.mod +++ b/go.mod @@ -70,3 +70,5 @@ require ( google.golang.org/protobuf v1.28.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/nyaruka/gocommon => github.com/Ilhasoft/gocommon v1.22.2-weni diff --git a/go.sum b/go.sum index 54dadc46d..9a820e297 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/Ilhasoft/gocommon v1.22.2-weni h1:jkfipZyJHzc2SppjuQ8LeAC5rR+c0U+9xT/9nF1IP3A= +github.com/Ilhasoft/gocommon v1.22.2-weni/go.mod h1:g6/d9drZXDUrtRSPe2Kf8lTUS+baHt/0G0dwHq3qeIU= github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92 h1:4EgP6xLAdrD/TRlbSw4n2W6h68K2P3+R7lKqFoL5U9Q= github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92/go.mod h1:/+CnyD/DzHRnv2eRxrVbieRU/FIF6N0C+7oTtyUtCKk= github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa h1:lL66YnJWy1tHlhjSx8fXnpgmv8kQVYnI4ilbYpNB6Zs= @@ -91,8 +93,6 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.22.2 h1:iEusd0CijvYvhW+bEZ6LWYQuaXS4Oac5WH9W8iK6DEw= -github.com/nyaruka/gocommon v1.22.2/go.mod h1:g6/d9drZXDUrtRSPe2Kf8lTUS+baHt/0G0dwHq3qeIU= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/null v1.1.1 h1:kRy1Luj7jUHWEFqc2J6VXrKYi/beLEZdS1C7rA6vqTE= From 633bb38200a809ba4385cbc119800db361197dee Mon Sep 17 00:00:00 2001 From: Roberta Moreira <30378788+Robi9@users.noreply.github.com> Date: Thu, 5 Jan 2023 17:59:30 -0300 Subject: [PATCH 133/146] Update Dockerfile --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index a6172651d..5a08e40ea 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.17.5-alpine3.14 +FROM golang:1.18.9-alpine3.17 WORKDIR /app From 8fc7d8d687a13897ce5d897295c578d2ea81bdd0 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Mon, 10 Apr 2023 18:49:42 -0300 Subject: [PATCH 134/146] Use the new package to find out the mime type of attachments --- backends/rapidpro/msg.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/backends/rapidpro/msg.go b/backends/rapidpro/msg.go index b16bd2fd3..341395563 100644 --- a/backends/rapidpro/msg.go +++ b/backends/rapidpro/msg.go @@ -15,6 +15,7 @@ import ( "time" "github.com/buger/jsonparser" + "github.com/gabriel-vasile/mimetype" "github.com/pkg/errors" "mime" @@ -269,16 +270,23 @@ func downloadMediaToS3(ctx context.Context, b *backend, channel courier.Channel, } // first try getting our mime type from the first 300 bytes of our body - fileType, err := filetype.Match(body[:300]) - if fileType != filetype.Unknown { - mimeType = fileType.MIME.Value - extension = fileType.Extension - } else { - // if that didn't work, try from our extension - fileType = filetype.GetType(extension) - if fileType != filetype.Unknown { + fileType, _ := filetype.Match(body[:300]) + mimeT := mimetype.Detect(body) + + if fileType != filetype.Unknown || mimeT.String() != "application/octet-stream" { + if mimeT.String() == fileType.MIME.Type { mimeType = fileType.MIME.Value extension = fileType.Extension + } else if mimeT.String() != "application/zip" && mimeT.String() != "application/octet-stream" { + mimeType = mimeT.String() + extension = mimeT.Extension()[1:] + } else { + // if that didn't work, try from our extension + fileType = filetype.GetType(extension) + if fileType != filetype.Unknown { + mimeType = fileType.MIME.Value + extension = fileType.Extension + } } } From f0bb2c48c71789aa1b9f28354d6b4eba36543d23 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 14 Apr 2023 13:16:12 -0300 Subject: [PATCH 135/146] Fix maximum message size limits for WAC, FBA and IG. --- handlers/facebookapp/facebookapp.go | 13 ++++++++++--- handlers/facebookapp/facebookapp_test.go | 4 +++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index f5bd32a57..57f1633db 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -32,7 +32,9 @@ var ( configWACPhoneNumberID = "wac_phone_number_id" // max for the body - maxMsgLength = 1000 + maxMsgLengthIG = 1000 + maxMsgLengthFBA = 2000 + maxMsgLengthWAC = 4096 // Sticker ID substitutions stickerIDToEmoji = map[int64]string{ @@ -1012,7 +1014,12 @@ func (h *handler) sendFacebookInstagramMsg(ctx context.Context, msg courier.Msg) msgParts := make([]string, 0) if msg.Text() != "" { - msgParts = handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) + if msg.Channel().ChannelType() == "IG" { + msgParts = handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLengthIG) + } else { + msgParts = handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLengthFBA) + } + } // send each part and each attachment separately. we send attachments first as otherwise quick replies @@ -1245,7 +1252,7 @@ func (h *handler) sendCloudAPIWhatsappMsg(ctx context.Context, msg courier.Msg) msgParts := make([]string, 0) if msg.Text() != "" { - msgParts = handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLength) + msgParts = handlers.SplitMsgByChannel(msg.Channel(), msg.Text(), maxMsgLengthWAC) } qrs := msg.QuickReplies() diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 821fbf707..913e361f5 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -711,7 +711,9 @@ var SendTestCasesWAC = []ChannelSendTestCase{ func TestSending(t *testing.T) { // shorter max msg length for testing - maxMsgLength = 100 + maxMsgLengthFBA = 100 + maxMsgLengthIG = 100 + maxMsgLengthWAC = 100 var ChannelFBA = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "FBA", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) var ChannelIG = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "IG", "12345", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) var ChannelWAC = courier.NewMockChannel("8eb23e93-5ecb-45ba-b726-3b064e0c56ab", "WAC", "12345_ID", "", map[string]interface{}{courier.ConfigAuthToken: "a123"}) From 11c85dd292f84ea45f838ae961a9ca6934811b1b Mon Sep 17 00:00:00 2001 From: Robi9 Date: Mon, 17 Apr 2023 18:40:03 -0300 Subject: [PATCH 136/146] Add support for receiving contact type messages in WAC --- handlers/facebookapp/facebookapp.go | 24 +++++++++ handlers/facebookapp/facebookapp_test.go | 2 + .../facebookapp/testdata/wac/contactWAC.json | 51 +++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 handlers/facebookapp/testdata/wac/contactWAC.json diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 57f1633db..7ef0b5d97 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -188,6 +188,18 @@ type moPayload struct { Title string `json:"title"` } `json:"list_reply,omitempty"` } `json:"interactive,omitempty"` + Contacts []struct { + Name struct { + FirstName string `json:"first_name"` + LastName string `json:"last_name"` + FormattedName string `json:"formatted_name"` + } `json:"name"` + Phones []struct { + Phone string `json:"phone"` + WaID string `json:"wa_id"` + Type string `json:"type"` + } `json:"phones"` + } `json:"contacts"` } `json:"messages"` Statuses []struct { ID string `json:"id"` @@ -483,6 +495,18 @@ func (h *handler) processCloudWhatsAppPayload(ctx context.Context, channel couri text = msg.Interactive.ButtonReply.Title } else if msg.Type == "interactive" && msg.Interactive.Type == "list_reply" { text = msg.Interactive.ListReply.Title + } else if msg.Type == "contacts" { + + if len(msg.Contacts) == 0 { + return nil, nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, errors.New("no shared contact")) + } + + // put phones in a comma-separated string + var phones []string + for _, phone := range msg.Contacts[0].Phones { + phones = append(phones, phone.Phone) + } + text = strings.Join(phones, ", ") } else { // we received a message type we do not support. courier.LogRequestError(r, channel, fmt.Errorf("unsupported message type %s", msg.Type)) diff --git a/handlers/facebookapp/facebookapp_test.go b/handlers/facebookapp/facebookapp_test.go index 913e361f5..26c43c18e 100644 --- a/handlers/facebookapp/facebookapp_test.go +++ b/handlers/facebookapp/facebookapp_test.go @@ -278,6 +278,8 @@ var testCasesWAC = []ChannelHandleTestCase{ {Label: "Receive Valid Interactive List Reply Message", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/listReplyWAC.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, Text: Sp("Yes"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), PrepRequest: addValidSignatureWAC}, + {Label: "Receive Valid Contact Message", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/contactWAC.json")), Status: 200, Response: "Handled", NoQueueErrorCheck: true, NoInvalidChannelCheck: true, + Text: Sp("+1 415-858-6273, +1 415-858-6274"), URN: Sp("whatsapp:5678"), ExternalID: Sp("external_id"), Date: Tp(time.Date(2016, 1, 30, 1, 57, 9, 0, time.UTC)), PrepRequest: addValidSignatureWAC}, {Label: "Receive Invalid JSON", URL: wacReceiveURL, Data: "not json", Status: 400, Response: "unable to parse", PrepRequest: addValidSignatureWAC}, {Label: "Receive Invalid JSON", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/invalidFrom.json")), Status: 400, Response: "invalid whatsapp id", PrepRequest: addValidSignatureWAC}, {Label: "Receive Invalid JSON", URL: wacReceiveURL, Data: string(courier.ReadFile("./testdata/wac/invalidTimestamp.json")), Status: 400, Response: "invalid timestamp", PrepRequest: addValidSignatureWAC}, diff --git a/handlers/facebookapp/testdata/wac/contactWAC.json b/handlers/facebookapp/testdata/wac/contactWAC.json new file mode 100644 index 000000000..2157583eb --- /dev/null +++ b/handlers/facebookapp/testdata/wac/contactWAC.json @@ -0,0 +1,51 @@ +{ + "object": "whatsapp_business_account", + "entry": [ + { + "id": "8856996819413533", + "changes": [ + { + "value": { + "messaging_product": "whatsapp", + "metadata": { + "display_phone_number": "+250 788 123 200", + "phone_number_id": "12345" + }, + "contacts": [ + { + "profile": { + "name": "Kerry Fisher" + }, + "wa_id": "5678" + } + ],"messages": [ + { + "from": "5678", + "id": "external_id", + "timestamp": "1454119029", + "type": "contacts", + "contacts": [{ + "name": { + "first_name": "Test", + "last_name": "Contact", + "formatted_name": "Test Contact" + }, + "phones": [{ + "phone": "+1 415-858-6273", + "wa_id": "14158586273", + "type": "CELL" + }, + { + "phone": "+1 415-858-6274", + "wa_id": "14158586272", + "type": "CELL" + }] + }] + }] + }, + "field": "messages" + } + ] + } + ] + } \ No newline at end of file From 83e2f41e72f23ab6eef059f50f51ed95fb8e4586 Mon Sep 17 00:00:00 2001 From: Marcello Alexandre Date: Fri, 3 Mar 2023 10:02:36 -0300 Subject: [PATCH 137/146] Create build-courier-push-tag-sp-india-ire.yaml --- .../build-courier-push-tag-sp-india-ire.yaml | 93 +++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 .github/workflows/build-courier-push-tag-sp-india-ire.yaml diff --git a/.github/workflows/build-courier-push-tag-sp-india-ire.yaml b/.github/workflows/build-courier-push-tag-sp-india-ire.yaml new file mode 100644 index 000000000..5803bedcf --- /dev/null +++ b/.github/workflows/build-courier-push-tag-sp-india-ire.yaml @@ -0,0 +1,93 @@ +name: Build Courier in SP, India and Ireland (Push Tag) + +on: + push: + tags: + - '*.*.*-staging' + - '*.*.*' + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - name: Set variables + run: | + TAG="$( echo "${GITHUB_REF}" | cut -d'/' -f3 )" + if grep -qs -e '^.*.*-develop' <<< "${TAG}" ; then + echo "Found environment: DEVELOP - ${TAG}" + echo "ENVIRONMENT=develop" | tee -a "${GITHUB_ENV}" + exit 1 # stop action + elif grep -qs -e '^.*.*-staging' <<< "${TAG}" ; then + echo "Found environment: STAGING - ${TAG}" + echo "ENVIRONMENT=staging" | tee -a "${GITHUB_ENV}" + elif grep -qs -e '^.*.*' <<< "${TAG}" ; then + echo "No environment found, assuming: PRODUCTION - ${TAG}" + echo "ENVIRONMENT=production" | tee -a "${GITHUB_ENV}" + else + echo 'Not a valid tag. Skipping...' + exit 1 + fi + echo "TAG=$TAG" | tee -a "${GITHUB_ENV}" + VERSION="${TAG}" + echo "VERSION=${VERSION}" | tee -a "${GITHUB_ENV}" + echo "COMMIT_SHA=$GITHUB_SHA" | tee -a "${GITHUB_ENV}" + if [[ "${TAG}" == *"-india"* ]]; then + echo "IMAGE_TAG=${{ secrets.ECR_INDIA }}/courier:${TAG}" | tee -a "${GITHUB_ENV}" + elif [[ "${TAG}" == *"-ire"* ]]; then + echo "IMAGE_TAG=${{ secrets.ECR_IRE }}/rp-courier-rapidpro:${TAG}" | tee -a "${GITHUB_ENV}" + else + echo "IMAGE_TAG=${{ secrets.ECR_SP }}/push-courier:${TAG}" | tee -a "${GITHUB_ENV}" + fi + echo "IMAGE_SOURCE_URL=https://github.com/weni-ai/courier" | tee -a "${GITHUB_ENV}" + echo "MANIFESTS_REPOSITORY=Ilhasoft/kubernetes-manifests-platform" | tee -a "${GITHUB_ENV}" + echo "MANIFESTS_APPLICATION=weni-flows/courier" | tee -a "${GITHUB_ENV}" + echo "MANIFESTS_PATCH_TARGET=deployment.json" | tee -a "${GITHUB_ENV}" + + - name: Check out the repo + uses: actions/checkout@v3 + with: + ref: "${{env.GITHUB_SHA}}" + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Registry India + if: ${{ contains(env.TAG, '-india') }} + uses: docker/login-action@v2 + with: + registry: ${{ secrets.ECR_INDIA }} + username: ${{ secrets.AWS_ACCESS_KEY_ID_INDIA }} + password: ${{ secrets.AWS_SECRET_ACCESS_KEY_INDIA }} + + - name: Login to Registry Ireland + if: ${{ contains(env.TAG, '-ire') }} + uses: docker/login-action@v2 + with: + registry: ${{ secrets.ECR_IRE }} + username: ${{ secrets.AWS_ACCESS_KEY_ID_IRE }} + password: ${{ secrets.AWS_SECRET_ACCESS_KEY_IRE }} + + - name: Login to Registry SP + if: ${{ !contains(env.TAG, '-india') && !contains(env.TAG, '-ire') }} + uses: docker/login-action@v2 + with: + registry: ${{ secrets.ECR_SP }} + username: ${{ secrets.AWS_ACCESS_KEY_ID_SP }} + password: ${{ secrets.AWS_SECRET_ACCESS_KEY_SP }} + + - name: Build and push - Courier Image + uses: docker/build-push-action@v3 + with: + context: . + labels: | + tag=${{env.TAG}} + commit=${{env.COMMIT_SHA}} + repository=${{env.IMAGE_SOURCE_URL}} + file: docker/Dockerfile + push: true + tags: "${{env.IMAGE_TAG}}" + no-cache: true + From 8f052975ca9715e8590969cd5483dcdd8b50cbc6 Mon Sep 17 00:00:00 2001 From: Paulo Bernardo Date: Thu, 27 Apr 2023 15:12:32 -0300 Subject: [PATCH 138/146] fix: remove last seen on --- backend.go | 4 ---- backends/rapidpro/backend.go | 6 ------ sender.go | 12 ------------ test.go | 5 ----- 4 files changed, 27 deletions(-) diff --git a/backend.go b/backend.go index 4755a6d69..9a740e7ec 100644 --- a/backend.go +++ b/backend.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" "github.com/gomodule/redigo/redis" "github.com/nyaruka/gocommon/urns" @@ -33,9 +32,6 @@ type Backend interface { // GetContact returns (or creates) the contact for the passed in channel and URN GetContact(context context.Context, channel Channel, urn urns.URN, auth string, name string) (Contact, error) - // UpdateContactLastSeenOn updates last seen on (and modified on) on the passed in contact - UpdateContactLastSeenOn(ctx context.Context, contactUUID ContactUUID, lastSeenOn time.Time) error - // AddURNtoContact adds a URN to the passed in contact AddURNtoContact(context context.Context, channel Channel, contact Contact, urn urns.URN) (urns.URN, error) diff --git a/backends/rapidpro/backend.go b/backends/rapidpro/backend.go index e19e28da0..62c83809f 100644 --- a/backends/rapidpro/backend.go +++ b/backends/rapidpro/backend.go @@ -61,12 +61,6 @@ func (b *backend) GetContact(ctx context.Context, c courier.Channel, urn urns.UR return contactForURN(ctx, b, dbChannel.OrgID_, dbChannel, urn, auth, name) } -// UpdateContactLastSeenOn updates last seen on (and modified on) on the passed in contact -func (b *backend) UpdateContactLastSeenOn(ctx context.Context, contactUUID courier.ContactUUID, lastSeenOn time.Time) error { - _, err := b.db.ExecContext(ctx, `UPDATE contacts_contact SET last_seen_on = $2, modified_on = NOW() WHERE uuid = $1`, contactUUID.String(), lastSeenOn) - return err -} - // AddURNtoContact adds a URN to the passed in contact func (b *backend) AddURNtoContact(ctx context.Context, c courier.Channel, contact courier.Contact, urn urns.URN) (urns.URN, error) { tx, err := b.db.BeginTxx(ctx, nil) diff --git a/sender.go b/sender.go index aa1e6e464..094e1b437 100644 --- a/sender.go +++ b/sender.go @@ -215,18 +215,6 @@ func (w *Sender) sendMessage(msg Msg) { log.WithField("elapsed", duration).Info("msg sent") analytics.Gauge(fmt.Sprintf("courier.msg_send_%s", msg.Channel().ChannelType()), secondDuration) } - - // update last seen on if message is no error and no fail - if status.Status() != MsgErrored && status.Status() != MsgFailed { - ctt, err := w.foreman.server.Backend().GetContact(context.Background(), msg.Channel(), msg.URN(), "", "") - if err != nil { - log.WithError(err).Info("error getting contact") - } - err = w.foreman.server.Backend().UpdateContactLastSeenOn(context.Background(), ctt.UUID(), time.Now()) - if err != nil { - log.WithError(err).Info("error updating contact last seen on") - } - } } // we allot 10 seconds to write our status to the db diff --git a/test.go b/test.go index 6c173d66d..86240b5df 100644 --- a/test.go +++ b/test.go @@ -286,11 +286,6 @@ func (mb *MockBackend) GetContact(ctx context.Context, channel Channel, urn urns return contact, nil } -// UpdateContactLastSeenOn updates last seen on (and modified on) on the passed in contact -func (mb *MockBackend) UpdateContactLastSeenOn(ctx context.Context, contactUUID ContactUUID, lastSeenOn time.Time) error { - return nil -} - // AddURNtoContact adds a URN to the passed in contact func (mb *MockBackend) AddURNtoContact(context context.Context, channel Channel, contact Contact, urn urns.URN) (urns.URN, error) { mb.contacts[urn] = contact From c0b16549521528039b70b5a9c3edbe7a5dbe64a2 Mon Sep 17 00:00:00 2001 From: Roberta Moreira <30378788+Robi9@users.noreply.github.com> Date: Thu, 27 Apr 2023 15:34:10 -0300 Subject: [PATCH 139/146] Update WENI-CHANGELOG.md --- WENI-CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/WENI-CHANGELOG.md b/WENI-CHANGELOG.md index 478f6be84..6dd4bf0f6 100644 --- a/WENI-CHANGELOG.md +++ b/WENI-CHANGELOG.md @@ -1,3 +1,8 @@ +1.5.1-courier-7.2.0 +---------- + * Fix WAC handler + * Fix: Remove last seen on + 1.5.0-courier-7.2.0 ---------- * Merge tag v7.2.0 from nyaruka into our 1.4.5-courier-7.1.0 From bef2a3a2d011b904096cc121c49356779be44e3f Mon Sep 17 00:00:00 2001 From: Robi9 Date: Tue, 2 May 2023 15:14:47 -0300 Subject: [PATCH 140/146] Fix changelog version --- WENI-CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WENI-CHANGELOG.md b/WENI-CHANGELOG.md index 6dd4bf0f6..4f0a4433a 100644 --- a/WENI-CHANGELOG.md +++ b/WENI-CHANGELOG.md @@ -1,4 +1,4 @@ -1.5.1-courier-7.2.0 +1.5.1-courier-7.4.0 ---------- * Fix WAC handler * Fix: Remove last seen on From 84d9e0a4b2df17eb46ed3e1571422f7941412794 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Mon, 27 Feb 2023 17:14:28 -0300 Subject: [PATCH 141/146] Add module to send webhooks --- handlers/webhook.go | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 handlers/webhook.go diff --git a/handlers/webhook.go b/handlers/webhook.go new file mode 100644 index 000000000..ca648f5a6 --- /dev/null +++ b/handlers/webhook.go @@ -0,0 +1,31 @@ +package handlers + +import ( + "fmt" + "net/http" + + "github.com/nyaruka/courier" + "github.com/nyaruka/courier/utils" +) + +func SendWebhooks(channel courier.Channel, r *http.Request, webhook interface{}) error { + webhookURL, ok := webhook.(map[string]interface{}) + if !ok { + return fmt.Errorf("conversion error") + } + req, err := http.NewRequest(http.MethodPost, webhookURL["url"].(string), r.Body) + if err != nil { + return err + } + + resp, err := utils.MakeHTTPRequest(req) + if err != nil { + return err + } + + if resp.StatusCode/100 != 2 { + return err + } + + return nil +} From 85cefa47bf158474242c97776ae0a26b68aeb39e Mon Sep 17 00:00:00 2001 From: Robi9 Date: Mon, 27 Feb 2023 17:40:00 -0300 Subject: [PATCH 142/146] Add verification and forwarding of webhooks for WAC/WA --- handlers/facebookapp/facebookapp.go | 8 +++++++- handlers/whatsapp/whatsapp.go | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 7ef0b5d97..46898c198 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -421,7 +421,13 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h events, data, err = h.processFacebookInstagramPayload(ctx, channel, payload, w, r) } else { events, data, err = h.processCloudWhatsAppPayload(ctx, channel, payload, w, r) - + webhook := channel.ConfigForKey("webhook", nil) + if webhook != nil { + er := handlers.SendWebhooks(channel, r, webhook) + if er != nil { + return nil, er + } + } } if err != nil { diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index dc41fd06e..ef75f25e6 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -337,6 +337,14 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h data = append(data, courier.NewStatusData(event)) } + webhook := channel.ConfigForKey("webhook", nil) + if webhook != nil { + err = handlers.SendWebhooks(channel, r, webhook) + if err != nil { + return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + } + } + return events, courier.WriteDataResponse(ctx, w, http.StatusOK, "Events Handled", data) } From 0e289f7e412e5e50ae1974163e7fde909a4b1beb Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 2 Mar 2023 12:24:41 -0300 Subject: [PATCH 143/146] Add method field and headers in webhook --- handlers/webhook.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/handlers/webhook.go b/handlers/webhook.go index ca648f5a6..bea52ca76 100644 --- a/handlers/webhook.go +++ b/handlers/webhook.go @@ -8,21 +8,26 @@ import ( "github.com/nyaruka/courier/utils" ) -func SendWebhooks(channel courier.Channel, r *http.Request, webhook interface{}) error { - webhookURL, ok := webhook.(map[string]interface{}) +func SendWebhooks(channel courier.Channel, r *http.Request, configWebhook interface{}) error { + webhook, ok := configWebhook.(map[string]interface{}) if !ok { return fmt.Errorf("conversion error") } - req, err := http.NewRequest(http.MethodPost, webhookURL["url"].(string), r.Body) - if err != nil { - return err + + method := webhook["method"].(string) + if method == "" { + method = "POST" } - resp, err := utils.MakeHTTPRequest(req) - if err != nil { - return err + req, _ := http.NewRequest(method, webhook["url"].(string), r.Body) + + headers := webhook["headers"].(map[string]interface{}) + for name, value := range headers { + req.Header.Set(name, value.(string)) } + resp, err := utils.MakeHTTPRequest(req) + if resp.StatusCode/100 != 2 { return err } From e955fc6eb9afd64e515b6cbf6241915853ac10cd Mon Sep 17 00:00:00 2001 From: Robi9 Date: Thu, 27 Apr 2023 12:02:42 -0300 Subject: [PATCH 144/146] Improve URL verification for webhooks --- handlers/facebookapp/facebookapp.go | 2 +- handlers/webhook.go | 11 +++++++++++ handlers/whatsapp/whatsapp.go | 6 +++--- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/handlers/facebookapp/facebookapp.go b/handlers/facebookapp/facebookapp.go index 46898c198..a0364f77e 100644 --- a/handlers/facebookapp/facebookapp.go +++ b/handlers/facebookapp/facebookapp.go @@ -425,7 +425,7 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h if webhook != nil { er := handlers.SendWebhooks(channel, r, webhook) if er != nil { - return nil, er + courier.LogRequestError(r, channel, fmt.Errorf("could not send webhook: %s", er)) } } } diff --git a/handlers/webhook.go b/handlers/webhook.go index bea52ca76..c21a22a29 100644 --- a/handlers/webhook.go +++ b/handlers/webhook.go @@ -3,6 +3,7 @@ package handlers import ( "fmt" "net/http" + "net/url" "github.com/nyaruka/courier" "github.com/nyaruka/courier/utils" @@ -14,6 +15,16 @@ func SendWebhooks(channel courier.Channel, r *http.Request, configWebhook interf return fmt.Errorf("conversion error") } + // check if url is valid + _, err := url.ParseRequestURI(webhook["url"].(string)) + if err != nil { + return err + } + u, err := url.Parse(webhook["url"].(string)) + if err != nil || u.Scheme == "" || u.Host == "" { + return fmt.Errorf("invalid url %s", err) + } + method := webhook["method"].(string) if method == "" { method = "POST" diff --git a/handlers/whatsapp/whatsapp.go b/handlers/whatsapp/whatsapp.go index ef75f25e6..784426e7a 100644 --- a/handlers/whatsapp/whatsapp.go +++ b/handlers/whatsapp/whatsapp.go @@ -339,9 +339,9 @@ func (h *handler) receiveEvent(ctx context.Context, channel courier.Channel, w h webhook := channel.ConfigForKey("webhook", nil) if webhook != nil { - err = handlers.SendWebhooks(channel, r, webhook) - if err != nil { - return nil, handlers.WriteAndLogRequestError(ctx, h, channel, w, r, err) + er := handlers.SendWebhooks(channel, r, webhook) + if er != nil { + courier.LogRequestError(r, channel, fmt.Errorf("could not send webhook: %s", er)) } } From b0509f721c3b55a69e42346eb2c7195a9de9f809 Mon Sep 17 00:00:00 2001 From: Roberta Moreira <30378788+Robi9@users.noreply.github.com> Date: Thu, 4 May 2023 19:09:17 -0300 Subject: [PATCH 145/146] Update WENI-CHANGELOG.md for 1.5.2-courier-7.4.0 --- WENI-CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/WENI-CHANGELOG.md b/WENI-CHANGELOG.md index 4f0a4433a..3bb4a141d 100644 --- a/WENI-CHANGELOG.md +++ b/WENI-CHANGELOG.md @@ -1,3 +1,7 @@ +1.5.2-courier-7.4.0 +---------- + * Add module to send webhooks for WAC and WA + 1.5.1-courier-7.4.0 ---------- * Fix WAC handler From fbbeb5a1edb60365f3c634a782021672bcfe44f9 Mon Sep 17 00:00:00 2001 From: Robi9 Date: Fri, 5 May 2023 16:29:25 -0300 Subject: [PATCH 146/146] Fix test TestMsgSuite/TestWriteAttachment --- backends/rapidpro/backend_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backends/rapidpro/backend_test.go b/backends/rapidpro/backend_test.go index c84f5c8b1..d5e7faf8a 100644 --- a/backends/rapidpro/backend_test.go +++ b/backends/rapidpro/backend_test.go @@ -962,14 +962,14 @@ func (ts *BackendTestSuite) TestWriteAttachment() { content := "" switch r.URL.Path { case "/test.jpg": - content = "malformedjpegbody" + content = "\xFF\xD8\xFF" case "/giffy": content = "GIF87aandstuff" case "/header": w.Header().Add("Content-Type", "image/png") - content = "nothingbody" + content = "\x89\x50\x4E\x47\x0D\x0A\x1A\x0A" default: content = "unknown"