diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c030223d9..2d3baaaa9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,11 +57,11 @@ jobs: uses: actions/checkout@v1 - name: Fetch GoFlow docs - # for now just grab en_US/ docs and bundle as docs/ + # for backward compatibility, English docs are copied to root of docs directory run: | - GOFLOW_VERSION=$(grep goflow go.mod | cut -d" " -f2) - curl https://codeload.github.com/nyaruka/goflow/tar.gz/$GOFLOW_VERSION | tar --wildcards --strip=2 -zx "*/docs/en_US/*" - mv en_US docs + GOFLOW_VERSION=$(grep goflow go.mod | cut -d" " -f2 | cut -c2-) + curl https://codeload.github.com/nyaruka/goflow/tar.gz/v$GOFLOW_VERSION | tar --wildcards --strip=1 -zx "goflow-${GOFLOW_VERSION}/docs/*" + cp ./docs/en-us/*.* docs/ - name: Install Go uses: actions/setup-go@v1 diff --git a/.gitignore b/.gitignore index 036beb944..494792c01 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ docs/* docs .DS_Store +_storage/ # Test binary, build with `go test -c` *.test diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d155d3d9..38ce20750 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,67 @@ +v5.7.14 +---------- + * Stop writing is_blocked and is_stopped + +v5.7.13 +---------- + * Read from contact.status intead of is_stopped/is_blocked + * Implement saving of zendesk ticket files as attachments + * Abstract S3 code so tests and dev envs can use file storage + +v5.7.12 +---------- + * Fix inserting channel logs and add test + +v5.7.11 +---------- + * Always write contact.status when writing is_blocked or is_stopped + * Convert IVR code to use goflow's httpx package + +v5.7.10 +---------- + * Tweak goreleaser config to include subdirectories inside docs folder + +v5.7.9 +---------- + * Update to goflow v0.101.2 + * Bundle localized goflow docs in release + +v5.7.8 +---------- + * Recalculate event fires for campaign events based on last_seen_on + +v5.7.7 +---------- + * Update to latest goflow v0.100.0 + +v5.7.6 +---------- + * Remove protection for overwriting last_seen_on with older values + +v5.7.5 +---------- + * Update last_seen_on when handling certain channel events + * Update last_seen_on when we receive a message from a contact + +v5.7.4 +---------- + * Fail outgoing messages for suspended orgs + * Refresh groups as well as fields for contact query parsing + +v5.7.3 +---------- + * Update to goflow v0.99.0 + +v5.7.2 +---------- + * Update to latest goflow v0.98.0 + * Render rich errors with code and extra field + +v5.7.1 +---------- + * Update to latest goflow v0.96.0 + * Add loop protection by passing session history to new flow action triggers + v5.7.0 ---------- * Set user and origin on manual triggers diff --git a/config/config.go b/config/config.go index ae77583a6..f6b6b4c8d 100644 --- a/config/config.go +++ b/config/config.go @@ -94,8 +94,8 @@ func NewMailroomConfig() *Config { S3MediaPrefix: "/media/", S3DisableSSL: false, S3ForcePathStyle: false, - AWSAccessKeyID: "missing_aws_access_key_id", - AWSSecretAccessKey: "missing_aws_secret_access_key", + AWSAccessKeyID: "", + AWSSecretAccessKey: "", RetryPendingMessages: true, diff --git a/courier/courier.go b/courier/courier.go index dc242bdf7..ac700f376 100644 --- a/courier/courier.go +++ b/courier/courier.go @@ -52,8 +52,8 @@ func QueueMessages(rc redis.Conn, msgs []*models.Msg) error { } for _, msg := range msgs { - // no channel, continue - if msg.ChannelUUID() == "" { + // ignore any message without a channel or already marked as failed (maybe org is suspended) + if msg.ChannelUUID() == "" || msg.Status() == models.MsgStatusFailed { continue } diff --git a/courier/courier_test.go b/courier/courier_test.go new file mode 100644 index 000000000..8a0d05661 --- /dev/null +++ b/courier/courier_test.go @@ -0,0 +1,168 @@ +package courier_test + +import ( + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/mailroom/courier" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/testsuite" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type msgSpec struct { + ChannelUUID assets.ChannelUUID + Text string + ContactID models.ContactID + URN urns.URN + URNID models.URNID + Failed bool +} + +func createMsg(t *testing.T, m msgSpec) *models.Msg { + ctx := testsuite.CTX() + db := testsuite.DB() + db.MustExec(`UPDATE orgs_org SET is_suspended = $1 WHERE id = $2`, m.Failed, models.Org1) + + oa, err := models.GetOrgAssetsWithRefresh(ctx, db, models.Org1, models.RefreshOrg) + require.NoError(t, err) + + channel := oa.ChannelByUUID(m.ChannelUUID) + + flowMsg := flows.NewMsgOut(m.URN, channel.ChannelReference(), m.Text, nil, nil, nil, flows.NilMsgTopic) + msg, err := models.NewOutgoingMsg(oa.Org(), channel, m.ContactID, flowMsg, time.Now()) + require.NoError(t, err) + return msg +} + +func TestQueueMessages(t *testing.T) { + db := testsuite.DB() + rc := testsuite.RC() + testsuite.Reset() + models.FlushCache() + + // convert the Twitter channel to be an Android channel + db.MustExec(`UPDATE channels_channel SET name = 'Android', channel_type = 'A' WHERE id = $1`, models.TwitterChannelID) + androidChannelUUID := models.TwitterChannelUUID + + tests := []struct { + Description string + Msgs []msgSpec + QueueSizes map[string]int + }{ + { + Description: "2 queueable messages", + Msgs: []msgSpec{ + { + ChannelUUID: models.TwilioChannelUUID, + Text: "normal msg 1", + ContactID: models.CathyID, + URN: urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)), + URNID: models.CathyURNID, + }, + { + ChannelUUID: models.TwilioChannelUUID, + Text: "normal msg 2", + ContactID: models.CathyID, + URN: urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)), + URNID: models.CathyURNID, + }, + }, + QueueSizes: map[string]int{ + "msgs:74729f45-7f29-4868-9dc4-90e491e3c7d8|10/0": 2, + }, + }, + { + Description: "1 queueable message and 1 failed", + Msgs: []msgSpec{ + { + ChannelUUID: models.TwilioChannelUUID, + Text: "normal msg 1", + ContactID: models.CathyID, + URN: urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)), + URNID: models.CathyURNID, + Failed: true, + }, + { + ChannelUUID: models.TwilioChannelUUID, + Text: "normal msg 1", + ContactID: models.CathyID, + URN: urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)), + URNID: models.CathyURNID, + }, + }, + QueueSizes: map[string]int{ + "msgs:74729f45-7f29-4868-9dc4-90e491e3c7d8|10/0": 1, + }, + }, + { + Description: "1 queueable message and 1 Android", + Msgs: []msgSpec{ + { + ChannelUUID: androidChannelUUID, + Text: "normal msg 1", + ContactID: models.CathyID, + URN: urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)), + URNID: models.CathyURNID, + }, + { + ChannelUUID: models.TwilioChannelUUID, + Text: "normal msg 1", + ContactID: models.CathyID, + URN: urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)), + URNID: models.CathyURNID, + }, + }, + QueueSizes: map[string]int{ + "msgs:74729f45-7f29-4868-9dc4-90e491e3c7d8|10/0": 1, + }, + }, + { + Description: "0 messages", + Msgs: []msgSpec{}, + QueueSizes: map[string]int{ + "msgs:74729f45-7f29-4868-9dc4-90e491e3c7d8|10/0": 0, + }, + }, + } + + for _, tc := range tests { + msgs := make([]*models.Msg, len(tc.Msgs)) + for i := range tc.Msgs { + msgs[i] = createMsg(t, tc.Msgs[i]) + } + + rc.Do("FLUSHDB") + courier.QueueMessages(rc, msgs) + + for queueKey, size := range tc.QueueSizes { + if size == 0 { + result, err := rc.Do("ZCARD", queueKey) + require.NoError(t, err) + assert.Equal(t, size, int(result.(int64))) + } else { + result, err := rc.Do("ZPOPMAX", queueKey) + require.NoError(t, err) + + results := result.([]interface{}) + assert.Equal(t, 2, len(results)) // result is (item, score) + + batchJSON := results[0].([]byte) + var batch []map[string]interface{} + err = json.Unmarshal(batchJSON, &batch) + require.NoError(t, err) + + assert.Equal(t, size, len(batch)) + } + } + } + + testsuite.Reset() +} diff --git a/docker/Dockerfile b/docker/Dockerfile index c303dcd53..30420921a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -11,8 +11,8 @@ RUN addgroup -S golang \ COPY . . -RUN curl https://codeload.github.com/nyaruka/goflow/tar.gz/$(grep goflow go.mod | cut -d" " -f2) | tar --wildcards --strip=1 -zx "*/docs/*" -RUN cp docs/en_US/completion.json docs/completion.json && cp docs/en_US/functions.json docs/functions.json +RUN export GOFLOW_VERSION=$(grep goflow go.mod | cut -d" " -f2 | cut -c2-) && curl https://codeload.github.com/nyaruka/goflow/tar.gz/v$GOFLOW_VERSION | tar --wildcards --strip=1 -zx "goflow-${GOFLOW_VERSION}/docs/*" +RUN cp ./docs/en-us/*.* docs/ RUN go install -v ./cmd/... diff --git a/go.mod b/go.mod index 1f3704e3d..4f315d351 100644 --- a/go.mod +++ b/go.mod @@ -18,8 +18,8 @@ require ( github.com/lib/pq v1.4.0 github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.2.0 - github.com/nyaruka/goflow v0.95.1 + github.com/nyaruka/gocommon v1.3.0 + github.com/nyaruka/goflow v0.102.1 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d github.com/nyaruka/null v1.2.0 @@ -28,7 +28,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 - github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 + github.com/shopspring/decimal v1.2.0 github.com/sirupsen/logrus v1.5.0 github.com/stretchr/testify v1.5.1 gopkg.in/go-playground/validator.v9 v9.31.0 diff --git a/go.sum b/go.sum index 7ff223348..dc6cc1e92 100644 --- a/go.sum +++ b/go.sum @@ -128,10 +128,10 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.2.0 h1:gCmVCXYZFwKDMqQj8R1jNlK+7a06khKFq3zX8fBBbzw= -github.com/nyaruka/gocommon v1.2.0/go.mod h1:9Y21Fd6iZXDLHWTRiZAc6b4LQSCi6HEEQK4SB45Yav4= -github.com/nyaruka/goflow v0.95.1 h1:FmYlZH4+mM+5Xft1BxUG97TIWx4eDqZk0Dbnh86Qqs4= -github.com/nyaruka/goflow v0.95.1/go.mod h1:ttUI9hd9ufGU/3EqIst6Hdc68imMG1utlqcELbYrtOQ= +github.com/nyaruka/gocommon v1.3.0 h1:IqaPT4KQ2oVq/2Ivp/c+RVCs8v71+RzPU2VhMoRrgpU= +github.com/nyaruka/gocommon v1.3.0/go.mod h1:w7lKxIkm/qLAoO9Y3aI1LV7EiYogn6+1C8MTEjxTC9M= +github.com/nyaruka/goflow v0.102.1 h1:7QX2jTwV7uIbaGnkkpmB+ao+E7Cmyar9g7sRQH4Bu3M= +github.com/nyaruka/goflow v0.102.1/go.mod h1:wuvXZTs6a6S1rjSRLaQGVxDfKomDJ/1XQoLXCqFekK4= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d h1:hyp9u36KIwbTCo2JAJ+TuJcJBc+UZzEig7RI/S5Dvkc= @@ -170,9 +170,8 @@ github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shopspring/decimal v0.0.0-20180319170823-2df3e6ddaf6e/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= -github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= +github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= diff --git a/goflow/engine_test.go b/goflow/engine_test.go index a00e68dff..77bfa1dab 100644 --- a/goflow/engine_test.go +++ b/goflow/engine_test.go @@ -4,9 +4,9 @@ import ( "net/http" "testing" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/mailroom/goflow" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" diff --git a/goflow/http.go b/goflow/http.go index a37cf1032..1dfb85800 100644 --- a/goflow/http.go +++ b/goflow/http.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "github.com/nyaruka/goflow/utils/httpx" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/mailroom/config" ) diff --git a/goflow/modifiers.go b/goflow/modifiers.go index 69cac2275..7194c873d 100644 --- a/goflow/modifiers.go +++ b/goflow/modifiers.go @@ -5,7 +5,7 @@ import ( "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/goflow/flows/actions/modifiers" + "github.com/nyaruka/goflow/flows/modifiers" "github.com/pkg/errors" ) diff --git a/goreleaser.yml b/goreleaser.yml index d4f42d28a..63c28e3e9 100644 --- a/goreleaser.yml +++ b/goreleaser.yml @@ -11,4 +11,4 @@ archives: - files: - LICENSE - README.md - - docs/* + - docs/**/* diff --git a/hooks/airtime_transferred_test.go b/hooks/airtime_transferred_test.go index e69314d9f..5970fa399 100644 --- a/hooks/airtime_transferred_test.go +++ b/hooks/airtime_transferred_test.go @@ -4,9 +4,9 @@ import ( "strings" "testing" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/actions" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" diff --git a/hooks/campaigns.go b/hooks/campaigns.go index 35cb0fc79..225d0e724 100644 --- a/hooks/campaigns.go +++ b/hooks/campaigns.go @@ -52,6 +52,10 @@ func (h *UpdateCampaignEventsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *r continue } fieldChanges[field.ID()] = true + + case *events.MsgReceivedEvent: + field := oa.FieldByKey(models.LastSeenOnKey) + fieldChanges[field.ID()] = true } } diff --git a/hooks/campaigns_test.go b/hooks/campaigns_test.go index e8bf87f91..634a8696b 100644 --- a/hooks/campaigns_test.go +++ b/hooks/campaigns_test.go @@ -17,13 +17,20 @@ func TestCampaigns(t *testing.T) { doctors := assets.NewGroupReference(models.DoctorsGroupUUID, "Doctors") joined := assets.NewFieldReference("joined", "Joined") - // insert an event on our campaign that is based on created on + // insert an event on our campaign that is based on created_on testsuite.DB().MustExec( `INSERT INTO campaigns_campaignevent(is_active, created_on, modified_on, uuid, "offset", unit, event_type, delivery_hour, campaign_id, created_by_id, modified_by_id, flow_id, relative_to_id, start_mode) VALUES(TRUE, NOW(), NOW(), $1, 1000, 'W', 'F', -1, $2, 1, 1, $3, $4, 'I')`, uuids.New(), models.DoctorRemindersCampaignID, models.FavoritesFlowID, models.CreatedOnFieldID) + // insert an event on our campaign that is based on last_seen_on + testsuite.DB().MustExec( + `INSERT INTO campaigns_campaignevent(is_active, created_on, modified_on, uuid, "offset", unit, event_type, delivery_hour, + campaign_id, created_by_id, modified_by_id, flow_id, relative_to_id, start_mode) + VALUES(TRUE, NOW(), NOW(), $1, 2, 'D', 'F', -1, $2, 1, 1, $3, $4, 'I')`, + uuids.New(), models.DoctorRemindersCampaignID, models.FavoritesFlowID, models.LastSeenOnFieldID) + // init their values testsuite.DB().MustExec( `update contacts_contact set fields = fields - '8c1c1256-78d6-4a5b-9f1c-1761d5728251' @@ -35,7 +42,10 @@ func TestCampaigns(t *testing.T) { WHERE id = $1`, models.BobID) tcs := []HookTestCase{ - HookTestCase{ + { + Msgs: ContactMsgMap{ + models.CathyID: flows.NewMsgIn(flows.MsgUUID(uuids.New()), models.CathyURN, nil, "Hi there", nil), + }, Actions: ContactActionMap{ models.CathyID: []flows.Action{ actions.NewRemoveContactGroups(newActionUUID(), []*assets.GroupReference{doctors}, false), @@ -58,7 +68,7 @@ func TestCampaigns(t *testing.T) { { SQL: `select count(*) FROM campaigns_eventfire WHERE contact_id = $1`, Args: []interface{}{models.CathyID}, - Count: 1, + Count: 2, }, { SQL: `select count(*) FROM campaigns_eventfire WHERE contact_id = $1`, diff --git a/hooks/contact_groups_changed.go b/hooks/contact_groups_changed.go index 77784fb05..6bf95185b 100644 --- a/hooks/contact_groups_changed.go +++ b/hooks/contact_groups_changed.go @@ -100,7 +100,7 @@ func handleContactGroupsChanged(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool // add our add event scene.AppendToEventPreCommitHook(commitGroupChangesHook, hookEvent) scene.AppendToEventPreCommitHook(updateCampaignEventsHook, hookEvent) - scene.AppendToEventPreCommitHook(contactModifiedHook, scene.Contact().ID()) + scene.AppendToEventPreCommitHook(contactModifiedHook, scene.ContactID()) } // add each of our groups @@ -123,7 +123,7 @@ func handleContactGroupsChanged(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool scene.AppendToEventPreCommitHook(commitGroupChangesHook, hookEvent) scene.AppendToEventPreCommitHook(updateCampaignEventsHook, hookEvent) - scene.AppendToEventPreCommitHook(contactModifiedHook, scene.Contact().ID()) + scene.AppendToEventPreCommitHook(contactModifiedHook, scene.ContactID()) } return nil diff --git a/hooks/contact_last_seen.go b/hooks/contact_last_seen.go new file mode 100644 index 000000000..e92362ea1 --- /dev/null +++ b/hooks/contact_last_seen.go @@ -0,0 +1,33 @@ +package hooks + +import ( + "context" + + "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/mailroom/models" + + "github.com/gomodule/redigo/redis" + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" +) + +// ContactLastSeenHook is our hook for contact changes that require an update to last_seen_on +type ContactLastSeenHook struct{} + +var contactLastSeenHook = &ContactLastSeenHook{} + +// Apply squashes and updates modified_on on all the contacts passed in +func (h *ContactLastSeenHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *models.OrgAssets, scenes map[*models.Scene][]interface{}) error { + + for scene, evts := range scenes { + lastEvent := evts[len(evts)-1].(flows.Event) + lastSeenOn := lastEvent.CreatedOn() + + err := models.UpdateContactLastSeenOn(ctx, tx, scene.ContactID(), lastSeenOn) + if err != nil { + return errors.Wrapf(err, "error updating last_seen_on on contacts") + } + } + + return nil +} diff --git a/hooks/contact_modified.go b/hooks/contact_modified.go index 9f1b651de..299d1d43d 100644 --- a/hooks/contact_modified.go +++ b/hooks/contact_modified.go @@ -3,9 +3,10 @@ package hooks import ( "context" + "github.com/nyaruka/mailroom/models" + "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" - "github.com/nyaruka/mailroom/models" "github.com/pkg/errors" ) @@ -16,8 +17,9 @@ var contactModifiedHook = &ContactModifiedHook{} // Apply squashes and updates modified_on on all the contacts passed in func (h *ContactModifiedHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *models.OrgAssets, scenes map[*models.Scene][]interface{}) error { - // our list of contact ids + // our lists of contact ids contactIDs := make([]models.ContactID, 0, len(scenes)) + for scene := range scenes { contactIDs = append(contactIDs, scene.ContactID()) } diff --git a/hooks/contact_status_changed_test.go b/hooks/contact_status_changed_test.go index f56156b44..9cb5f13cf 100644 --- a/hooks/contact_status_changed_test.go +++ b/hooks/contact_status_changed_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/goflow/flows/actions/modifiers" + "github.com/nyaruka/goflow/flows/modifiers" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" ) @@ -14,46 +14,45 @@ func TestContactStatusChanged(t *testing.T) { db := testsuite.DB() // make sure cathyID contact is active - db.Exec(`UPDATE contacts_contact SET is_blocked = FALSE WHERE id = $1`, models.CathyID) - db.Exec(`UPDATE contacts_contact SET is_stopped = FALSE WHERE id = $1`, models.CathyID) + db.Exec(`UPDATE contacts_contact SET status = 'A' WHERE id = $1`, models.CathyID) tcs := []HookTestCase{ - HookTestCase{ + { Modifiers: ContactModifierMap{ models.CathyID: []flows.Modifier{modifiers.NewStatus(flows.ContactStatusBlocked)}, }, SQLAssertions: []SQLAssertion{ - SQLAssertion{ - SQL: `select count(*) from contacts_contact where id = $1 AND is_blocked = TRUE`, + { + SQL: `select count(*) from contacts_contact where id = $1 AND status = 'B'`, Args: []interface{}{models.CathyID}, Count: 1, }, }, }, - HookTestCase{ + { Modifiers: ContactModifierMap{ models.CathyID: []flows.Modifier{modifiers.NewStatus(flows.ContactStatusStopped)}, }, SQLAssertions: []SQLAssertion{ - SQLAssertion{ - SQL: `select count(*) from contacts_contact where id = $1 AND is_stopped = TRUE`, + { + SQL: `select count(*) from contacts_contact where id = $1 AND status = 'S'`, Args: []interface{}{models.CathyID}, Count: 1, }, }, }, - HookTestCase{ + { Modifiers: ContactModifierMap{ models.CathyID: []flows.Modifier{modifiers.NewStatus(flows.ContactStatusActive)}, }, SQLAssertions: []SQLAssertion{ - SQLAssertion{ - SQL: `select count(*) from contacts_contact where id = $1 AND is_stopped = FALSE`, + { + SQL: `select count(*) from contacts_contact where id = $1 AND status = 'A'`, Args: []interface{}{models.CathyID}, Count: 1, }, - SQLAssertion{ - SQL: `select count(*) from contacts_contact where id = $1 AND is_blocked = FALSE`, + { + SQL: `select count(*) from contacts_contact where id = $1 AND status = 'A'`, Args: []interface{}{models.CathyID}, Count: 1, }, diff --git a/hooks/contact_urns_changed.go b/hooks/contact_urns_changed.go index 59ddf4bbc..80ea40d49 100644 --- a/hooks/contact_urns_changed.go +++ b/hooks/contact_urns_changed.go @@ -55,7 +55,7 @@ func handleContactURNsChanged(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, // add our callback scene.AppendToEventPreCommitHook(commitURNChangesHook, change) - scene.AppendToEventPreCommitHook(contactModifiedHook, scene.Contact().ID()) + scene.AppendToEventPreCommitHook(contactModifiedHook, scene.ContactID()) return nil } diff --git a/hooks/hooks_test.go b/hooks/hooks_test.go index 0100f2f41..0a3a77f63 100644 --- a/hooks/hooks_test.go +++ b/hooks/hooks_test.go @@ -36,6 +36,7 @@ type modifyResult struct { } type HookTestCase struct { + FlowType flows.FlowType Actions ContactActionMap Msgs ContactMsgMap Modifiers ContactModifierMap @@ -134,12 +135,17 @@ func createTestFlow(t *testing.T, uuid assets.FlowUUID, tc HookTestCase) flows.F nodes := []flows.Node{entry} nodes = append(nodes, exitNodes...) + flowType := tc.FlowType + if flowType == "" { + flowType = flows.FlowTypeMessaging + } + // we have our nodes, lets create our flow flow, err := definition.NewFlow( uuid, "Test Flow", envs.Language("eng"), - flows.FlowTypeMessaging, + flowType, 1, 300, definition.NewLocalization(), diff --git a/hooks/msg_created.go b/hooks/msg_created.go index 79fffc653..a758f372a 100644 --- a/hooks/msg_created.go +++ b/hooks/msg_created.go @@ -215,7 +215,7 @@ func handleMsgCreated(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *mode } } - msg, err := models.NewOutgoingMsg(oa.OrgID(), channel, scene.ContactID(), event.Msg, event.CreatedOn()) + msg, err := models.NewOutgoingMsg(oa.Org(), channel, scene.ContactID(), event.Msg, event.CreatedOn()) if err != nil { return errors.Wrapf(err, "error creating outgoing message to %s", event.Msg.URN()) } diff --git a/hooks/msg_received.go b/hooks/msg_received.go index 1b91e4453..b87de4b16 100644 --- a/hooks/msg_received.go +++ b/hooks/msg_received.go @@ -3,11 +3,12 @@ package hooks import ( "context" - "github.com/gomodule/redigo/redis" - "github.com/jmoiron/sqlx" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/mailroom/models" + + "github.com/gomodule/redigo/redis" + "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" ) @@ -19,21 +20,24 @@ func init() { func handleMsgReceived(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *models.OrgAssets, scene *models.Scene, e flows.Event) error { event := e.(*events.MsgReceivedEvent) - // we only care about msg received events when dealing with surveyor flows - if scene.Session().SessionType() != models.SurveyorFlow { - return nil - } + // for surveyor sessions we need to actually create the message + if scene.Session() != nil && scene.Session().SessionType() == models.SurveyorFlow { + logrus.WithFields(logrus.Fields{ + "contact_uuid": scene.ContactUUID(), + "session_id": scene.SessionID(), + "text": event.Msg.Text(), + "urn": event.Msg.URN(), + }).Debug("msg received event") - logrus.WithFields(logrus.Fields{ - "contact_uuid": scene.ContactUUID(), - "session_id": scene.SessionID(), - "text": event.Msg.Text(), - "urn": event.Msg.URN(), - }).Debug("msg received event") + msg := models.NewIncomingMsg(oa.OrgID(), nil, scene.ContactID(), &event.Msg, event.CreatedOn()) + + // we'll commit this message with all the others + scene.AppendToEventPreCommitHook(commitMessagesHook, msg) + } - msg := models.NewIncomingMsg(oa.OrgID(), nil, scene.ContactID(), &event.Msg, event.CreatedOn()) + // update the contact's last seen date + scene.AppendToEventPreCommitHook(contactLastSeenHook, event) + scene.AppendToEventPreCommitHook(updateCampaignEventsHook, event) - // we'll commit this message with all the others - scene.AppendToEventPreCommitHook(commitMessagesHook, msg) return nil } diff --git a/hooks/msg_received_test.go b/hooks/msg_received_test.go new file mode 100644 index 000000000..f38cd3b0e --- /dev/null +++ b/hooks/msg_received_test.go @@ -0,0 +1,68 @@ +package hooks + +import ( + "testing" + "time" + + "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/goflow/flows/actions" + "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/testsuite" +) + +func TestMsgReceived(t *testing.T) { + testsuite.Reset() + db := testsuite.DB() + + now := time.Now() + + tcs := []HookTestCase{ + { + Actions: ContactActionMap{ + models.CathyID: []flows.Action{ + actions.NewSendMsg(newActionUUID(), "Hello World", nil, nil, false), + }, + models.GeorgeID: []flows.Action{ + actions.NewSendMsg(newActionUUID(), "Hello world", nil, nil, false), + }, + }, + Msgs: ContactMsgMap{ + models.CathyID: createIncomingMsg(db, models.Org1, models.CathyID, models.CathyURN, models.CathyURNID, "start"), + }, + SQLAssertions: []SQLAssertion{ + { + SQL: "SELECT COUNT(*) FROM contacts_contact WHERE id = $1 AND last_seen_on > $2", + Args: []interface{}{models.CathyID, now}, + Count: 1, + }, + { + SQL: "SELECT COUNT(*) FROM contacts_contact WHERE id = $1 AND last_seen_on IS NULL", + Args: []interface{}{models.GeorgeID}, + Count: 1, + }, + }, + }, + { + FlowType: flows.FlowTypeMessagingOffline, + Actions: ContactActionMap{ + models.BobID: []flows.Action{ + actions.NewSendMsg(newActionUUID(), "Hello World", nil, nil, false), + }, + }, + Msgs: ContactMsgMap{ + models.BobID: flows.NewMsgIn(flows.MsgUUID(uuids.New()), urns.NilURN, nil, "Hi offline", nil), + }, + SQLAssertions: []SQLAssertion{ + { + SQL: "SELECT COUNT(*) FROM msgs_msg WHERE contact_id = $1 AND direction = 'I'", + Args: []interface{}{models.BobID}, + Count: 1, + }, + }, + }, + } + + RunHookTestCases(t, tcs) +} diff --git a/hooks/service_called_test.go b/hooks/service_called_test.go index 8142e6522..a17718320 100644 --- a/hooks/service_called_test.go +++ b/hooks/service_called_test.go @@ -3,10 +3,10 @@ package hooks import ( "testing" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/actions" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/mailroom/models" ) diff --git a/hooks/session_triggered.go b/hooks/session_triggered.go index 8ad351709..1a816f0a3 100644 --- a/hooks/session_triggered.go +++ b/hooks/session_triggered.go @@ -5,6 +5,7 @@ import ( "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" + "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/mailroom/models" @@ -90,6 +91,11 @@ func (h *InsertStartHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool return errors.Wrapf(err, "error loading contacts by reference") } + historyJSON, err := jsonx.Marshal(event.History) + if err != nil { + return errors.Wrapf(err, "error marshaling session history") + } + // create our start start := models.NewFlowStart(oa.OrgID(), models.StartTypeFlowAction, flow.FlowType(), flow.ID(), models.DoRestartParticipants, models.DoIncludeActive). WithGroupIDs(groupIDs). @@ -97,7 +103,8 @@ func (h *InsertStartHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool WithURNs(event.URNs). WithQuery(event.ContactQuery). WithCreateContact(event.CreateContact). - WithParentSummary(event.RunSummary) + WithParentSummary(event.RunSummary). + WithSessionHistory(historyJSON) starts = append(starts, start) diff --git a/hooks/session_triggered_test.go b/hooks/session_triggered_test.go index 1e6deddf7..8dc571186 100644 --- a/hooks/session_triggered_test.go +++ b/hooks/session_triggered_test.go @@ -4,14 +4,16 @@ import ( "encoding/json" "testing" - "github.com/gomodule/redigo/redis" - "github.com/jmoiron/sqlx" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/actions" + "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" "github.com/nyaruka/mailroom/testsuite" + + "github.com/gomodule/redigo/redis" + "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" ) @@ -36,6 +38,9 @@ func TestSessionTriggered(t *testing.T) { UUID: models.TestersGroupUUID, } + uuids.SetGenerator(uuids.NewSeededGenerator(1234567)) + defer uuids.SetGenerator(uuids.DefaultGenerator) + tcs := []HookTestCase{ { Actions: ContactActionMap{ @@ -50,7 +55,7 @@ func TestSessionTriggered(t *testing.T) { Count: 1, }, { - SQL: "select count(*) from flows_flowstart where org_id = 1 AND start_type = 'F' AND flow_id = $1 AND status = 'P' AND parent_summary IS NOT NULL;", + SQL: "select count(*) from flows_flowstart where org_id = 1 AND start_type = 'F' AND flow_id = $1 AND status = 'P' AND parent_summary IS NOT NULL AND session_history IS NOT NULL;", Args: []interface{}{models.SingleMessageFlowID}, Count: 1, }, @@ -73,10 +78,11 @@ func TestSessionTriggered(t *testing.T) { start := models.FlowStart{} err = json.Unmarshal(task.Task, &start) assert.NoError(t, err) - assert.Equal(t, start.CreateContact(), true) + assert.True(t, start.CreateContact()) assert.Equal(t, []models.ContactID{models.GeorgeID}, start.ContactIDs()) assert.Equal(t, []models.GroupID{models.TestersGroupID}, start.GroupIDs()) - assert.Equal(t, start.FlowID(), simpleFlow.ID()) + assert.Equal(t, simpleFlow.ID(), start.FlowID()) + assert.JSONEq(t, `{"parent_uuid":"36284611-ea19-4f1f-8611-9bc48e206654", "ancestors":1, "ancestors_since_input":1}`, string(start.SessionHistory())) return nil }, }, diff --git a/hooks/ticket_opened_test.go b/hooks/ticket_opened_test.go index 317bcb963..3d777c615 100644 --- a/hooks/ticket_opened_test.go +++ b/hooks/ticket_opened_test.go @@ -3,10 +3,10 @@ package hooks import ( "testing" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/actions" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" diff --git a/hooks/webhook_called_test.go b/hooks/webhook_called_test.go index ed37ed479..f5f520db0 100644 --- a/hooks/webhook_called_test.go +++ b/hooks/webhook_called_test.go @@ -6,9 +6,9 @@ import ( "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/actions" - "github.com/nyaruka/goflow/utils/httpx" ) func TestWebhookCalled(t *testing.T) { diff --git a/httputils/httputils.go b/httputils/httputils.go deleted file mode 100644 index 4c41fb50a..000000000 --- a/httputils/httputils.go +++ /dev/null @@ -1,96 +0,0 @@ -package httputils - -import ( - "net/http" - "net/http/httputil" - "time" - - "github.com/pkg/errors" -) - -// RoundTrip represents a single request / response round trip created by our transport. In the -// case of connection errors, the status code will be set to 499 and the response body will -// contain more information about the error encountered -type RoundTrip struct { - Method string - URL string - RequestBody []byte - Status int - ResponseBody []byte - StartedOn time.Time - Elapsed time.Duration -} - -// LoggingTransport is a transport which keeps track of all requests and responses -type LoggingTransport struct { - tripper http.RoundTripper - RoundTrips []*RoundTrip -} - -// RoundTrip satisfier the roundtripper interface in http to allow for capturing -// requests and responses of the parent http client. -func (t *LoggingTransport) RoundTrip(request *http.Request) (*http.Response, error) { - rt := &RoundTrip{ - StartedOn: time.Now(), - Method: request.Method, - URL: request.URL.String(), - } - - requestBody, err := httputil.DumpRequestOut(request, true) - if err != nil { - return nil, err - } - rt.RequestBody = requestBody - t.RoundTrips = append(t.RoundTrips, rt) - - response, err := t.tripper.RoundTrip(request) - rt.Elapsed = time.Since(rt.StartedOn) - - if err != nil { - err = errors.Wrapf(err, "error making http request") - rt.Status = 444 - rt.ResponseBody = []byte(err.Error()) - return response, err - } - - defer response.Body.Close() - - responseBody, err := httputil.DumpResponse(response, true) - if err != nil { - err = errors.Wrapf(err, "error dumping http response") - rt.Status = 444 - rt.ResponseBody = []byte(err.Error()) - return response, err - } - rt.ResponseBody = responseBody - rt.Status = response.StatusCode - - return response, err -} - -// NewLoggingTransport creates a new logging transport -func NewLoggingTransport(tripper http.RoundTripper) *LoggingTransport { - return &LoggingTransport{ - tripper: tripper, - } -} - -// UserAgentTransport just injects a custom user agent on the request before sending it out -type UserAgentTransport struct { - tripper http.RoundTripper - agent string -} - -// RoundTrip just injects our custom user agent, passing the request down the chain -func (t *UserAgentTransport) RoundTrip(request *http.Request) (*http.Response, error) { - request.Header.Set("User-Agent", t.agent) - return t.tripper.RoundTrip(request) -} - -// NewUserAgentTransport creates a new transport that injects a user agent in all requests -func NewUserAgentTransport(tripper http.RoundTripper, agent string) *UserAgentTransport { - return &UserAgentTransport{ - tripper: tripper, - agent: agent, - } -} diff --git a/ivr/ivr.go b/ivr/ivr.go index 5626fa649..b8628ad7a 100644 --- a/ivr/ivr.go +++ b/ivr/ivr.go @@ -7,11 +7,10 @@ import ( "net/http" "net/url" "path" - "path/filepath" "strconv" - "strings" "time" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/excellent/types" @@ -21,12 +20,10 @@ import ( "github.com/nyaruka/goflow/utils" "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/config" - "github.com/nyaruka/mailroom/httputils" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/runner" - "github.com/nyaruka/mailroom/s3utils" + "github.com/nyaruka/mailroom/utils/storage" - "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" "github.com/pkg/errors" @@ -46,9 +43,6 @@ const ( ErrorMessage = "An error has occurred, please try again later." ) -// WriteAttachments controls whether we write attachments, used during unit testing -var WriteAttachments = true - // CallEndedError is our constant error for when a call has ended var CallEndedError = fmt.Errorf("call ended") @@ -56,7 +50,7 @@ var CallEndedError = fmt.Errorf("call ended") var constructors = make(map[models.ChannelType]ClientConstructor) // ClientConstructor defines our signature for creating a new IVR client from a channel -type ClientConstructor func(c *models.Channel) (Client, error) +type ClientConstructor func(*http.Client, *models.Channel) (Client, error) // RegisterClientType registers the passed in channel type with the passed in constructor func RegisterClientType(channelType models.ChannelType, constructor ClientConstructor) { @@ -70,14 +64,14 @@ func GetClient(channel *models.Channel) (Client, error) { return nil, errors.Errorf("no ivr client for channel type: %s", channel.Type()) } - return constructor(channel) + return constructor(http.DefaultClient, channel) } // Client defines the interface IVR clients must satisfy type Client interface { - RequestCall(client *http.Client, number urns.URN, handleURL string, statusURL string) (CallID, error) + RequestCall(number urns.URN, handleURL string, statusURL string) (CallID, *httpx.Trace, error) - HangupCall(client *http.Client, externalID string) error + HangupCall(externalID string) (*httpx.Trace, error) WriteSessionResponse(session *models.Session, number urns.URN, resumeURL string, req *http.Request, w http.ResponseWriter) error @@ -123,27 +117,19 @@ func HangupCall(ctx context.Context, config *config.Config, db *sqlx.DB, conn *m return errors.Wrapf(err, "unable to create ivr client") } - // we create our own HTTP client with our own transport so we can log the request and set our user agent - logger := httputils.NewLoggingTransport(http.DefaultTransport) - client := &http.Client{Transport: httputils.NewUserAgentTransport(logger, userAgent+config.Version)} - // try to request our call hangup - err = c.HangupCall(client, conn.ExternalID()) + trace, err := c.HangupCall(conn.ExternalID()) - // insert any logged requests - for _, rt := range logger.RoundTrips { + // insert an channel log if we have an HTTP trace + if trace != nil { desc := "Hangup Requested" isError := false - if rt.Status/100 != 2 { + if trace.Response == nil || trace.Response.StatusCode/100 != 2 { desc = "Error Hanging up Call" isError = true } - _, err := models.InsertChannelLog( - ctx, db, desc, isError, - rt.Method, rt.URL, rt.RequestBody, rt.Status, rt.ResponseBody, - rt.StartedOn, rt.Elapsed, - channel, conn, - ) + log := models.NewChannelLog(trace, isError, desc, channel, conn) + err := models.InsertChannelLogs(ctx, db, []*models.ChannelLog{log}) // log any error inserting our channel log, but try to continue if err != nil { @@ -262,27 +248,19 @@ func RequestCallStartForConnection(ctx context.Context, config *config.Config, d return errors.Wrapf(err, "unable to create ivr client") } - // we create our own HTTP client with our own transport so we can log the request and set our user agent - logger := httputils.NewLoggingTransport(http.DefaultTransport) - client := &http.Client{Transport: httputils.NewUserAgentTransport(logger, userAgent+config.Version)} - // try to request our call start - callID, err := c.RequestCall(client, telURN, resumeURL, statusURL) + callID, trace, err := c.RequestCall(telURN, resumeURL, statusURL) - // insert any logged requests - for _, rt := range logger.RoundTrips { + /// insert an channel log if we have an HTTP trace + if trace != nil { desc := "Call Requested" isError := false - if rt.Status/100 != 2 { + if trace.Response == nil || trace.Response.StatusCode/100 != 2 { desc = "Error Requesting Call" isError = true } - _, err := models.InsertChannelLog( - ctx, db, desc, isError, - rt.Method, rt.URL, rt.RequestBody, rt.Status, rt.ResponseBody, - rt.StartedOn, rt.Elapsed, - channel, conn, - ) + log := models.NewChannelLog(trace, isError, desc, channel, conn) + err := models.InsertChannelLogs(ctx, db, []*models.ChannelLog{log}) // log any error inserting our channel log, but try to continue if err != nil { @@ -353,13 +331,21 @@ func StartIVRFlow( } } + var history *flows.SessionHistory + if len(start.SessionHistory()) > 0 { + history, err = models.ReadSessionHistory(start.SessionHistory()) + if err != nil { + return errors.Wrap(err, "unable to read JSON from flow start history") + } + } + // our builder for the triggers that will be created for contacts flowRef := assets.NewFlowReference(flow.UUID(), flow.Name()) var trigger flows.Trigger if len(start.ParentSummary()) > 0 { trigger = triggers.NewBuilder(oa.Env(), flowRef, contact). - FlowAction(start.ParentSummary()). + FlowAction(history, start.ParentSummary()). WithConnection(channel.ChannelReference(), urn). Build() } else { @@ -405,7 +391,7 @@ func StartIVRFlow( // ResumeIVRFlow takes care of resuming the flow in the passed in start for the passed in contact and URN func ResumeIVRFlow( - ctx context.Context, config *config.Config, db *sqlx.DB, rp *redis.Pool, s3Client s3iface.S3API, + ctx context.Context, config *config.Config, db *sqlx.DB, rp *redis.Pool, store storage.Storage, resumeURL string, client Client, oa *models.OrgAssets, channel *models.Channel, conn *models.ChannelConnection, c *models.Contact, urn urns.URN, r *http.Request, w http.ResponseWriter) error { @@ -492,24 +478,12 @@ func ResumeIVRFlow( } resp.Body.Close() - // check our content type - contentType := http.DetectContentType(body) - // filename is based on our org id and msg UUID filename := string(msgUUID) + path.Ext(attachment.URL()) - path := filepath.Join(config.S3MediaPrefix, fmt.Sprintf("%d", oa.OrgID()), filename[:4], filename[4:8], filename) - if !strings.HasPrefix(path, "/") { - path = fmt.Sprintf("/%s", path) - } - if WriteAttachments { - // write to S3 - logrus.WithField("path", path).Info("** uploading s3 file") - url, err := s3utils.PutS3File(s3Client, config.S3MediaBucket, path, contentType, body) - if err != nil { - return errors.Wrapf(err, "unable to write attachment to s3") - } - attachment = utils.Attachment(contentType + ":" + url) + attachment, err = oa.Org().StoreAttachment(store, config.S3MediaPrefix, filename, body) + if err != nil { + return errors.Wrapf(err, "unable to store IVR attachment") } } diff --git a/ivr/nexmo/nexmo.go b/ivr/nexmo/nexmo.go index 7e4a33595..bc1da095a 100644 --- a/ivr/nexmo/nexmo.go +++ b/ivr/nexmo/nexmo.go @@ -9,7 +9,6 @@ import ( "encoding/base64" "encoding/json" "fmt" - "io" "io/ioutil" "math/rand" "net/http" @@ -19,6 +18,7 @@ import ( "strings" "time" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/events" @@ -65,6 +65,7 @@ const ( var indentMarshal = true type client struct { + httpClient *http.Client channel *models.Channel baseURL string appID string @@ -76,7 +77,7 @@ func init() { } // NewClientFromChannel creates a new Twilio IVR client for the passed in account and and auth token -func NewClientFromChannel(channel *models.Channel) (ivr.Client, error) { +func NewClientFromChannel(httpClient *http.Client, channel *models.Channel) (ivr.Client, error) { appID := channel.ConfigValue(appIDConfig, "") key := channel.ConfigValue(privateKeyConfig, "") if appID == "" || key == "" { @@ -89,6 +90,7 @@ func NewClientFromChannel(channel *models.Channel) (ivr.Client, error) { } return &client{ + httpClient: httpClient, channel: channel, baseURL: BaseURL, appID: appID, @@ -280,7 +282,7 @@ type CallResponse struct { } // RequestCall causes this client to request a new outgoing call for this provider -func (c *client) RequestCall(client *http.Client, number urns.URN, resumeURL string, statusURL string) (ivr.CallID, error) { +func (c *client) RequestCall(number urns.URN, resumeURL string, statusURL string) (ivr.CallID, *httpx.Trace, error) { callR := &CallRequest{ AnswerURL: []string{resumeURL + "&sig=" + url.QueryEscape(c.calculateSignature(resumeURL))}, AnswerMethod: http.MethodPost, @@ -290,64 +292,54 @@ func (c *client) RequestCall(client *http.Client, number urns.URN, resumeURL str } rawTo, err := strconv.Atoi(number.Path()) if err != nil { - return ivr.NilCallID, errors.Wrapf(err, "unable to turn urn path into number: %s", number.Path()) + return ivr.NilCallID, nil, errors.Wrapf(err, "unable to turn urn path into number: %s", number.Path()) } callR.To = append(callR.To, Phone{Type: "phone", Number: rawTo}) rawFrom, err := strconv.Atoi(c.channel.Address()) if err != nil { - return ivr.NilCallID, errors.Wrapf(err, "unable to turn urn path into number: %s", number.Path()) + return ivr.NilCallID, nil, errors.Wrapf(err, "unable to turn urn path into number: %s", number.Path()) } callR.From = Phone{Type: "phone", Number: rawFrom} - resp, err := c.makeRequest(client, http.MethodPost, BaseURL, callR) + trace, err := c.makeRequest(http.MethodPost, BaseURL, callR) if err != nil { - return ivr.NilCallID, errors.Wrapf(err, "error trying to start call") + return ivr.NilCallID, trace, errors.Wrapf(err, "error trying to start call") } - defer resp.Body.Close() - if resp.StatusCode != http.StatusCreated { - io.Copy(ioutil.Discard, resp.Body) - return ivr.NilCallID, errors.Errorf("received non 200 status for call start: %d", resp.StatusCode) - } - - // read our body - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return ivr.NilCallID, errors.Wrapf(err, "error reading response body") + if trace.Response.StatusCode != http.StatusCreated { + return ivr.NilCallID, trace, errors.Errorf("received non 200 status for call start: %d", trace.Response.StatusCode) } // parse out our call sid call := &CallResponse{} - err = json.Unmarshal(body, call) + err = json.Unmarshal(trace.ResponseBody, call) if err != nil || call.UUID == "" { - return ivr.NilCallID, errors.Errorf("unable to read call uuid") + return ivr.NilCallID, trace, errors.Errorf("unable to read call uuid") } if call.Status == statusFailed { - return ivr.NilCallID, errors.Errorf("call status returned as failed") + return ivr.NilCallID, trace, errors.Errorf("call status returned as failed") } - logrus.WithField("body", string(body)).WithField("status", resp.StatusCode).Debug("requested call") + logrus.WithField("body", string(trace.ResponseBody)).WithField("status", trace.Response.StatusCode).Debug("requested call") - return ivr.CallID(call.UUID), nil + return ivr.CallID(call.UUID), trace, nil } // HangupCall asks Nexmo to hang up the call that is passed in -func (c *client) HangupCall(client *http.Client, callID string) error { +func (c *client) HangupCall(callID string) (*httpx.Trace, error) { hangupBody := map[string]string{"action": "hangup"} url := BaseURL + "/" + callID - resp, err := c.makeRequest(client, http.MethodPut, url, hangupBody) + trace, err := c.makeRequest(http.MethodPut, url, hangupBody) if err != nil { - return errors.Wrapf(err, "error trying to hangup call") + return trace, errors.Wrapf(err, "error trying to hangup call") } - defer resp.Body.Close() - io.Copy(ioutil.Discard, resp.Body) - if resp.StatusCode != 204 { - return errors.Errorf("received non 204 status for call hangup: %d", resp.StatusCode) + if trace.Response.StatusCode != 204 { + return trace, errors.Errorf("received non 204 status for call hangup: %d", trace.Response.StatusCode) } - return nil + return trace, nil } type NCCOInput struct { @@ -533,7 +525,7 @@ func (c *client) WriteEmptyResponse(w http.ResponseWriter, msg string) error { return err } -func (c *client) makeRequest(client *http.Client, method string, sendURL string, body interface{}) (*http.Response, error) { +func (c *client) makeRequest(method string, sendURL string, body interface{}) (*httpx.Trace, error) { bb, err := json.Marshal(body) if err != nil { return nil, errors.Wrapf(err, "error json encoding request") @@ -549,7 +541,7 @@ func (c *client) makeRequest(client *http.Client, method string, sendURL string, req.Header.Set("Accept", "application/json") req.Header.Set("Content-Type", "application/json") - return client.Do(req) + return httpx.DoTrace(c.httpClient, req, nil, nil, -1) } // calculateSignature calculates a signature for the passed in URL diff --git a/ivr/nexmo/nexmo_test.go b/ivr/nexmo/nexmo_test.go index 20d6dade4..561809ddf 100644 --- a/ivr/nexmo/nexmo_test.go +++ b/ivr/nexmo/nexmo_test.go @@ -1,6 +1,7 @@ package nexmo import ( + "net/http" "testing" "github.com/nyaruka/gocommon/urns" @@ -48,7 +49,7 @@ func TestResponseForSprint(t *testing.T) { channel := oa.ChannelByUUID(models.NexmoChannelUUID) assert.NotNil(t, channel) - c, err := NewClientFromChannel(channel) + c, err := NewClientFromChannel(http.DefaultClient, channel) assert.NoError(t, err) client := c.(*client) diff --git a/ivr/twiml/twiml.go b/ivr/twiml/twiml.go index 3cea2d86f..c52fee633 100644 --- a/ivr/twiml/twiml.go +++ b/ivr/twiml/twiml.go @@ -9,19 +9,15 @@ import ( "encoding/json" "encoding/xml" "fmt" - "io" - "io/ioutil" "net/http" "net/url" "sort" "strconv" "strings" - "github.com/nyaruka/goflow/envs" - - "github.com/gomodule/redigo/redis" - "github.com/jmoiron/sqlx" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/goflow/flows/routers/waits" @@ -29,6 +25,9 @@ import ( "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/ivr" "github.com/nyaruka/mailroom/models" + + "github.com/gomodule/redigo/redis" + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -100,6 +99,7 @@ var validLanguageCodes = map[string]bool{ var indentMarshal = true type client struct { + httpClient *http.Client channel *models.Channel baseURL string accountSID string @@ -114,7 +114,7 @@ func init() { } // NewClientFromChannel creates a new Twilio IVR client for the passed in account and and auth token -func NewClientFromChannel(channel *models.Channel) (ivr.Client, error) { +func NewClientFromChannel(httpClient *http.Client, channel *models.Channel) (ivr.Client, error) { accountSID := channel.ConfigValue(accountSIDConfig, "") authToken := channel.ConfigValue(authTokenConfig, "") if accountSID == "" || authToken == "" { @@ -123,6 +123,7 @@ func NewClientFromChannel(channel *models.Channel) (ivr.Client, error) { baseURL := channel.ConfigValue(baseURLConfig, channel.ConfigValue(sendURLConfig, BaseURL)) return &client{ + httpClient: httpClient, channel: channel, baseURL: baseURL, accountSID: accountSID, @@ -132,8 +133,9 @@ func NewClientFromChannel(channel *models.Channel) (ivr.Client, error) { } // NewClient creates a new Twilio IVR client for the passed in account and and auth token -func NewClient(accountSID string, authToken string) ivr.Client { +func NewClient(httpClient *http.Client, accountSID string, authToken string) ivr.Client { return &client{ + httpClient: httpClient, baseURL: BaseURL, accountSID: accountSID, authToken: authToken, @@ -173,7 +175,7 @@ type CallResponse struct { } // RequestCall causes this client to request a new outgoing call for this provider -func (c *client) RequestCall(client *http.Client, number urns.URN, callbackURL string, statusURL string) (ivr.CallID, error) { +func (c *client) RequestCall(number urns.URN, callbackURL string, statusURL string) (ivr.CallID, *httpx.Trace, error) { form := url.Values{} form.Set("To", number.Path()) form.Set("From", c.channel.Address()) @@ -182,57 +184,47 @@ func (c *client) RequestCall(client *http.Client, number urns.URN, callbackURL s sendURL := c.baseURL + strings.Replace(callPath, "{AccountSID}", c.accountSID, -1) - resp, err := c.postRequest(client, sendURL, form) + trace, err := c.postRequest(sendURL, form) if err != nil { - return ivr.NilCallID, errors.Wrapf(err, "error trying to start call") - } - defer resp.Body.Close() - - if resp.StatusCode != 201 { - io.Copy(ioutil.Discard, resp.Body) - return ivr.NilCallID, errors.Errorf("received non 201 status for call start: %d", resp.StatusCode) + return ivr.NilCallID, trace, errors.Wrapf(err, "error trying to start call") } - // read our body - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return ivr.NilCallID, errors.Wrapf(err, "error reading response body") + if trace.Response.StatusCode != 201 { + return ivr.NilCallID, trace, errors.Errorf("received non 201 status for call start: %d", trace.Response.StatusCode) } // parse out our call sid call := &CallResponse{} - err = json.Unmarshal(body, call) + err = json.Unmarshal(trace.ResponseBody, call) if err != nil || call.SID == "" { - return ivr.NilCallID, errors.Errorf("unable to read call id") + return ivr.NilCallID, trace, errors.Errorf("unable to read call id") } if call.Status == statusFailed { - return ivr.NilCallID, errors.Errorf("call status returned as failed") + return ivr.NilCallID, trace, errors.Errorf("call status returned as failed") } - return ivr.CallID(call.SID), nil + return ivr.CallID(call.SID), trace, nil } // HangupCall asks Twilio to hang up the call that is passed in -func (c *client) HangupCall(client *http.Client, callID string) error { +func (c *client) HangupCall(callID string) (*httpx.Trace, error) { form := url.Values{} form.Set("Status", "completed") sendURL := c.baseURL + strings.Replace(hangupPath, "{AccountSID}", c.accountSID, -1) sendURL = strings.Replace(sendURL, "{SID}", callID, -1) - resp, err := c.postRequest(client, sendURL, form) + trace, err := c.postRequest(sendURL, form) if err != nil { - return errors.Wrapf(err, "error trying to hangup call") + return trace, errors.Wrapf(err, "error trying to hangup call") } - defer resp.Body.Close() - io.Copy(ioutil.Discard, resp.Body) - if resp.StatusCode != 200 { - return errors.Errorf("received non 204 trying to hang up call: %d", resp.StatusCode) + if trace.Response.StatusCode != 200 { + return trace, errors.Errorf("received non 204 trying to hang up call: %d", trace.Response.StatusCode) } - return nil + return trace, nil } // InputForRequest returns the input for the passed in request, if any @@ -376,13 +368,13 @@ func (c *client) WriteEmptyResponse(w http.ResponseWriter, msg string) error { return err } -func (c *client) postRequest(client *http.Client, sendURL string, form url.Values) (*http.Response, error) { +func (c *client) postRequest(sendURL string, form url.Values) (*httpx.Trace, error) { req, _ := http.NewRequest(http.MethodPost, sendURL, strings.NewReader(form.Encode())) req.SetBasicAuth(c.accountSID, c.authToken) req.Header.Set("Content-Type", "application/x-www-form-urlencoded") req.Header.Set("Accept", "application/json") - return client.Do(req) + return httpx.DoTrace(c.httpClient, req, nil, nil, -1) } // see https://www.twilio.com/docs/api/security diff --git a/mailroom.go b/mailroom.go index 8986d9f71..cfda2408f 100644 --- a/mailroom.go +++ b/mailroom.go @@ -11,14 +11,9 @@ import ( "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/queue" - "github.com/nyaruka/mailroom/s3utils" + "github.com/nyaruka/mailroom/utils/storage" "github.com/nyaruka/mailroom/web" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" "github.com/nyaruka/librato" @@ -52,7 +47,7 @@ type Mailroom struct { DB *sqlx.DB RP *redis.Pool ElasticClient *elastic.Client - S3Client s3iface.S3API + Storage storage.Storage Quit chan bool CTX context.Context @@ -163,25 +158,18 @@ func (mr *Mailroom) Start() error { log.Info("redis ok") } - // create our s3 client - s3Session, err := session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials(mr.Config.AWSAccessKeyID, mr.Config.AWSSecretAccessKey, ""), - Endpoint: aws.String(mr.Config.S3Endpoint), - Region: aws.String(mr.Config.S3Region), - DisableSSL: aws.Bool(mr.Config.S3DisableSSL), - S3ForcePathStyle: aws.Bool(mr.Config.S3ForcePathStyle), - }) + // create our storage (S3 or file system) + mr.Storage, err = storage.New(mr.Config) if err != nil { return err } - mr.S3Client = s3.New(s3Session) - // test out our S3 credentials - err = s3utils.TestS3(mr.S3Client, mr.Config.S3MediaBucket) + // test our storage + err = mr.Storage.Test() if err != nil { - log.WithError(err).Error("s3 bucket not reachable") + log.WithError(err).Error(mr.Storage.Name() + " storage not available") } else { - log.Info("s3 bucket ok") + log.Info(mr.Storage.Name() + " storage ok") } // initialize our elastic client @@ -211,7 +199,7 @@ func (mr *Mailroom) Start() error { mr.handlerForeman.Start() // start our web server - mr.webserver = web.NewServer(mr.CTX, mr.Config, mr.DB, mr.RP, mr.S3Client, mr.ElasticClient, mr.WaitGroup) + mr.webserver = web.NewServer(mr.CTX, mr.Config, mr.DB, mr.RP, mr.Storage, mr.ElasticClient, mr.WaitGroup) mr.webserver.Start() logrus.Info("mailroom started") diff --git a/mailroom_test.dump b/mailroom_test.dump index 6035262fb..8e1f40120 100644 Binary files a/mailroom_test.dump and b/mailroom_test.dump differ diff --git a/models/assets.go b/models/assets.go index cb999e291..86c55e4c4 100644 --- a/models/assets.go +++ b/models/assets.go @@ -130,13 +130,19 @@ func NewOrgAssets(ctx context.Context, db *sqlx.DB, orgID OrgID, prev *OrgAssets } if prev == nil || refresh&RefreshFields > 0 { - oa.fields, err = loadFields(ctx, db, orgID) + userFields, systemFields, err := loadFields(ctx, db, orgID) if err != nil { return nil, errors.Wrapf(err, "error loading field assets for org %d", orgID) } - oa.fieldsByUUID = make(map[assets.FieldUUID]*Field) - oa.fieldsByKey = make(map[string]*Field) - for _, f := range oa.fields { + oa.fields = userFields + oa.fieldsByUUID = make(map[assets.FieldUUID]*Field, len(userFields)+len(systemFields)) + oa.fieldsByKey = make(map[string]*Field, len(userFields)+len(systemFields)) + for _, f := range userFields { + field := f.(*Field) + oa.fieldsByUUID[field.UUID()] = field + oa.fieldsByKey[field.Key()] = field + } + for _, f := range systemFields { field := f.(*Field) oa.fieldsByUUID[field.UUID()] = field oa.fieldsByKey[field.Key()] = field diff --git a/models/campaigns.go b/models/campaigns.go index f53033853..c043b1b7f 100644 --- a/models/campaigns.go +++ b/models/campaigns.go @@ -37,9 +37,12 @@ type OffsetUnit string type StartMode string const ( - // CreatedOnKey + // CreatedOnKey is key of created on system field CreatedOnKey = "created_on" + // LastSeenOnKey is key of last seen on system field + LastSeenOnKey = "last_seen_on" + // OffsetMinute means our offset is in minutes OffsetMinute = OffsetUnit("M") @@ -126,12 +129,15 @@ func (e *CampaignEvent) QualifiesByGroup(contact *flows.Contact) bool { // QualifiesByField returns whether the passed in contact qualifies for this event by group membership func (e *CampaignEvent) QualifiesByField(contact *flows.Contact) bool { - if e.RelativeToKey() == CreatedOnKey { + switch e.RelativeToKey() { + case CreatedOnKey: return true + case LastSeenOnKey: + return contact.LastSeenOn() != nil + default: + value := contact.Fields()[e.RelativeToKey()] + return value != nil } - - value := contact.Fields()[e.RelativeToKey()] - return value != nil } // ScheduleForContact calculates the next fire ( if any) for the passed in contact @@ -143,10 +149,16 @@ func (e *CampaignEvent) ScheduleForContact(tz *time.Location, now time.Time, con var start time.Time - // created on is a special case - if e.RelativeToKey() == CreatedOnKey { + switch e.RelativeToKey() { + case CreatedOnKey: start = contact.CreatedOn() - } else { + case LastSeenOnKey: + value := contact.LastSeenOn() + if value == nil { + return nil, nil + } + start = *value + default: // everything else is just a normal field value := contact.Fields()[e.RelativeToKey()] diff --git a/models/channel_event.go b/models/channel_event.go index dbb86240f..2997c49c8 100644 --- a/models/channel_event.go +++ b/models/channel_event.go @@ -12,40 +12,54 @@ import ( type ChannelEventType string type ChannelEventID int64 +// channel event types const ( NewConversationEventType = ChannelEventType("new_conversation") - WelcomeMessateEventType = ChannelEventType("welcome_message") + WelcomeMessageEventType = ChannelEventType("welcome_message") ReferralEventType = ChannelEventType("referral") + MTMissEventType = ChannelEventType("mt_miss") + MTCallEventType = ChannelEventType("mt_call") MOMissEventType = ChannelEventType("mo_miss") MOCallEventType = ChannelEventType("mo_call") + StopContactEventType = ChannelEventType("stop_contact") ) +// ContactSeenEvents are those which count as the contact having been seen +var ContactSeenEvents = map[ChannelEventType]bool{ + NewConversationEventType: true, + ReferralEventType: true, + MOMissEventType: true, + MOCallEventType: true, + StopContactEventType: true, +} + // ChannelEvent represents an event that occurred associated with a channel, such as a referral, missed call, etc.. type ChannelEvent struct { e struct { - ID ChannelEventID `json:"id" db:"id"` - EventType ChannelEventType `json:"event_type" db:"event_type"` - OrgID OrgID `json:"org_id" db:"org_id"` - ChannelID ChannelID `json:"channel_id" db:"channel_id"` - ContactID ContactID `json:"contact_id" db:"contact_id"` - URNID URNID `json:"urn_id" db:"contact_urn_id"` - Extra null.Map `json:"extra" db:"extra"` + ID ChannelEventID `json:"id" db:"id"` + EventType ChannelEventType `json:"event_type" db:"event_type"` + OrgID OrgID `json:"org_id" db:"org_id"` + ChannelID ChannelID `json:"channel_id" db:"channel_id"` + ContactID ContactID `json:"contact_id" db:"contact_id"` + URNID URNID `json:"urn_id" db:"contact_urn_id"` + Extra null.Map `json:"extra" db:"extra"` + OccurredOn time.Time `json:"occurred_on" db:"occurred_on"` // only in JSON representation NewContact bool `json:"new_contact"` // only in DB representation - CreatedOn time.Time `db:"created_on"` - OccurredOn time.Time `db:"occurred_on"` + CreatedOn time.Time `db:"created_on"` } } -func (e *ChannelEvent) ID() ChannelEventID { return e.e.ID } -func (e *ChannelEvent) ContactID() ContactID { return e.e.ContactID } -func (e *ChannelEvent) URNID() URNID { return e.e.URNID } -func (e *ChannelEvent) OrgID() OrgID { return e.e.OrgID } -func (e *ChannelEvent) ChannelID() ChannelID { return e.e.ChannelID } -func (e *ChannelEvent) IsNewContact() bool { return e.e.NewContact } +func (e *ChannelEvent) ID() ChannelEventID { return e.e.ID } +func (e *ChannelEvent) ContactID() ContactID { return e.e.ContactID } +func (e *ChannelEvent) URNID() URNID { return e.e.URNID } +func (e *ChannelEvent) OrgID() OrgID { return e.e.OrgID } +func (e *ChannelEvent) ChannelID() ChannelID { return e.e.ChannelID } +func (e *ChannelEvent) IsNewContact() bool { return e.e.NewContact } +func (e *ChannelEvent) OccurredOn() time.Time { return e.e.OccurredOn } func (e *ChannelEvent) Extra() map[string]interface{} { return e.e.Extra.Map() diff --git a/models/channel_event_test.go b/models/channel_event_test.go index a1029f7fe..d8962b2ca 100644 --- a/models/channel_event_test.go +++ b/models/channel_event_test.go @@ -3,6 +3,7 @@ package models import ( "encoding/json" "testing" + "time" "github.com/nyaruka/mailroom/testsuite" "github.com/stretchr/testify/assert" @@ -12,12 +13,15 @@ func TestChannelEvents(t *testing.T) { ctx := testsuite.CTX() db := testsuite.DB() + start := time.Now() + // no extra e := NewChannelEvent(MOMissEventType, Org1, TwilioChannelID, CathyID, CathyURNID, nil, false) err := e.Insert(ctx, db) assert.NoError(t, err) assert.NotZero(t, e.ID()) assert.Equal(t, e.Extra(), map[string]interface{}{}) + assert.True(t, e.OccurredOn().After(start)) // with extra e2 := NewChannelEvent(MOMissEventType, Org1, TwilioChannelID, CathyID, CathyURNID, map[string]interface{}{"referral_id": "foobar"}, false) @@ -33,4 +37,5 @@ func TestChannelEvents(t *testing.T) { err = json.Unmarshal(asJSON, e3) assert.NoError(t, err) assert.Equal(t, e2.Extra(), e3.Extra()) + assert.True(t, e.OccurredOn().After(start)) } diff --git a/models/channel_logs.go b/models/channel_logs.go index 82e3a14bc..7f8aa931c 100644 --- a/models/channel_logs.go +++ b/models/channel_logs.go @@ -5,6 +5,7 @@ import ( "time" "github.com/jmoiron/sqlx" + "github.com/nyaruka/gocommon/httpx" "github.com/pkg/errors" ) @@ -35,17 +36,52 @@ func (l *ChannelLog) ID() ChannelLogID { return l.l.ID } const insertChannelLogSQL = ` INSERT INTO - channels_channellog( - description, is_error, url, method, request, response, response_status, - created_on, request_time, channel_id, connection_id) - VALUES( - :description, :is_error, :url, :method, :request, :response, :response_status, - :created_on, :request_time, :channel_id, :connection_id) - + channels_channellog( description, is_error, url, method, request, response, response_status, created_on, request_time, channel_id, connection_id) + VALUES(:description, :is_error, :url, :method, :request, :response, :response_status, :created_on, :request_time, :channel_id, :connection_id) RETURNING id as id ` +// NewChannelLog creates a new channel log +func NewChannelLog(trace *httpx.Trace, isError bool, desc string, channel *Channel, conn *ChannelConnection) *ChannelLog { + log := &ChannelLog{} + l := &log.l + + statusCode := 0 + if trace.Response != nil { + statusCode = trace.Response.StatusCode + } + + l.Description = desc + l.IsError = isError + l.URL = trace.Request.URL.String() + l.Method = trace.Request.Method + l.Request = string(trace.RequestTrace) + l.Response = trace.ResponseTraceUTF8("...") + l.Status = statusCode + l.CreatedOn = trace.StartTime + l.RequestTime = int((trace.EndTime.Sub(trace.StartTime)) / time.Millisecond) + l.ChannelID = channel.ID() + if conn != nil { + l.ConnectionID = conn.ID() + } + return log +} + +// InsertChannelLogs writes the given channel logs to the db +func InsertChannelLogs(ctx context.Context, db *sqlx.DB, logs []*ChannelLog) error { + ls := make([]interface{}, len(logs)) + for i := range logs { + ls[i] = &logs[i].l + } + + err := BulkSQL(ctx, "insert channel log", db, insertChannelLogSQL, ls) + if err != nil { + return errors.Wrapf(err, "error inserting channel log") + } + return nil +} + // InsertChannelLog writes a channel log to the db returning the inserted log func InsertChannelLog(ctx context.Context, db *sqlx.DB, desc string, isError bool, method string, url string, request []byte, status int, response []byte, diff --git a/models/channel_logs_test.go b/models/channel_logs_test.go new file mode 100644 index 000000000..d444c64a5 --- /dev/null +++ b/models/channel_logs_test.go @@ -0,0 +1,45 @@ +package models_test + +import ( + "net/http" + "testing" + + "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/testsuite" + + "github.com/stretchr/testify/require" +) + +func TestChannelLogs(t *testing.T) { + ctx := testsuite.CTX() + db := testsuite.DB() + + db.MustExec(`DELETE FROM channels_channellog;`) + + defer httpx.SetRequestor(httpx.DefaultRequestor) + httpx.SetRequestor(httpx.NewMockRequestor(map[string][]httpx.MockResponse{ + "http://rapidpro.io": {httpx.NewMockResponse(200, nil, "OK")}, + "http://rapidpro.io/bad": {httpx.NewMockResponse(400, nil, "Oops")}, + })) + + oa, err := models.GetOrgAssets(ctx, db, models.Org1) + require.NoError(t, err) + + channel := oa.ChannelByID(models.TwilioChannelID) + + req1, _ := httpx.NewRequest("GET", "http://rapidpro.io", nil, nil) + trace1, err := httpx.DoTrace(http.DefaultClient, req1, nil, nil, -1) + log1 := models.NewChannelLog(trace1, false, "test request", channel, nil) + + req2, _ := httpx.NewRequest("GET", "http://rapidpro.io/bad", nil, nil) + trace2, err := httpx.DoTrace(http.DefaultClient, req2, nil, nil, -1) + log2 := models.NewChannelLog(trace2, true, "test request", channel, nil) + + err = models.InsertChannelLogs(ctx, db, []*models.ChannelLog{log1, log2}) + require.NoError(t, err) + + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM channels_channellog`, nil, 2) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM channels_channellog WHERE url = 'http://rapidpro.io' AND is_error = FALSE AND channel_id = $1`, []interface{}{channel.ID()}, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM channels_channellog WHERE url = 'http://rapidpro.io/bad' AND is_error = TRUE AND channel_id = $1`, []interface{}{channel.ID()}, 1) +} diff --git a/models/contacts.go b/models/contacts.go index f7b565b33..cb0b81e75 100644 --- a/models/contacts.go +++ b/models/contacts.go @@ -19,10 +19,10 @@ import ( "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/null" - "github.com/olivere/elastic" "github.com/jmoiron/sqlx" "github.com/lib/pq" + "github.com/olivere/elastic" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -41,6 +41,31 @@ const ( NilContactID = ContactID(0) ) +// ContactStatus is the type for contact statuses +type ContactStatus string + +// possible contact statuses +const ( + ContactStatusActive = "A" + ContactStatusBlocked = "B" + ContactStatusStopped = "S" + ContactStatusArchived = "V" +) + +var contactToModelStatus = map[flows.ContactStatus]ContactStatus{ + flows.ContactStatusActive: ContactStatusActive, + flows.ContactStatusBlocked: ContactStatusBlocked, + flows.ContactStatusStopped: ContactStatusStopped, + flows.ContactStatusArchived: ContactStatusArchived, +} + +var contactToFlowStatus = map[ContactStatus]flows.ContactStatus{ + ContactStatusActive: flows.ContactStatusActive, + ContactStatusBlocked: flows.ContactStatusBlocked, + ContactStatusStopped: flows.ContactStatusStopped, + ContactStatusArchived: flows.ContactStatusArchived, +} + // LoadContact loads a contact from the passed in id func LoadContact(ctx context.Context, db Queryer, org *OrgAssets, id ContactID) (*Contact, error) { contacts, err := LoadContacts(ctx, db, org, []ContactID{id}) @@ -76,10 +101,10 @@ func LoadContacts(ctx context.Context, db Queryer, org *OrgAssets, ids []Contact uuid: e.UUID, name: e.Name, language: e.Language, - isStopped: e.IsStopped, - isBlocked: e.IsBlocked, - modifiedOn: e.ModifiedOn, + status: e.Status, createdOn: e.CreatedOn, + modifiedOn: e.ModifiedOn, + lastSeenOn: e.LastSeenOn, } // load our real groups @@ -183,7 +208,7 @@ func ContactIDsFromReferences(ctx context.Context, tx Queryer, org *OrgAssets, r } // BuildElasticQuery turns the passed in contact ql query into an elastic query -func BuildElasticQuery(org *OrgAssets, group assets.GroupUUID, query *contactql.ContactQuery) (elastic.Query, error) { +func BuildElasticQuery(org *OrgAssets, group assets.GroupUUID, query *contactql.ContactQuery) elastic.Query { // filter by org and active contacts eq := elastic.NewBoolQuery().Must( elastic.NewTermQuery("org_id", org.OrgID()), @@ -197,15 +222,11 @@ func BuildElasticQuery(org *OrgAssets, group assets.GroupUUID, query *contactql. // and by our query if present if query != nil { - q, err := es.ToElasticQuery(org.Env(), org.SessionAssets(), query) - if err != nil { - return nil, errors.Wrapf(err, "error converting contactql to elastic query: %s", query) - } - + q := es.ToElasticQuery(org.Env(), query) eq = eq.Must(q) } - return eq, nil + return eq } // ContactIDsForQueryPage returns the ids of the contacts for the passed in query page @@ -220,18 +241,15 @@ func ContactIDsForQueryPage(ctx context.Context, client *elastic.Client, org *Or } if query != "" { - parsed, err = contactql.ParseQuery(query, env.RedactionPolicy(), env.DefaultCountry(), org.SessionAssets()) + parsed, err = contactql.ParseQuery(env, query, org.SessionAssets()) if err != nil { return nil, nil, 0, errors.Wrapf(err, "error parsing query: %s", query) } } - eq, err := BuildElasticQuery(org, group, parsed) - if err != nil { - return nil, nil, 0, errors.Wrapf(err, "error parsing query: %s", query) - } + eq := BuildElasticQuery(org, group, parsed) - fieldSort, err := es.ToElasticFieldSort(org.SessionAssets(), sort) + fieldSort, err := es.ToElasticFieldSort(sort, org.SessionAssets()) if err != nil { return nil, nil, 0, errors.Wrapf(err, "error parsing sort") } @@ -282,15 +300,12 @@ func ContactIDsForQuery(ctx context.Context, client *elastic.Client, org *OrgAss } // turn into elastic query - parsed, err := contactql.ParseQuery(query, env.RedactionPolicy(), env.DefaultCountry(), org.SessionAssets()) + parsed, err := contactql.ParseQuery(env, query, org.SessionAssets()) if err != nil { return nil, errors.Wrapf(err, "error parsing query: %s", query) } - eq, err := BuildElasticQuery(org, "", parsed) - if err != nil { - return nil, errors.Wrapf(err, "error converting contactql to elastic query: %s", query) - } + eq := BuildElasticQuery(org, "", parsed) // only include unblocked and unstopped contacts eq = elastic.NewBoolQuery().Must( @@ -346,9 +361,10 @@ func (c *Contact) FlowContact(org *OrgAssets) (*flows.Contact, error) { flows.ContactID(c.id), c.name, c.language, - c.Status(), + contactToFlowStatus[c.Status()], org.Env().Timezone(), c.createdOn, + c.lastSeenOn, c.urns, groups, c.fields, @@ -372,13 +388,13 @@ func (c *Contact) URNForID(urnID URNID) urns.URN { return urns.NilURN } -// Unstop sets the is_stopped attribute to false for this contact +// Unstop sets the status to stopped for this contact func (c *Contact) Unstop(ctx context.Context, db *sqlx.DB) error { - _, err := db.ExecContext(ctx, `UPDATE contacts_contact SET is_stopped = FALSE, modified_on = NOW() WHERE id = $1`, c.id) + _, err := db.ExecContext(ctx, `UPDATE contacts_contact SET status = 'A', modified_on = NOW() WHERE id = $1`, c.id) if err != nil { return errors.Wrapf(err, "error unstopping contact") } - c.isStopped = false + c.status = ContactStatusActive return nil } @@ -388,35 +404,26 @@ type Contact struct { uuid flows.ContactUUID name string language envs.Language - isStopped bool - isBlocked bool + status ContactStatus fields map[string]*flows.Value groups []*Group urns []urns.URN - modifiedOn time.Time createdOn time.Time + modifiedOn time.Time + lastSeenOn *time.Time } func (c *Contact) ID() ContactID { return c.id } func (c *Contact) UUID() flows.ContactUUID { return c.uuid } func (c *Contact) Name() string { return c.name } func (c *Contact) Language() envs.Language { return c.language } -func (c *Contact) IsStopped() bool { return c.isStopped } -func (c *Contact) IsBlocked() bool { return c.isBlocked } +func (c *Contact) Status() ContactStatus { return c.status } func (c *Contact) Fields() map[string]*flows.Value { return c.fields } func (c *Contact) Groups() []*Group { return c.groups } func (c *Contact) URNs() []urns.URN { return c.urns } -func (c *Contact) ModifiedOn() time.Time { return c.modifiedOn } func (c *Contact) CreatedOn() time.Time { return c.createdOn } - -func (c *Contact) Status() flows.ContactStatus { - if c.isBlocked { - return flows.ContactStatusBlocked - } else if c.isStopped { - return flows.ContactStatusStopped - } - return flows.ContactStatusActive -} +func (c *Contact) ModifiedOn() time.Time { return c.modifiedOn } +func (c *Contact) LastSeenOn() *time.Time { return c.lastSeenOn } // fieldValueEnvelope is our utility struct for the value of a field type fieldValueEnvelope struct { @@ -473,13 +480,13 @@ type contactEnvelope struct { UUID flows.ContactUUID `json:"uuid"` Name string `json:"name"` Language envs.Language `json:"language"` - IsStopped bool `json:"is_stopped"` - IsBlocked bool `json:"is_blocked"` + Status ContactStatus `json:"status"` Fields map[assets.FieldUUID]*fieldValueEnvelope `json:"fields"` GroupIDs []GroupID `json:"group_ids"` URNs []ContactURN `json:"urns"` - ModifiedOn time.Time `json:"modified_on"` CreatedOn time.Time `json:"created_on"` + ModifiedOn time.Time `json:"modified_on"` + LastSeenOn *time.Time `json:"last_seen_on"` } const selectContactSQL = ` @@ -489,11 +496,11 @@ SELECT ROW_TO_JSON(r) FROM (SELECT uuid, name, language, - is_stopped, - is_blocked, + status, is_active, created_on, modified_on, + last_seen_on, fields, g.groups AS group_ids, u.urns AS urns @@ -623,9 +630,9 @@ func CreateContact(ctx context.Context, db *sqlx.DB, org *OrgAssets, urn urns.UR err = tx.GetContext(ctx, &contactID, `INSERT INTO contacts_contact - (org_id, is_active, is_blocked, is_stopped, uuid, created_on, modified_on, created_by_id, modified_by_id, name) + (org_id, is_active, status, uuid, created_on, modified_on, created_by_id, modified_by_id, name) VALUES - ($1, TRUE, FALSE, FALSE, $2, NOW(), NOW(), 1, 1, '') + ($1, TRUE, 'A', $2, NOW(), NOW(), 1, 1, '') RETURNING id`, org.OrgID(), uuids.New(), ) @@ -925,12 +932,18 @@ const markContactStoppedSQL = ` UPDATE contacts_contact SET - is_stopped = TRUE, + status = 'S', modified_on = NOW() WHERE id = $1 ` +// UpdateLastSeenOn updates last seen on (and modified on) +func (c *Contact) UpdateLastSeenOn(ctx context.Context, tx Queryer, lastSeenOn time.Time) error { + c.lastSeenOn = &lastSeenOn + return UpdateContactLastSeenOn(ctx, tx, c.id, lastSeenOn) +} + // UpdatePreferredURN updates the URNs for the contact (if needbe) to have the passed in URN as top priority // with the passed in channel as the preferred channel func (c *Contact) UpdatePreferredURN(ctx context.Context, tx Queryer, org *OrgAssets, urnID URNID, channel *Channel) error { @@ -1077,6 +1090,12 @@ func UpdateContactModifiedOn(ctx context.Context, tx Queryer, contactIDs []Conta return err } +// UpdateContactLastSeenOn updates last seen on (and modified on) on the passed in contact +func UpdateContactLastSeenOn(ctx context.Context, tx Queryer, contactID ContactID, lastSeenOn time.Time) error { + _, err := tx.ExecContext(ctx, `UPDATE contacts_contact SET last_seen_on = $2, modified_on = NOW() WHERE id = $1`, contactID, lastSeenOn) + return err +} + // UpdateContactURNs updates the contact urns in our database to match the passed in changes func UpdateContactURNs(ctx context.Context, tx Queryer, org *OrgAssets, changes []*ContactURNsChanged) error { // keep track of all our inserts @@ -1308,9 +1327,8 @@ type ContactStatusChange struct { } type contactStatusUpdate struct { - ContactID ContactID `db:"id"` - Blocked bool `db:"is_blocked"` - Stopped bool `db:"is_stopped"` + ContactID ContactID `db:"id"` + Status ContactStatus `db:"status"` } // UpdateContactStatus updates the contacts status as the passed changes @@ -1322,6 +1340,7 @@ func UpdateContactStatus(ctx context.Context, tx Queryer, changes []*ContactStat for _, ch := range changes { blocked := ch.Status == flows.ContactStatusBlocked stopped := ch.Status == flows.ContactStatusStopped + status := contactToModelStatus[ch.Status] if blocked || stopped { archiveTriggersForContactIDs = append(archiveTriggersForContactIDs, ch.ContactID) @@ -1331,8 +1350,7 @@ func UpdateContactStatus(ctx context.Context, tx Queryer, changes []*ContactStat statusUpdates, &contactStatusUpdate{ ContactID: ch.ContactID, - Blocked: blocked, - Stopped: stopped, + Status: status, }, ) @@ -1356,13 +1374,12 @@ const updateContactStatusSQL = ` UPDATE contacts_contact c SET - is_blocked = r.is_blocked::boolean, - is_stopped = r.is_stopped::boolean, + status = r.status, modified_on = NOW() FROM ( - VALUES(:id, :is_blocked, :is_stopped) + VALUES(:id, :status) ) AS - r(id, is_blocked, is_stopped) + r(id, status) WHERE c.id = r.id::int ` diff --git a/models/contacts_test.go b/models/contacts_test.go index dbde3ce89..c82a6c53c 100644 --- a/models/contacts_test.go +++ b/models/contacts_test.go @@ -3,6 +3,7 @@ package models import ( "fmt" "testing" + "time" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/flows" @@ -11,6 +12,7 @@ import ( "github.com/olivere/elastic" "github.com/shopspring/decimal" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestElasticContacts(t *testing.T) { @@ -313,16 +315,65 @@ func TestCreateContact(t *testing.T) { assert.NoError(t, err, "%d: error creating contact", i) assert.Equal(t, tc.ContactID, id, "%d: mismatch in contact id", i) } +} + +func TestStopContact(t *testing.T) { + ctx := testsuite.CTX() + db := testsuite.DB() // stop kathy - err = StopContact(ctx, db, Org1, CathyID) + err := StopContact(ctx, db, Org1, CathyID) assert.NoError(t, err) // verify she's only in the stopped group testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contactgroup_contacts WHERE contact_id = $1`, []interface{}{CathyID}, 1) // verify she's stopped - testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND is_stopped = TRUE AND is_active = TRUE and is_blocked = FALSE`, []interface{}{CathyID}, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND status = 'S' AND is_active = TRUE`, []interface{}{CathyID}, 1) +} + +func TestUpdateContactLastSeenAndModifiedOn(t *testing.T) { + ctx := testsuite.CTX() + db := testsuite.DB() + testsuite.Reset() + + oa, err := GetOrgAssets(ctx, db, Org1) + require.NoError(t, err) + + t0 := time.Now() + + err = UpdateContactModifiedOn(ctx, db, []ContactID{CathyID}) + assert.NoError(t, err) + + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE modified_on > $1 AND last_seen_on IS NULL`, []interface{}{t0}, 1) + + t1 := time.Now().Truncate(time.Millisecond) + time.Sleep(time.Millisecond * 5) + + err = UpdateContactLastSeenOn(ctx, db, CathyID, t1) + assert.NoError(t, err) + + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE modified_on > $1 AND last_seen_on = $1`, []interface{}{t1}, 1) + + cathy, err := LoadContact(ctx, db, oa, CathyID) + require.NoError(t, err) + assert.NotNil(t, cathy.LastSeenOn()) + assert.True(t, t1.Equal(*cathy.LastSeenOn())) + assert.True(t, cathy.ModifiedOn().After(t1)) + + t2 := time.Now().Truncate(time.Millisecond) + time.Sleep(time.Millisecond * 5) + + // can update directly from the contact object + err = cathy.UpdateLastSeenOn(ctx, db, t2) + require.NoError(t, err) + assert.True(t, t2.Equal(*cathy.LastSeenOn())) + + // and that also updates the database + cathy, err = LoadContact(ctx, db, oa, CathyID) + require.NoError(t, err) + assert.True(t, t2.Equal(*cathy.LastSeenOn())) + assert.True(t, cathy.ModifiedOn().After(t2)) } func TestUpdateContactModifiedBy(t *testing.T) { @@ -344,7 +395,6 @@ func TestUpdateContactModifiedBy(t *testing.T) { assert.NoError(t, err) testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND modified_by_id = $2`, []interface{}{CathyID, UserID(1)}, 1) - } func TestUpdateContactStatus(t *testing.T) { @@ -355,24 +405,24 @@ func TestUpdateContactStatus(t *testing.T) { err := UpdateContactStatus(ctx, db, []*ContactStatusChange{}) assert.NoError(t, err) - testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND is_blocked = TRUE`, []interface{}{CathyID}, 0) - testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND is_stopped = TRUE`, []interface{}{CathyID}, 0) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND status = 'B'`, []interface{}{CathyID}, 0) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND status = 'S'`, []interface{}{CathyID}, 0) changes := make([]*ContactStatusChange, 0, 1) changes = append(changes, &ContactStatusChange{CathyID, flows.ContactStatusBlocked}) err = UpdateContactStatus(ctx, db, changes) - testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND is_blocked = TRUE`, []interface{}{CathyID}, 1) - testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND is_stopped = TRUE`, []interface{}{CathyID}, 0) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND status = 'B'`, []interface{}{CathyID}, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND status = 'S'`, []interface{}{CathyID}, 0) changes = make([]*ContactStatusChange, 0, 1) changes = append(changes, &ContactStatusChange{CathyID, flows.ContactStatusStopped}) err = UpdateContactStatus(ctx, db, changes) - testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND is_blocked = TRUE`, []interface{}{CathyID}, 0) - testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND is_stopped = TRUE`, []interface{}{CathyID}, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND status = 'B'`, []interface{}{CathyID}, 0) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND status = 'S'`, []interface{}{CathyID}, 1) } diff --git a/models/events.go b/models/events.go index e3375c0cd..3ddb1c23e 100644 --- a/models/events.go +++ b/models/events.go @@ -55,9 +55,6 @@ func (s *Scene) ContactUUID() flows.ContactUUID { return s.contact.UUID() } // Session returns the session for this scene if any func (s *Scene) Session() *Session { - if s.session == nil { - panic("attempt to retrieve session on scene without one") - } return s.session } @@ -184,3 +181,56 @@ func ApplyEventPostCommitHooks(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, return nil } + +// HandleAndCommitEvents takes a set of contacts and events, handles the events and applies any hooks, and commits everything +func HandleAndCommitEvents(ctx context.Context, db *sqlx.DB, rp *redis.Pool, oa *OrgAssets, contactEvents map[*flows.Contact][]flows.Event) error { + // create scenes for each contact + scenes := make([]*Scene, 0, len(contactEvents)) + for contact := range contactEvents { + scene := NewSceneForContact(contact) + scenes = append(scenes, scene) + } + + // begin the transaction for handling and pre-commit hooks + tx, err := db.BeginTxx(ctx, nil) + if err != nil { + return errors.Wrapf(err, "error beginning transaction") + } + + // handle the events to create the hooks on each scene + for _, scene := range scenes { + err := HandleEvents(ctx, tx, rp, oa, scene, contactEvents[scene.Contact()]) + if err != nil { + return errors.Wrapf(err, "error applying events") + } + } + + // gather all our pre commit events, group them by hook and apply them + err = ApplyEventPreCommitHooks(ctx, tx, rp, oa, scenes) + if err != nil { + return errors.Wrapf(err, "error applying pre commit hooks") + } + + // commit the transaction + if err := tx.Commit(); err != nil { + return errors.Wrapf(err, "error committing pre commit hooks") + } + + // begin the transaction for post-commit hooks + tx, err = db.BeginTxx(ctx, nil) + if err != nil { + return errors.Wrapf(err, "error beginning transaction for post commit") + } + + // apply the post commit hooks + err = ApplyEventPostCommitHooks(ctx, tx, rp, oa, scenes) + if err != nil { + return errors.Wrapf(err, "error applying post commit hooks") + } + + // commit the transaction + if err := tx.Commit(); err != nil { + return errors.Wrapf(err, "error committing post commit hooks") + } + return nil +} diff --git a/models/fields.go b/models/fields.go index 6544ee497..1f1b199b2 100644 --- a/models/fields.go +++ b/models/fields.go @@ -21,6 +21,7 @@ type Field struct { Key string `json:"key"` Name string `json:"name"` FieldType assets.FieldType `json:"field_type"` + System bool `json:"is_system"` } } @@ -39,29 +40,39 @@ func (f *Field) Name() string { return f.f.Name } // Type returns the value type for this field func (f *Field) Type() assets.FieldType { return f.f.FieldType } +// System returns whether this is a system field +func (f *Field) System() bool { return f.f.System } + // loadFields loads the assets for the passed in db -func loadFields(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets.Field, error) { +func loadFields(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets.Field, []assets.Field, error) { start := time.Now() rows, err := db.Queryx(selectFieldsSQL, orgID) if err != nil { - return nil, errors.Wrapf(err, "error querying fields for org: %d", orgID) + return nil, nil, errors.Wrapf(err, "error querying fields for org: %d", orgID) } defer rows.Close() - fields := make([]assets.Field, 0, 10) + userFields := make([]assets.Field, 0, 10) + systemFields := make([]assets.Field, 0, 10) + for rows.Next() { field := &Field{} err = readJSONRow(rows, &field.f) if err != nil { - return nil, errors.Wrap(err, "error reading field") + return nil, nil, errors.Wrap(err, "error reading field") + } + + if field.System() { + systemFields = append(systemFields, field) + } else { + userFields = append(userFields, field) } - fields = append(fields, field) } - logrus.WithField("elapsed", time.Since(start)).WithField("org_id", orgID).WithField("count", len(fields)).Debug("loaded contact fields") + logrus.WithField("elapsed", time.Since(start)).WithField("org_id", orgID).WithField("count", len(userFields)).Debug("loaded contact fields") - return fields, nil + return userFields, systemFields, nil } const selectFieldsSQL = ` @@ -77,13 +88,13 @@ SELECT ROW_TO_JSON(f) FROM (SELECT WHEN 'S' THEN 'state' WHEN 'I' THEN 'district' WHEN 'W' THEN 'ward' - END) as field_type -FROM + END) as field_type, + field_type = 'S' as is_system +FROM contacts_contactfield WHERE org_id = $1 AND - is_active = TRUE AND - field_type = 'U' + is_active = TRUE ORDER BY key ASC ) f; diff --git a/models/fields_test.go b/models/fields_test.go index 07b31ab4b..4595aa417 100644 --- a/models/fields_test.go +++ b/models/fields_test.go @@ -12,10 +12,10 @@ func TestFields(t *testing.T) { ctx := testsuite.CTX() db := testsuite.DB() - fields, err := loadFields(ctx, db, 1) + userFields, systemFields, err := loadFields(ctx, db, 1) assert.NoError(t, err) - tcs := []struct { + expectedUserFields := []struct { Key string Name string ValueType assets.FieldType @@ -28,10 +28,29 @@ func TestFields(t *testing.T) { {"ward", "Ward", assets.FieldTypeWard}, } - assert.Equal(t, 6, len(fields)) - for i, tc := range tcs { - assert.Equal(t, tc.Key, fields[i].Key()) - assert.Equal(t, tc.Name, fields[i].Name()) - assert.Equal(t, tc.ValueType, fields[i].Type()) + assert.Equal(t, len(expectedUserFields), len(userFields)) + for i, tc := range expectedUserFields { + assert.Equal(t, tc.Key, userFields[i].Key()) + assert.Equal(t, tc.Name, userFields[i].Name()) + assert.Equal(t, tc.ValueType, userFields[i].Type()) + } + + expectedSystemFields := []struct { + Key string + Name string + ValueType assets.FieldType + }{ + {"created_on", "Created On", assets.FieldTypeDatetime}, + {"id", "ID", assets.FieldTypeNumber}, + {"language", "Language", assets.FieldTypeText}, + {"last_seen_on", "Last Seen On", assets.FieldTypeDatetime}, + {"name", "Name", assets.FieldTypeText}, + } + + assert.Equal(t, len(expectedSystemFields), len(systemFields)) + for i, tc := range expectedSystemFields { + assert.Equal(t, tc.Key, systemFields[i].Key()) + assert.Equal(t, tc.Name, systemFields[i].Name()) + assert.Equal(t, tc.ValueType, systemFields[i].Type()) } } diff --git a/models/http_logs_test.go b/models/http_logs_test.go index 3210f3798..3324e26b6 100644 --- a/models/http_logs_test.go +++ b/models/http_logs_test.go @@ -5,8 +5,8 @@ import ( "testing" "time" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" diff --git a/models/msgs.go b/models/msgs.go index 7d1f3e149..d64b989e2 100644 --- a/models/msgs.go +++ b/models/msgs.go @@ -19,7 +19,7 @@ import ( "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/config" - "github.com/nyaruka/mailroom/gsm7" + "github.com/nyaruka/mailroom/utils/gsm7" "github.com/nyaruka/null" "github.com/gomodule/redigo/redis" @@ -281,21 +281,26 @@ func NewOutgoingIVR(orgID OrgID, conn *ChannelConnection, out *flows.MsgOut, cre return msg, nil } -// NewOutgoingMsg creates an outgoing message for the passed in flow message. Note that this message is created in a queued state! -func NewOutgoingMsg(orgID OrgID, channel *Channel, contactID ContactID, out *flows.MsgOut, createdOn time.Time) (*Msg, error) { +// NewOutgoingMsg creates an outgoing message for the passed in flow message. +func NewOutgoingMsg(org *Org, channel *Channel, contactID ContactID, out *flows.MsgOut, createdOn time.Time) (*Msg, error) { msg := &Msg{} - m := &msg.m + // we fail messages for suspended orgs right away + status := MsgStatusQueued + if org.Suspended() { + status = MsgStatusFailed + } + m.UUID = out.UUID() m.Text = out.Text() m.HighPriority = false m.Direction = DirectionOut - m.Status = MsgStatusQueued + m.Status = status m.Visibility = VisibilityVisible m.MsgType = TypeFlow m.ContactID = contactID - m.OrgID = orgID + m.OrgID = org.ID() m.TopupID = NilTopupID m.CreatedOn = createdOn @@ -769,7 +774,7 @@ func CreateBroadcastMessages(ctx context.Context, db Queryer, rp *redis.Pool, oa // utility method to build up our message buildMessage := func(c *Contact, forceURN urns.URN) (*Msg, error) { - if c.IsStopped() || c.IsBlocked() { + if c.Status() != ContactStatusActive { return nil, nil } @@ -878,7 +883,7 @@ func CreateBroadcastMessages(ctx context.Context, db Queryer, rp *redis.Pool, oa // create our outgoing message out := flows.NewMsgOut(urn, channel.ChannelReference(), text, t.Attachments, t.QuickReplies, nil, flows.NilMsgTopic) - msg, err := NewOutgoingMsg(oa.OrgID(), channel, c.ID(), out, time.Now()) + msg, err := NewOutgoingMsg(oa.Org(), channel, c.ID(), out, time.Now()) msg.SetBroadcastID(bcast.BroadcastID()) if err != nil { return nil, errors.Wrapf(err, "error creating outgoing message") diff --git a/models/msgs_test.go b/models/msgs_test.go index 07d596a4f..5856f1925 100644 --- a/models/msgs_test.go +++ b/models/msgs_test.go @@ -1,4 +1,4 @@ -package models +package models_test import ( "fmt" @@ -10,116 +10,127 @@ import ( "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/config" + "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func TestMsgs(t *testing.T) { +func TestOutgoingMsgs(t *testing.T) { ctx := testsuite.CTX() db := testsuite.DB() - orgID := OrgID(1) - channels, err := loadChannels(ctx, db, orgID) - assert.NoError(t, err) - - channel := channels[0].(*Channel) - chanUUID := channels[0].UUID() - tcs := []struct { - ChannelUUID assets.ChannelUUID - Channel *Channel - Text string - ContactID ContactID - URN urns.URN - ContactURNID URNID - Attachments []utils.Attachment - QuickReplies []string - Topic flows.MsgTopic + ChannelUUID assets.ChannelUUID + Text string + ContactID models.ContactID + URN urns.URN + URNID models.URNID + Attachments []utils.Attachment + QuickReplies []string + Topic flows.MsgTopic + SuspendedOrg bool + + ExpectedStatus models.MsgStatus ExpectedMetadata map[string]interface{} ExpectedMsgCount int - HasErr bool + HasError bool }{ { - chanUUID, channel, - "missing urn id", - CathyID, - urns.URN("tel:+250700000001"), - URNID(0), - nil, - nil, - flows.NilMsgTopic, - map[string]interface{}{}, - 1, - true, + ChannelUUID: "74729f45-7f29-4868-9dc4-90e491e3c7d8", + Text: "missing urn id", + ContactID: models.CathyID, + URN: urns.URN("tel:+250700000001"), + URNID: models.URNID(0), + ExpectedStatus: models.MsgStatusQueued, + ExpectedMetadata: map[string]interface{}{}, + ExpectedMsgCount: 1, + HasError: true, }, { - chanUUID, - channel, - "test outgoing", - CathyID, - urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", CathyURNID)), - CathyURNID, - nil, - []string{"yes", "no"}, - flows.MsgTopicPurchase, - map[string]interface{}{ + ChannelUUID: "74729f45-7f29-4868-9dc4-90e491e3c7d8", + Text: "test outgoing", + ContactID: models.CathyID, + URN: urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)), + URNID: models.CathyURNID, + QuickReplies: []string{"yes", "no"}, + Topic: flows.MsgTopicPurchase, + ExpectedStatus: models.MsgStatusQueued, + ExpectedMetadata: map[string]interface{}{ "quick_replies": []string{"yes", "no"}, "topic": "purchase", }, - 1, - false, + ExpectedMsgCount: 1, }, { - chanUUID, - channel, - "test outgoing", - CathyID, - urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", CathyURNID)), - CathyURNID, - []utils.Attachment{utils.Attachment("image/jpeg:https://dl-foo.com/image.jpg")}, - nil, - flows.NilMsgTopic, - map[string]interface{}{}, - 2, - false}, + ChannelUUID: "74729f45-7f29-4868-9dc4-90e491e3c7d8", + Text: "test outgoing", + ContactID: models.CathyID, + URN: urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)), + URNID: models.CathyURNID, + Attachments: []utils.Attachment{utils.Attachment("image/jpeg:https://dl-foo.com/image.jpg")}, + ExpectedStatus: models.MsgStatusQueued, + ExpectedMetadata: map[string]interface{}{}, + ExpectedMsgCount: 2, + }, + { + ChannelUUID: "74729f45-7f29-4868-9dc4-90e491e3c7d8", + Text: "suspended org", + ContactID: models.CathyID, + URN: urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)), + URNID: models.CathyURNID, + SuspendedOrg: true, + ExpectedStatus: models.MsgStatusFailed, + ExpectedMetadata: map[string]interface{}{}, + ExpectedMsgCount: 1, + }, } now := time.Now() - time.Sleep(time.Millisecond * 10) for _, tc := range tcs { tx, err := db.BeginTxx(ctx, nil) - assert.NoError(t, err) + require.NoError(t, err) + + db.MustExec(`UPDATE orgs_org SET is_suspended = $1 WHERE id = $2`, tc.SuspendedOrg, models.Org1) + + oa, err := models.GetOrgAssetsWithRefresh(ctx, db, models.Org1, models.RefreshOrg) + require.NoError(t, err) + + channel := oa.ChannelByUUID(tc.ChannelUUID) flowMsg := flows.NewMsgOut(tc.URN, assets.NewChannelReference(tc.ChannelUUID, "Test Channel"), tc.Text, tc.Attachments, tc.QuickReplies, nil, tc.Topic) - msg, err := NewOutgoingMsg(orgID, tc.Channel, tc.ContactID, flowMsg, now) + msg, err := models.NewOutgoingMsg(oa.Org(), channel, tc.ContactID, flowMsg, now) + + if tc.HasError { + assert.Error(t, err) + } else { + assert.NoError(t, err) - if err == nil { - assert.False(t, tc.HasErr) - err = InsertMessages(ctx, tx, []*Msg{msg}) + err = models.InsertMessages(ctx, tx, []*models.Msg{msg}) assert.NoError(t, err) - assert.Equal(t, orgID, msg.OrgID()) + assert.Equal(t, oa.OrgID(), msg.OrgID()) assert.Equal(t, tc.Text, msg.Text()) assert.Equal(t, tc.ContactID, msg.ContactID()) - assert.Equal(t, tc.Channel, msg.Channel()) + assert.Equal(t, channel, msg.Channel()) assert.Equal(t, tc.ChannelUUID, msg.ChannelUUID()) assert.Equal(t, tc.URN, msg.URN()) - if tc.ContactURNID != NilURNID { - assert.Equal(t, tc.ContactURNID, *msg.ContactURNID()) + if tc.URNID != models.NilURNID { + assert.Equal(t, tc.URNID, *msg.ContactURNID()) } else { assert.Nil(t, msg.ContactURNID()) } + + assert.Equal(t, tc.ExpectedStatus, msg.Status()) assert.Equal(t, tc.ExpectedMetadata, msg.Metadata()) assert.Equal(t, tc.ExpectedMsgCount, msg.MsgCount()) assert.Equal(t, now, msg.CreatedOn()) assert.True(t, msg.ID() > 0) assert.True(t, msg.QueuedOn().After(now)) assert.True(t, msg.ModifiedOn().After(now)) - } else { - if !tc.HasErr { - assert.Fail(t, "unexpected error: %s", err.Error()) - } } + tx.Rollback() } } @@ -140,6 +151,6 @@ func TestNormalizeAttachment(t *testing.T) { } for _, tc := range tcs { - assert.Equal(t, tc.normalized, string(NormalizeAttachment(utils.Attachment(tc.raw)))) + assert.Equal(t, tc.normalized, string(models.NormalizeAttachment(utils.Attachment(tc.raw)))) } } diff --git a/models/orgs.go b/models/orgs.go index f609f1e3c..a7f3bf63f 100644 --- a/models/orgs.go +++ b/models/orgs.go @@ -3,17 +3,22 @@ package models import ( "context" "encoding/json" + "fmt" "net/http" + "path/filepath" + "strings" "time" + "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/services/airtime/dtone" "github.com/nyaruka/goflow/services/email/smtp" - "github.com/nyaruka/goflow/utils/httpx" - "github.com/nyaruka/goflow/utils/jsonx" + "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/goflow" + "github.com/nyaruka/mailroom/utils/storage" "github.com/nyaruka/null" "github.com/jmoiron/sqlx" @@ -164,6 +169,44 @@ func (o *Org) AirtimeService(httpClient *http.Client, httpRetries *httpx.RetryCo return dtone.NewService(httpClient, httpRetries, login, token, currency), nil } +// StoreAttachment saves an attachment to storage +func (o *Org) StoreAttachment(s storage.Storage, prefix string, filename string, content []byte) (utils.Attachment, error) { + contentType := http.DetectContentType(content) + + path := o.attachmentPath(prefix, filename) + + url, err := s.Put(path, contentType, content) + if err != nil { + return "", err + } + + return utils.Attachment(contentType + ":" + url), nil +} + +func (o *Org) attachmentPath(prefix string, filename string) string { + parts := []string{prefix, fmt.Sprintf("%d", o.ID())} + + // not all filesystems like having a directory with a huge number of files, so if filename is long enough, + // use parts of it to create intermediate subdirectories + if len(filename) > 4 { + parts = append(parts, filename[:4]) + + if len(filename) > 8 { + parts = append(parts, filename[4:8]) + } + } + parts = append(parts, filename) + + path := filepath.Join(parts...) + + // ensure path begins with / + if !strings.HasPrefix(path, "/") { + path = fmt.Sprintf("/%s", path) + } + + return path +} + // gets the underlying org for the given engine session func orgFromSession(session flows.Session) *Org { return session.Assets().Source().(*OrgAssets).Org() diff --git a/models/orgs_test.go b/models/orgs_test.go index 363d04692..bba3ca08e 100644 --- a/models/orgs_test.go +++ b/models/orgs_test.go @@ -1,13 +1,16 @@ package models import ( + "io/ioutil" "testing" "time" "github.com/nyaruka/goflow/envs" + "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/testsuite" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestOrgs(t *testing.T) { @@ -53,3 +56,22 @@ func TestOrgs(t *testing.T) { _, err = loadOrg(ctx, tx, 99) assert.Error(t, err) } + +func TestStoreAttachment(t *testing.T) { + ctx := testsuite.CTX() + db := testsuite.DB() + + store := testsuite.Storage() + defer testsuite.ResetStorage() + + image, err := ioutil.ReadFile("testdata/test.jpg") + require.NoError(t, err) + + org, err := loadOrg(ctx, db, Org1) + assert.NoError(t, err) + + attachment, err := org.StoreAttachment(store, "media", "668383ba-387c-49bc-b164-1213ac0ea7aa.jpg", image) + require.NoError(t, err) + + assert.Equal(t, utils.Attachment("image/jpeg:_test_storage/media/1/6683/83ba/668383ba-387c-49bc-b164-1213ac0ea7aa.jpg"), attachment) +} diff --git a/models/starts.go b/models/starts.go index 7944aaf5a..a4d8a78f6 100644 --- a/models/starts.go +++ b/models/starts.go @@ -6,7 +6,9 @@ import ( "encoding/json" "github.com/jmoiron/sqlx" + "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/null" "github.com/pkg/errors" @@ -91,8 +93,9 @@ type FlowStartBatch struct { FlowType FlowType `json:"flow_type"` ContactIDs []ContactID `json:"contact_ids"` - ParentSummary null.JSON `json:"parent_summary,omitempty"` - Extra null.JSON `json:"extra,omitempty"` + ParentSummary null.JSON `json:"parent_summary,omitempty"` + SessionHistory null.JSON `json:"session_history,omitempty"` + Extra null.JSON `json:"extra,omitempty"` RestartParticipants RestartParticipants `json:"restart_participants"` IncludeActive IncludeActive `json:"include_active"` @@ -113,8 +116,9 @@ func (b *FlowStartBatch) IncludeActive() IncludeActive { return b.b. func (b *FlowStartBatch) IsLast() bool { return b.b.IsLast } func (b *FlowStartBatch) TotalContacts() int { return b.b.TotalContacts } -func (b *FlowStartBatch) ParentSummary() json.RawMessage { return json.RawMessage(b.b.ParentSummary) } -func (b *FlowStartBatch) Extra() json.RawMessage { return json.RawMessage(b.b.Extra) } +func (b *FlowStartBatch) ParentSummary() json.RawMessage { return json.RawMessage(b.b.ParentSummary) } +func (b *FlowStartBatch) SessionHistory() json.RawMessage { return json.RawMessage(b.b.SessionHistory) } +func (b *FlowStartBatch) Extra() json.RawMessage { return json.RawMessage(b.b.Extra) } func (b *FlowStartBatch) MarshalJSON() ([]byte, error) { return json.Marshal(b.b) } func (b *FlowStartBatch) UnmarshalJSON(data []byte) error { return json.Unmarshal(data, &b.b) } @@ -139,8 +143,9 @@ type FlowStart struct { RestartParticipants RestartParticipants `json:"restart_participants" db:"restart_participants"` IncludeActive IncludeActive `json:"include_active" db:"include_active"` - Extra null.JSON `json:"extra,omitempty" db:"extra"` - ParentSummary null.JSON `json:"parent_summary,omitempty" db:"parent_summary"` + Extra null.JSON `json:"extra,omitempty" db:"extra"` + ParentSummary null.JSON `json:"parent_summary,omitempty" db:"parent_summary"` + SessionHistory null.JSON `json:"session_history,omitempty" db:"session_history"` CreatedBy string `json:"created_by"` } @@ -190,6 +195,12 @@ func (s *FlowStart) WithParentSummary(sum json.RawMessage) *FlowStart { return s } +func (s *FlowStart) SessionHistory() json.RawMessage { return json.RawMessage(s.s.SessionHistory) } +func (s *FlowStart) WithSessionHistory(history json.RawMessage) *FlowStart { + s.s.SessionHistory = null.JSON(history) + return s +} + func (s *FlowStart) Extra() json.RawMessage { return json.RawMessage(s.s.Extra) } func (s *FlowStart) WithExtra(extra json.RawMessage) *FlowStart { s.s.Extra = null.JSON(extra) @@ -202,7 +213,7 @@ func (s *FlowStart) UnmarshalJSON(data []byte) error { return json.Unmarshal(dat // GetFlowStartAttributes gets the basic attributes for the passed in start id, this includes ONLY its id, uuid, flow_id and extra func GetFlowStartAttributes(ctx context.Context, db Queryer, startID StartID) (*FlowStart, error) { start := &FlowStart{} - err := db.GetContext(ctx, &start.s, `SELECT id, uuid, flow_id, extra, parent_summary FROM flows_flowstart WHERE id = $1`, startID) + err := db.GetContext(ctx, &start.s, `SELECT id, uuid, flow_id, extra, parent_summary, session_history FROM flows_flowstart WHERE id = $1`, startID) if err != nil { return nil, errors.Wrapf(err, "unable to load start attributes for id: %d", startID) } @@ -290,8 +301,8 @@ func InsertFlowStarts(ctx context.Context, db Queryer, starts []*FlowStart) erro const insertStartSQL = ` INSERT INTO - flows_flowstart(uuid, org_id, flow_id, start_type, created_on, modified_on, restart_participants, include_active, query, status, extra, parent_summary) - VALUES(:uuid, :org_id, :flow_id, :start_type, NOW(), NOW(), :restart_participants, :include_active, :query, 'P', :extra, :parent_summary) + flows_flowstart(uuid, org_id, flow_id, start_type, created_on, modified_on, restart_participants, include_active, query, status, extra, parent_summary, session_history) + VALUES(:uuid, :org_id, :flow_id, :start_type, NOW(), NOW(), :restart_participants, :include_active, :query, 'P', :extra, :parent_summary, :session_history) RETURNING id ` @@ -320,6 +331,7 @@ func (s *FlowStart) CreateBatch(contactIDs []ContactID, last bool, totalContacts b.b.RestartParticipants = s.RestartParticipants() b.b.IncludeActive = s.IncludeActive() b.b.ParentSummary = null.JSON(s.ParentSummary()) + b.b.SessionHistory = null.JSON(s.SessionHistory()) b.b.Extra = null.JSON(s.Extra()) b.b.IsLast = last b.b.TotalContacts = totalContacts @@ -346,3 +358,9 @@ func (i StartID) Value() (driver.Value, error) { func (i *StartID) Scan(value interface{}) error { return null.ScanInt(value, (*null.Int)(i)) } + +// ReadSessionHistory reads a session history from the given JSON +func ReadSessionHistory(data []byte) (*flows.SessionHistory, error) { + h := &flows.SessionHistory{} + return h, jsonx.Unmarshal(data, h) +} diff --git a/models/starts_test.go b/models/starts_test.go index 8327b39f6..21527acac 100644 --- a/models/starts_test.go +++ b/models/starts_test.go @@ -4,6 +4,7 @@ import ( "encoding/json" "testing" + "github.com/nyaruka/goflow/flows" "github.com/nyaruka/mailroom/models" "github.com/stretchr/testify/assert" ) @@ -21,6 +22,8 @@ func TestStarts(t *testing.T) { "query": null, "restart_participants": true, "include_active": true, + "parent_summary": {"uuid": "b65b1a22-db6d-4f5a-9b3d-7302368a82e6"}, + "session_history": {"parent_uuid": "532a3899-492f-4ffe-aed7-e75ad524efab", "ancestors": 3, "ancestors_since_input": 1}, "extra": {"foo": "bar"} }`) @@ -35,6 +38,9 @@ func TestStarts(t *testing.T) { assert.Equal(t, "", start.Query()) assert.Equal(t, models.DoRestartParticipants, start.RestartParticipants()) assert.Equal(t, models.DoIncludeActive, start.IncludeActive()) + + assert.Equal(t, json.RawMessage(`{"uuid": "b65b1a22-db6d-4f5a-9b3d-7302368a82e6"}`), start.ParentSummary()) + assert.Equal(t, json.RawMessage(`{"parent_uuid": "532a3899-492f-4ffe-aed7-e75ad524efab", "ancestors": 3, "ancestors_since_input": 1}`), start.SessionHistory()) assert.Equal(t, json.RawMessage(`{"foo": "bar"}`), start.Extra()) batch := start.CreateBatch([]models.ContactID{4567, 5678}, false, 3) @@ -47,4 +53,15 @@ func TestStarts(t *testing.T) { assert.Equal(t, "rowan@nyaruka.com", batch.CreatedBy()) assert.False(t, batch.IsLast()) assert.Equal(t, 3, batch.TotalContacts()) + + assert.Equal(t, json.RawMessage(`{"uuid": "b65b1a22-db6d-4f5a-9b3d-7302368a82e6"}`), batch.ParentSummary()) + assert.Equal(t, json.RawMessage(`{"parent_uuid": "532a3899-492f-4ffe-aed7-e75ad524efab", "ancestors": 3, "ancestors_since_input": 1}`), batch.SessionHistory()) + assert.Equal(t, json.RawMessage(`{"foo": "bar"}`), batch.Extra()) + + history, err := models.ReadSessionHistory(batch.SessionHistory()) + assert.NoError(t, err) + assert.Equal(t, flows.SessionUUID("532a3899-492f-4ffe-aed7-e75ad524efab"), history.ParentUUID) + + history, err = models.ReadSessionHistory([]byte(`{`)) + assert.EqualError(t, err, "unexpected end of JSON input") } diff --git a/models/test_constants.go b/models/test_constants.go index 5ba6e1eda..4285f0207 100644 --- a/models/test_constants.go +++ b/models/test_constants.go @@ -88,14 +88,15 @@ var AllContactsGroupUUID = assets.GroupUUID("bc268217-9ffa-49e0-883e-e4e09c252a5 var TestersGroupID = GroupID(10001) var TestersGroupUUID = assets.GroupUUID("5e9d8fab-5e7e-4f51-b533-261af5dea70d") +var CreatedOnFieldID = FieldID(3) +var LastSeenOnFieldID = FieldID(5) + var AgeFieldUUID = assets.FieldUUID("903f51da-2717-47c7-a0d3-f2f32877013d") var GenderFieldUUID = assets.FieldUUID("3a5891e4-756e-4dc9-8e12-b7a766168824") -var JoinedFieldID = FieldID(7) +var JoinedFieldID = FieldID(8) var JoinedFieldUUID = assets.FieldUUID("d83aae24-4bbf-49d0-ab85-6bfd201eac6d") -var CreatedOnFieldID = FieldID(3) - var ReportingLabelID = LabelID(10000) var ReportingLabelUUID = assets.LabelUUID("ebc4dedc-91c4-4ed4-9dd6-daa05ea82698") diff --git a/models/testdata/test.jpg b/models/testdata/test.jpg new file mode 100644 index 000000000..f741d1131 Binary files /dev/null and b/models/testdata/test.jpg differ diff --git a/models/tickets.go b/models/tickets.go index dff9844b5..b9f81edd5 100644 --- a/models/tickets.go +++ b/models/tickets.go @@ -7,11 +7,11 @@ import ( "net/http" "time" + "github.com/nyaruka/gocommon/dates" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/dates" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/mailroom/goflow" "github.com/nyaruka/null" diff --git a/models/tickets_test.go b/models/tickets_test.go index 39624cac2..4e37866f0 100644 --- a/models/tickets_test.go +++ b/models/tickets_test.go @@ -3,8 +3,8 @@ package models_test import ( "testing" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/mailroom/models" _ "github.com/nyaruka/mailroom/services/tickets/mailgun" _ "github.com/nyaruka/mailroom/services/tickets/zendesk" diff --git a/runner/runner.go b/runner/runner.go index 571ffb836..7e4f24b15 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -12,9 +12,9 @@ import ( "github.com/nyaruka/goflow/flows/triggers" "github.com/nyaruka/librato" "github.com/nyaruka/mailroom/goflow" - "github.com/nyaruka/mailroom/locker" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" + "github.com/nyaruka/mailroom/utils/locker" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -180,13 +180,21 @@ func StartFlowBatch( } } + var history *flows.SessionHistory + if len(batch.SessionHistory()) > 0 { + history, err = models.ReadSessionHistory(batch.SessionHistory()) + if err != nil { + return nil, errors.Wrap(err, "unable to read JSON from flow start history") + } + } + // whether engine allows some functions is based on whether there is more than one contact being started batchStart := batch.TotalContacts() > 1 // this will build our trigger for each contact started triggerBuilder := func(contact *flows.Contact) flows.Trigger { if batch.ParentSummary() != nil { - tb := triggers.NewBuilder(oa.Env(), flow.FlowReference(), contact).FlowAction(batch.ParentSummary()) + tb := triggers.NewBuilder(oa.Env(), flow.FlowReference(), contact).FlowAction(history, batch.ParentSummary()) if batchStart { tb = tb.AsBatch() } diff --git a/s3utils/s3.go b/s3utils/s3.go deleted file mode 100644 index 9cb30f00c..000000000 --- a/s3utils/s3.go +++ /dev/null @@ -1,43 +0,0 @@ -package s3utils - -import ( - "bytes" - "fmt" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" -) - -var s3BucketURL = "https://%s.s3.amazonaws.com%s" - -// TestS3 tests whether the passed in s3 client is properly configured and the passed in bucket is accessible -func TestS3(s3Client s3iface.S3API, bucket string) error { - params := &s3.HeadBucketInput{ - Bucket: aws.String(bucket), - } - _, err := s3Client.HeadBucket(params) - if err != nil { - return err - } - - return nil -} - -// PutS3File writes the passed in file to the bucket with the passed in content type -func PutS3File(s3Client s3iface.S3API, bucket string, path string, contentType string, contents []byte) (string, error) { - params := &s3.PutObjectInput{ - Bucket: aws.String(bucket), - Body: bytes.NewReader(contents), - Key: aws.String(path), - ContentType: aws.String(contentType), - ACL: aws.String(s3.BucketCannedACLPublicRead), - } - _, err := s3Client.PutObject(params) - if err != nil { - return "", err - } - - url := fmt.Sprintf(s3BucketURL, bucket, path) - return url, nil -} diff --git a/services/tickets/mailgun/client.go b/services/tickets/mailgun/client.go index 6140490b1..b64a7a554 100644 --- a/services/tickets/mailgun/client.go +++ b/services/tickets/mailgun/client.go @@ -7,9 +7,9 @@ import ( "net/http" "sort" + "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/httpx" - "github.com/nyaruka/goflow/utils/jsonx" "github.com/nyaruka/goflow/utils/uuids" "github.com/pkg/errors" ) diff --git a/services/tickets/mailgun/client_test.go b/services/tickets/mailgun/client_test.go index f03bdd092..cb9909d0f 100644 --- a/services/tickets/mailgun/client_test.go +++ b/services/tickets/mailgun/client_test.go @@ -4,7 +4,7 @@ import ( "net/http" "testing" - "github.com/nyaruka/goflow/utils/httpx" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/services/tickets/mailgun" diff --git a/services/tickets/mailgun/service.go b/services/tickets/mailgun/service.go index 301ad7c88..de04e137a 100644 --- a/services/tickets/mailgun/service.go +++ b/services/tickets/mailgun/service.go @@ -7,9 +7,9 @@ import ( "strings" "text/template" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/services/tickets" diff --git a/services/tickets/mailgun/service_test.go b/services/tickets/mailgun/service_test.go index ad72b2f0f..2be0f4370 100644 --- a/services/tickets/mailgun/service_test.go +++ b/services/tickets/mailgun/service_test.go @@ -5,13 +5,13 @@ import ( "testing" "time" + "github.com/nyaruka/gocommon/dates" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/assets/static/types" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/test" - "github.com/nyaruka/goflow/utils/dates" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/services/tickets/mailgun" diff --git a/services/tickets/mailgun/web.go b/services/tickets/mailgun/web.go index 4bf64b36d..858fe8aed 100644 --- a/services/tickets/mailgun/web.go +++ b/services/tickets/mailgun/web.go @@ -112,7 +112,7 @@ func handleReceive(ctx context.Context, s *web.Server, r *http.Request, l *model return errors.Wrapf(err, "error updating ticket: %s", ticket.UUID()), http.StatusInternalServerError, nil } - msg, err := tickets.SendReply(ctx, s.DB, s.RP, ticket, request.StrippedText) + msg, err := tickets.SendReply(ctx, s.DB, s.RP, s.Storage, s.Config.S3MediaPrefix, ticket, request.StrippedText, nil) if err != nil { return err, http.StatusInternalServerError, nil } diff --git a/services/tickets/utils.go b/services/tickets/utils.go index 42d0f44d6..2a2f1d97d 100644 --- a/services/tickets/utils.go +++ b/services/tickets/utils.go @@ -2,12 +2,19 @@ package tickets import ( "context" + "net/http" + "path/filepath" + "time" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/goflow/utils" + "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/courier" "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/utils/storage" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" @@ -69,22 +76,37 @@ func FromTicketerUUID(ctx context.Context, db *sqlx.DB, uuid assets.TicketerUUID } // SendReply sends a message reply from the ticket system user to the contact -func SendReply(ctx context.Context, db *sqlx.DB, rp *redis.Pool, ticket *models.Ticket, text string) (*models.Msg, error) { +func SendReply(ctx context.Context, db *sqlx.DB, rp *redis.Pool, store storage.Storage, mediaPrefix string, ticket *models.Ticket, text string, fileURLs []string) (*models.Msg, error) { // look up our assets - assets, err := models.GetOrgAssets(ctx, db, ticket.OrgID()) + oa, err := models.GetOrgAssets(ctx, db, ticket.OrgID()) if err != nil { return nil, errors.Wrapf(err, "error looking up org #%d", ticket.OrgID()) } - // build a simple translation - translations := map[envs.Language]*models.BroadcastTranslation{ - envs.Language("base"): {Text: text}, + // fetch and files and prepare as attachments + attachments := make([]utils.Attachment, len(fileURLs)) + for i, fileURL := range fileURLs { + fileBody, err := fetchFile(fileURL) + if err != nil { + return nil, errors.Wrapf(err, "error fetching file %s for ticket reply", fileURL) + } + + filename := string(uuids.New()) + filepath.Ext(fileURL) + + attachments[i], err = oa.Org().StoreAttachment(store, mediaPrefix, filename, fileBody) + if err != nil { + return nil, errors.Wrapf(err, "error storing attachment %s for ticket reply", fileURL) + } } + // build a simple translation + base := &models.BroadcastTranslation{Text: text, Attachments: attachments} + translations := map[envs.Language]*models.BroadcastTranslation{envs.Language("base"): base} + // we'll use a broadcast to send this message - bcast := models.NewBroadcast(assets.OrgID(), models.NilBroadcastID, translations, models.TemplateStateEvaluated, envs.Language("base"), nil, nil, nil) + bcast := models.NewBroadcast(oa.OrgID(), models.NilBroadcastID, translations, models.TemplateStateEvaluated, envs.Language("base"), nil, nil, nil) batch := bcast.CreateBatch([]models.ContactID{ticket.ContactID()}) - msgs, err := models.CreateBroadcastMessages(ctx, db, rp, assets, batch) + msgs, err := models.CreateBroadcastMessages(ctx, db, rp, oa, batch) if err != nil { return nil, errors.Wrapf(err, "error creating message batch") } @@ -101,3 +123,17 @@ func SendReply(ctx context.Context, db *sqlx.DB, rp *redis.Pool, ticket *models. } return msg, nil } + +func fetchFile(url string) ([]byte, error) { + req, _ := httpx.NewRequest("GET", url, nil, nil) + + trace, err := httpx.DoTrace(http.DefaultClient, req, httpx.NewFixedRetries(time.Second*5, time.Second*10), nil, 10*1024*1024) + if err != nil { + return nil, err + } + if trace.Response.StatusCode/100 != 2 { + return nil, errors.New("fetch returned non-200 response") + } + + return trace.ResponseBody, nil +} diff --git a/services/tickets/utils_test.go b/services/tickets/utils_test.go index 922aa92df..9d2cc8047 100644 --- a/services/tickets/utils_test.go +++ b/services/tickets/utils_test.go @@ -1,10 +1,14 @@ package tickets_test import ( + "io/ioutil" "testing" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/goflow/utils" + "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/services/tickets" _ "github.com/nyaruka/mailroom/services/tickets/mailgun" @@ -118,6 +122,20 @@ func TestSendReply(t *testing.T) { testsuite.ResetDB() ctx := testsuite.CTX() db := testsuite.DB() + rp := testsuite.RP() + defer testsuite.ResetStorage() + + defer uuids.SetGenerator(uuids.DefaultGenerator) + uuids.SetGenerator(uuids.NewSeededGenerator(12345)) + + image, err := ioutil.ReadFile("../../models/testdata/test.jpg") + require.NoError(t, err) + + defer httpx.SetRequestor(httpx.DefaultRequestor) + httpx.SetRequestor(httpx.NewMockRequestor(map[string][]httpx.MockResponse{ + "http://coolfilesfortickets.com/a.jpg": {httpx.MockResponse{Status: 200, Body: image}}, + "http://badfiles.com/b.jpg": {httpx.MockResponse{Status: 400, Body: nil}}, + })) ticketUUID := flows.TicketUUID("f7358870-c3dd-450d-b5ae-db2eb50216ba") @@ -128,9 +146,15 @@ func TestSendReply(t *testing.T) { ticket, err := models.LookupTicketByUUID(ctx, db, ticketUUID) require.NoError(t, err) - msg, err := tickets.SendReply(ctx, db, testsuite.RP(), ticket, "I'll get back to you") + msg, err := tickets.SendReply(ctx, db, rp, testsuite.Storage(), "media", ticket, "I'll get back to you", []string{"http://coolfilesfortickets.com/a.jpg"}) require.NoError(t, err) assert.Equal(t, "I'll get back to you", msg.Text()) assert.Equal(t, models.CathyID, msg.ContactID()) + assert.Equal(t, []utils.Attachment{"image/jpeg:https:///_test_storage/media/1/1ae9/6956/1ae96956-4b34-433e-8d1a-f05fe6923d6d.jpg"}, msg.Attachments()) + assert.FileExists(t, "_test_storage/media/1/1ae9/6956/1ae96956-4b34-433e-8d1a-f05fe6923d6d.jpg") + + // try with file that can't be fetched + _, err = tickets.SendReply(ctx, db, rp, testsuite.Storage(), "media", ticket, "I'll get back to you", []string{"http://badfiles.com/b.jpg"}) + assert.EqualError(t, err, "error fetching file http://badfiles.com/b.jpg for ticket reply: fetch returned non-200 response") } diff --git a/services/tickets/zendesk/client.go b/services/tickets/zendesk/client.go index 48452fb0f..04ca35a14 100644 --- a/services/tickets/zendesk/client.go +++ b/services/tickets/zendesk/client.go @@ -2,7 +2,6 @@ package zendesk import ( "bytes" - "encoding/json" "errors" "fmt" "io" @@ -10,8 +9,8 @@ import ( "strings" "time" - "github.com/nyaruka/goflow/utils/httpx" - "github.com/nyaruka/goflow/utils/jsonx" + "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/jsonx" ) type baseClient struct { @@ -224,13 +223,19 @@ func NewPushClient(httpClient *http.Client, httpRetries *httpx.RetryConfig, subd return &PushClient{baseClient: newBaseClient(httpClient, httpRetries, subdomain, token)} } +// FieldValue is a value for the named field +type FieldValue struct { + ID string `json:"id"` + Value string `json:"value"` +} + // Author see https://developer.zendesk.com/rest_api/docs/support/channel_framework#author-object type Author struct { - ExternalID string `json:"external_id"` - Name string `json:"name,omitempty"` - ImageURL string `json:"image_url,omitempty"` - Locale string `json:"locale,omitempty"` - Fields json.RawMessage `json:"fields,omitempty"` + ExternalID string `json:"external_id"` + Name string `json:"name,omitempty"` + ImageURL string `json:"image_url,omitempty"` + Locale string `json:"locale,omitempty"` + Fields []FieldValue `json:"fields,omitempty"` } // DisplayInfo see https://developer.zendesk.com/rest_api/docs/support/channel_framework#display_info-object @@ -250,6 +255,8 @@ type ExternalResource struct { Author Author `json:"author"` DisplayInfo []DisplayInfo `json:"display_info,omitempty"` AllowChannelback bool `json:"allow_channelback"` + Fields []FieldValue `json:"fields,omitempty"` + FileURLs []string `json:"file_urls,omitempty"` } // Status see https://developer.zendesk.com/rest_api/docs/support/channel_framework#status-object diff --git a/services/tickets/zendesk/client_test.go b/services/tickets/zendesk/client_test.go index 9419da6eb..800dff00c 100644 --- a/services/tickets/zendesk/client_test.go +++ b/services/tickets/zendesk/client_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/nyaruka/goflow/utils/httpx" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/mailroom/services/tickets/zendesk" "github.com/stretchr/testify/assert" diff --git a/services/tickets/zendesk/service.go b/services/tickets/zendesk/service.go index 50fe626df..0d2c2a399 100644 --- a/services/tickets/zendesk/service.go +++ b/services/tickets/zendesk/service.go @@ -4,10 +4,10 @@ import ( "fmt" "net/http" + "github.com/nyaruka/gocommon/dates" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/dates" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" diff --git a/services/tickets/zendesk/service_test.go b/services/tickets/zendesk/service_test.go index a7da7a581..f6397ac6d 100644 --- a/services/tickets/zendesk/service_test.go +++ b/services/tickets/zendesk/service_test.go @@ -5,13 +5,13 @@ import ( "testing" "time" + "github.com/nyaruka/gocommon/dates" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/assets/static/types" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/test" - "github.com/nyaruka/goflow/utils/dates" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/services/tickets/zendesk" diff --git a/services/tickets/zendesk/utils.go b/services/tickets/zendesk/utils.go index ce4004111..63c8db1db 100644 --- a/services/tickets/zendesk/utils.go +++ b/services/tickets/zendesk/utils.go @@ -6,7 +6,7 @@ import ( "strings" "time" - "github.com/nyaruka/goflow/utils/dates" + "github.com/nyaruka/gocommon/dates" "github.com/nyaruka/mailroom/models" "github.com/pkg/errors" diff --git a/services/tickets/zendesk/utils_test.go b/services/tickets/zendesk/utils_test.go index 8c762528a..aeaf8938e 100644 --- a/services/tickets/zendesk/utils_test.go +++ b/services/tickets/zendesk/utils_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/nyaruka/goflow/utils/dates" + "github.com/nyaruka/gocommon/dates" "github.com/nyaruka/mailroom/services/tickets/zendesk" "github.com/stretchr/testify/assert" diff --git a/services/tickets/zendesk/web.go b/services/tickets/zendesk/web.go index 4986a28e9..bc7b60b71 100644 --- a/services/tickets/zendesk/web.go +++ b/services/tickets/zendesk/web.go @@ -76,7 +76,7 @@ func handleChannelback(ctx context.Context, s *web.Server, r *http.Request) (int return errors.Wrapf(err, "error updating ticket: %s", ticket.UUID()), http.StatusBadRequest, nil } - msg, err := tickets.SendReply(ctx, s.DB, s.RP, ticket, request.Message) + msg, err := tickets.SendReply(ctx, s.DB, s.RP, s.Storage, s.Config.S3MediaPrefix, ticket, request.Message, request.FileURLs) if err != nil { return err, http.StatusBadRequest, nil } diff --git a/tasks/campaigns/cron.go b/tasks/campaigns/cron.go index 71d153645..2e1ae5407 100644 --- a/tasks/campaigns/cron.go +++ b/tasks/campaigns/cron.go @@ -5,15 +5,16 @@ import ( "fmt" "time" - "github.com/gomodule/redigo/redis" - "github.com/jmoiron/sqlx" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/librato" "github.com/nyaruka/mailroom" - "github.com/nyaruka/mailroom/cron" - "github.com/nyaruka/mailroom/marker" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" + "github.com/nyaruka/mailroom/utils/cron" + "github.com/nyaruka/mailroom/utils/marker" + + "github.com/gomodule/redigo/redis" + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) diff --git a/tasks/campaigns/worker.go b/tasks/campaigns/worker.go index 23a774936..06d5eb029 100644 --- a/tasks/campaigns/worker.go +++ b/tasks/campaigns/worker.go @@ -10,10 +10,10 @@ import ( "github.com/jmoiron/sqlx" "github.com/nyaruka/goflow/flows/triggers" "github.com/nyaruka/mailroom" - "github.com/nyaruka/mailroom/marker" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" "github.com/nyaruka/mailroom/runner" + "github.com/nyaruka/mailroom/utils/marker" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) diff --git a/tasks/expirations/cron.go b/tasks/expirations/cron.go index ee72bb94a..8b79015db 100644 --- a/tasks/expirations/cron.go +++ b/tasks/expirations/cron.go @@ -5,14 +5,15 @@ import ( "fmt" "time" - "github.com/gomodule/redigo/redis" - "github.com/jmoiron/sqlx" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/mailroom" - "github.com/nyaruka/mailroom/cron" - "github.com/nyaruka/mailroom/marker" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/tasks/handler" + "github.com/nyaruka/mailroom/utils/cron" + "github.com/nyaruka/mailroom/utils/marker" + + "github.com/gomodule/redigo/redis" + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) diff --git a/tasks/expirations/cron_test.go b/tasks/expirations/cron_test.go index 91567576c..0303c363c 100644 --- a/tasks/expirations/cron_test.go +++ b/tasks/expirations/cron_test.go @@ -8,11 +8,12 @@ import ( "github.com/nyaruka/goflow/utils/uuids" _ "github.com/nyaruka/mailroom/hooks" - "github.com/nyaruka/mailroom/marker" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" "github.com/nyaruka/mailroom/tasks/handler" "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/utils/marker" + "github.com/stretchr/testify/assert" ) diff --git a/tasks/groups/worker.go b/tasks/groups/worker.go index 3f94b0332..d18dbaa46 100644 --- a/tasks/groups/worker.go +++ b/tasks/groups/worker.go @@ -7,9 +7,9 @@ import ( "time" "github.com/nyaruka/mailroom" - "github.com/nyaruka/mailroom/locker" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" + "github.com/nyaruka/mailroom/utils/locker" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) diff --git a/tasks/handler/cron.go b/tasks/handler/cron.go index 6e2687f1e..09db3953a 100644 --- a/tasks/handler/cron.go +++ b/tasks/handler/cron.go @@ -6,14 +6,15 @@ import ( "fmt" "time" - "github.com/gomodule/redigo/redis" - "github.com/jmoiron/sqlx" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/config" - "github.com/nyaruka/mailroom/cron" - "github.com/nyaruka/mailroom/marker" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" + "github.com/nyaruka/mailroom/utils/cron" + "github.com/nyaruka/mailroom/utils/marker" + + "github.com/gomodule/redigo/redis" + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) diff --git a/tasks/handler/handler_test.go b/tasks/handler/handler_test.go index f7fe77620..fcd529ad1 100644 --- a/tasks/handler/handler_test.go +++ b/tasks/handler/handler_test.go @@ -252,25 +252,28 @@ func TestChannelEvents(t *testing.T) { models.CathyID) tcs := []struct { - EventType models.ChannelEventType - ContactID models.ContactID - URNID models.URNID - OrgID models.OrgID - ChannelID models.ChannelID - Extra map[string]interface{} - Response string + EventType models.ChannelEventType + ContactID models.ContactID + URNID models.URNID + OrgID models.OrgID + ChannelID models.ChannelID + Extra map[string]interface{} + Response string + UpdateLastSeen bool }{ - {NewConversationEventType, models.CathyID, models.CathyURNID, models.Org1, models.TwitterChannelID, nil, "What is your favorite color?"}, - {NewConversationEventType, models.CathyID, models.CathyURNID, models.Org1, models.NexmoChannelID, nil, ""}, - {WelcomeMessageEventType, models.CathyID, models.CathyURNID, models.Org1, models.NexmoChannelID, nil, ""}, - {ReferralEventType, models.CathyID, models.CathyURNID, models.Org1, models.TwitterChannelID, nil, ""}, - {ReferralEventType, models.CathyID, models.CathyURNID, models.Org1, models.NexmoChannelID, nil, "Pick a number between 1-10."}, + {NewConversationEventType, models.CathyID, models.CathyURNID, models.Org1, models.TwitterChannelID, nil, "What is your favorite color?", true}, + {NewConversationEventType, models.CathyID, models.CathyURNID, models.Org1, models.NexmoChannelID, nil, "", true}, + {WelcomeMessageEventType, models.CathyID, models.CathyURNID, models.Org1, models.NexmoChannelID, nil, "", false}, + {ReferralEventType, models.CathyID, models.CathyURNID, models.Org1, models.TwitterChannelID, nil, "", true}, + {ReferralEventType, models.CathyID, models.CathyURNID, models.Org1, models.NexmoChannelID, nil, "Pick a number between 1-10.", true}, } models.FlushCache() - last := time.Now() for i, tc := range tcs { + start := time.Now() + time.Sleep(time.Millisecond * 5) + event := models.NewChannelEvent(tc.EventType, tc.OrgID, tc.ChannelID, tc.ContactID, tc.URNID, tc.Extra, false) eventJSON, err := json.Marshal(event) assert.NoError(t, err) @@ -291,13 +294,19 @@ func TestChannelEvents(t *testing.T) { assert.NoError(t, err, "%d: error when handling event", i) // if we are meant to have a response - var text string - err = db.Get(&text, `SELECT text FROM msgs_msg WHERE contact_id = $1 AND contact_urn_id = $2 AND created_on > $3 ORDER BY id DESC LIMIT 1`, tc.ContactID, tc.URNID, last) - if err != nil { - logrus.WithError(err).Error("error making query") + if tc.Response != "" { + var text string + err = db.Get(&text, `SELECT text FROM msgs_msg WHERE contact_id = $1 AND contact_urn_id = $2 AND created_on > $3 ORDER BY id DESC LIMIT 1`, tc.ContactID, tc.URNID, start) + assert.NoError(t, err) + assert.Equal(t, tc.Response, text, "%d: response: '%s' is not '%s'", i, text, tc.Response) + } + + if tc.UpdateLastSeen { + var lastSeen time.Time + err = db.Get(&lastSeen, `SELECT last_seen_on FROM contacts_contact WHERE id = $1`, tc.ContactID) + assert.NoError(t, err) + assert.True(t, lastSeen.Equal(start) || lastSeen.After(start), "%d: expected last seen to be updated", i) } - assert.Equal(t, tc.Response, text, "%d: response: '%s' is not '%s'", i, text, tc.Response) - last = time.Now() } } @@ -338,7 +347,7 @@ func TestStopEvent(t *testing.T) { testsuite.AssertQueryCount(t, db, `SELECT count(*) from contacts_contactgroup_contacts WHERE contactgroup_id = $1 AND contact_id = $2`, []interface{}{models.DoctorsGroupID, models.GeorgeID}, 1) // that cathy is stopped - testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND is_stopped = TRUE`, []interface{}{models.CathyID}, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contact WHERE id = $1 AND status = 'S'`, []interface{}{models.CathyID}, 1) // and has no upcoming events testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM campaigns_eventfire WHERE contact_id = $1`, []interface{}{models.CathyID}, 0) diff --git a/tasks/handler/worker.go b/tasks/handler/worker.go index de811aa94..d9199010b 100644 --- a/tasks/handler/worker.go +++ b/tasks/handler/worker.go @@ -11,15 +11,16 @@ import ( "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/excellent/types" "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/goflow/flows/resumes" "github.com/nyaruka/goflow/flows/triggers" "github.com/nyaruka/goflow/utils" "github.com/nyaruka/librato" "github.com/nyaruka/mailroom" - "github.com/nyaruka/mailroom/locker" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" "github.com/nyaruka/mailroom/runner" + "github.com/nyaruka/mailroom/utils/locker" "github.com/nyaruka/null" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -217,8 +218,8 @@ func handleTimedEvent(ctx context.Context, db *sqlx.DB, rp *redis.Pool, eventTyp return errors.Wrapf(err, "error loading contact") } - // contact has been deleted or is blocked, ignore this event - if len(contacts) == 0 || contacts[0].IsBlocked() { + // contact has been deleted or is blocked/stopped/archived, ignore this event + if len(contacts) == 0 || contacts[0].Status() != models.ContactStatusActive { return nil } @@ -314,14 +315,22 @@ func HandleChannelEvent(ctx context.Context, db *sqlx.DB, rp *redis.Pool, eventT } // contact has been deleted or is blocked, ignore this event - if len(contacts) == 0 || contacts[0].IsBlocked() { + if len(contacts) == 0 || contacts[0].Status() == models.ContactStatusBlocked { return nil, nil } modelContact := contacts[0] + if models.ContactSeenEvents[eventType] { + err = modelContact.UpdateLastSeenOn(ctx, db, event.OccurredOn()) + if err != nil { + return nil, errors.Wrap(err, "error updating contact last_seen_on") + } + } + // do we have associated trigger? var trigger *models.Trigger + switch eventType { case models.NewConversationEventType: @@ -336,7 +345,7 @@ func HandleChannelEvent(ctx context.Context, db *sqlx.DB, rp *redis.Pool, eventT case models.MOCallEventType: trigger = models.FindMatchingMOCallTrigger(oa, modelContact) - case models.WelcomeMessateEventType: + case models.WelcomeMessageEventType: trigger = nil default: @@ -449,11 +458,19 @@ func handleStopEvent(ctx context.Context, db *sqlx.DB, rp *redis.Pool, event *St if err != nil { return errors.Wrapf(err, "unable to start transaction for stopping contact") } + err = models.StopContact(ctx, tx, event.OrgID, event.ContactID) if err != nil { tx.Rollback() return err } + + err = models.UpdateContactLastSeenOn(ctx, tx, event.ContactID, event.OccurredOn) + if err != nil { + tx.Rollback() + return err + } + err = tx.Commit() if err != nil { return errors.Wrapf(err, "unable to commit for contact stop") @@ -509,7 +526,7 @@ func handleMsgEvent(ctx context.Context, db *sqlx.DB, rp *redis.Pool, event *Msg } // if this channel is no longer active or this contact is blocked, ignore this message (mark it as handled) - if channel == nil || modelContact.IsBlocked() { + if channel == nil || modelContact.Status() == models.ContactStatusBlocked { err := models.UpdateMessage(ctx, db, event.MsgID, models.MsgStatusHandled, models.VisibilityArchived, models.TypeInbox, topupID) if err != nil { return errors.Wrapf(err, "error marking blocked or nil channel message as handled") @@ -519,7 +536,7 @@ func handleMsgEvent(ctx context.Context, db *sqlx.DB, rp *redis.Pool, event *Msg // stopped contact? they are unstopped if they send us an incoming message newContact := event.NewContact - if modelContact.IsStopped() { + if modelContact.Status() == models.ContactStatusStopped { err := modelContact.Unstop(ctx, db) if err != nil { return errors.Wrapf(err, "error unstopping contact") @@ -631,10 +648,29 @@ func handleMsgEvent(ctx context.Context, db *sqlx.DB, rp *redis.Pool, event *Msg return nil } - err = models.UpdateMessage(ctx, db, event.MsgID, models.MsgStatusHandled, models.VisibilityVisible, models.TypeInbox, topupID) + // this message didn't trigger and new sessions or resume any existing ones, so handle as inbox + err = handleAsInbox(ctx, db, rp, oa, contact, msgIn, topupID) + if err != nil { + return errors.Wrapf(err, "error handling inbox message") + } + return nil +} + +func handleAsInbox(ctx context.Context, db *sqlx.DB, rp *redis.Pool, oa *models.OrgAssets, contact *flows.Contact, msg *flows.MsgIn, topupID models.TopupID) error { + msgEvent := events.NewMsgReceived(msg) + contact.SetLastSeenOn(msgEvent.CreatedOn()) + contactEvents := map[*flows.Contact][]flows.Event{contact: {msgEvent}} + + err := models.HandleAndCommitEvents(ctx, db, rp, oa, contactEvents) + if err != nil { + return errors.Wrap(err, "error handling inbox message events") + } + + err = models.UpdateMessage(ctx, db, msg.ID(), models.MsgStatusHandled, models.VisibilityVisible, models.TypeInbox, topupID) if err != nil { return errors.Wrapf(err, "error marking message as handled") } + return nil } @@ -662,11 +698,13 @@ type MsgEvent struct { Text string `json:"text"` Attachments []utils.Attachment `json:"attachments"` NewContact bool `json:"new_contact"` + CreatedOn time.Time `json:"created_on"` } type StopEvent struct { - ContactID models.ContactID `json:"contact_id"` - OrgID models.OrgID `json:"org_id"` + ContactID models.ContactID `json:"contact_id"` + OrgID models.OrgID `json:"org_id"` + OccurredOn time.Time `json:"occurred_on"` } // NewTimeoutEvent creates a new event task for the passed in timeout event diff --git a/tasks/ivr/cron.go b/tasks/ivr/cron.go index 71d2e4089..052c4be57 100644 --- a/tasks/ivr/cron.go +++ b/tasks/ivr/cron.go @@ -5,14 +5,14 @@ import ( "time" "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/ivr" "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/utils/cron" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" - "github.com/nyaruka/mailroom" - "github.com/nyaruka/mailroom/cron" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) diff --git a/tasks/ivr/worker_test.go b/tasks/ivr/worker_test.go index 55120b8df..1080043c1 100644 --- a/tasks/ivr/worker_test.go +++ b/tasks/ivr/worker_test.go @@ -8,6 +8,7 @@ import ( "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/config" @@ -70,7 +71,7 @@ func TestIVR(t *testing.T) { var client = &MockClient{} -func newMockClient(channel *models.Channel) (ivr.Client, error) { +func newMockClient(httpClient *http.Client, channel *models.Channel) (ivr.Client, error) { return client, nil } @@ -79,12 +80,12 @@ type MockClient struct { callError error } -func (c *MockClient) RequestCall(client *http.Client, number urns.URN, handleURL string, statusURL string) (ivr.CallID, error) { - return c.callID, c.callError +func (c *MockClient) RequestCall(number urns.URN, handleURL string, statusURL string) (ivr.CallID, *httpx.Trace, error) { + return c.callID, nil, c.callError } -func (c *MockClient) HangupCall(client *http.Client, externalID string) error { - return nil +func (c *MockClient) HangupCall(externalID string) (*httpx.Trace, error) { + return nil, nil } func (c *MockClient) WriteSessionResponse(session *models.Session, number urns.URN, resumeURL string, req *http.Request, w http.ResponseWriter) error { diff --git a/tasks/schedules/cron.go b/tasks/schedules/cron.go index 1af783007..f10108629 100644 --- a/tasks/schedules/cron.go +++ b/tasks/schedules/cron.go @@ -7,9 +7,9 @@ import ( "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" "github.com/nyaruka/mailroom" - "github.com/nyaruka/mailroom/cron" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" + "github.com/nyaruka/mailroom/utils/cron" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) diff --git a/tasks/stats/cron.go b/tasks/stats/cron.go index 572134dea..f352a70cf 100644 --- a/tasks/stats/cron.go +++ b/tasks/stats/cron.go @@ -4,13 +4,14 @@ import ( "context" "time" - "github.com/gomodule/redigo/redis" - "github.com/jmoiron/sqlx" "github.com/nyaruka/librato" "github.com/nyaruka/mailroom" - "github.com/nyaruka/mailroom/cron" "github.com/nyaruka/mailroom/queue" + "github.com/nyaruka/mailroom/utils/cron" "github.com/sirupsen/logrus" + + "github.com/gomodule/redigo/redis" + "github.com/jmoiron/sqlx" ) const ( diff --git a/tasks/timeouts/cron.go b/tasks/timeouts/cron.go index dffc94f5b..8e9a56c63 100644 --- a/tasks/timeouts/cron.go +++ b/tasks/timeouts/cron.go @@ -5,13 +5,14 @@ import ( "fmt" "time" - "github.com/gomodule/redigo/redis" - "github.com/jmoiron/sqlx" "github.com/nyaruka/mailroom" - "github.com/nyaruka/mailroom/cron" - "github.com/nyaruka/mailroom/tasks/handler" - "github.com/nyaruka/mailroom/marker" "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/tasks/handler" + "github.com/nyaruka/mailroom/utils/cron" + "github.com/nyaruka/mailroom/utils/marker" + + "github.com/gomodule/redigo/redis" + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) diff --git a/tasks/timeouts/cron_test.go b/tasks/timeouts/cron_test.go index 585be1e6b..47480286c 100644 --- a/tasks/timeouts/cron_test.go +++ b/tasks/timeouts/cron_test.go @@ -7,11 +7,12 @@ import ( "github.com/nyaruka/goflow/utils/uuids" _ "github.com/nyaruka/mailroom/hooks" - "github.com/nyaruka/mailroom/marker" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" "github.com/nyaruka/mailroom/tasks/handler" "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/utils/marker" + "github.com/stretchr/testify/assert" ) diff --git a/testsuite/testsuite.go b/testsuite/testsuite.go index 63f54671d..9ede02a35 100644 --- a/testsuite/testsuite.go +++ b/testsuite/testsuite.go @@ -9,12 +9,16 @@ import ( "strings" "testing" + "github.com/nyaruka/mailroom/utils/storage" + "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" ) +const storageDir = "_test_storage" + // Reset clears out both our database and redis DB func Reset() (context.Context, *sqlx.DB, *redis.Pool) { logrus.SetLevel(logrus.DebugLevel) @@ -97,6 +101,18 @@ func CTX() context.Context { return context.Background() } +// Storage returns our storage for tests +func Storage() storage.Storage { + return storage.NewFS(storageDir) +} + +// ResetStorage clears our storage for tests +func ResetStorage() { + if err := os.RemoveAll(storageDir); err != nil { + panic(err) + } +} + // utility function for running a command panicking if there is any error func mustExec(command string, args ...string) { cmd := exec.Command(command, args...) diff --git a/celery/celery.go b/utils/celery/celery.go similarity index 100% rename from celery/celery.go rename to utils/celery/celery.go diff --git a/celery/celery_test.go b/utils/celery/celery_test.go similarity index 83% rename from celery/celery_test.go rename to utils/celery/celery_test.go index 66273cf92..821609bcd 100644 --- a/celery/celery_test.go +++ b/utils/celery/celery_test.go @@ -1,11 +1,13 @@ -package celery +package celery_test import ( "encoding/json" "testing" - "github.com/gomodule/redigo/redis" "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/utils/celery" + + "github.com/gomodule/redigo/redis" ) func TestQueue(t *testing.T) { @@ -15,7 +17,7 @@ func TestQueue(t *testing.T) { // queue to our handler queue rc.Send("multi") - err := QueueTask(rc, "handler", "handle_event_task", []int64{}) + err := celery.QueueTask(rc, "handler", "handle_event_task", []int64{}) if err != nil { t.Error(err) } @@ -31,7 +33,7 @@ func TestQueue(t *testing.T) { } // make sure our task is valid json - task := Task{} + task := celery.Task{} err = json.Unmarshal([]byte(taskJSON), &task) if err != nil { t.Errorf("should be JSON: %s", err) diff --git a/cron/cron.go b/utils/cron/cron.go similarity index 98% rename from cron/cron.go rename to utils/cron/cron.go index 1bdf1134a..9fb81f002 100644 --- a/cron/cron.go +++ b/utils/cron/cron.go @@ -6,7 +6,7 @@ import ( "github.com/apex/log" "github.com/gomodule/redigo/redis" - "github.com/nyaruka/mailroom/locker" + "github.com/nyaruka/mailroom/utils/locker" "github.com/sirupsen/logrus" ) diff --git a/cron/cron_test.go b/utils/cron/cron_test.go similarity index 99% rename from cron/cron_test.go rename to utils/cron/cron_test.go index 6b998b045..ec7c4c0f5 100644 --- a/cron/cron_test.go +++ b/utils/cron/cron_test.go @@ -6,6 +6,7 @@ import ( "time" "github.com/nyaruka/mailroom/testsuite" + "github.com/stretchr/testify/assert" ) diff --git a/gsm7/gsm7.go b/utils/gsm7/gsm7.go similarity index 100% rename from gsm7/gsm7.go rename to utils/gsm7/gsm7.go diff --git a/gsm7/gsm7_test.go b/utils/gsm7/gsm7_test.go similarity index 84% rename from gsm7/gsm7_test.go rename to utils/gsm7/gsm7_test.go index 13546d3cf..7aa50ee48 100644 --- a/gsm7/gsm7_test.go +++ b/utils/gsm7/gsm7_test.go @@ -1,8 +1,10 @@ -package gsm7 +package gsm7_test import ( "testing" + "github.com/nyaruka/mailroom/utils/gsm7" + "github.com/stretchr/testify/assert" ) @@ -33,6 +35,6 @@ func TestSegments(t *testing.T) { } for _, tc := range tcs { - assert.Equal(t, tc.Segments, Segments(tc.Text), "unexpected num of segments for: %s", tc.Text) + assert.Equal(t, tc.Segments, gsm7.Segments(tc.Text), "unexpected num of segments for: %s", tc.Text) } } diff --git a/locker/locker.go b/utils/locker/locker.go similarity index 100% rename from locker/locker.go rename to utils/locker/locker.go diff --git a/locker/locker_test.go b/utils/locker/locker_test.go similarity index 59% rename from locker/locker_test.go rename to utils/locker/locker_test.go index 356bd8efa..78e09f3c4 100644 --- a/locker/locker_test.go +++ b/utils/locker/locker_test.go @@ -1,10 +1,12 @@ -package locker +package locker_test import ( "testing" "time" "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/utils/locker" + "github.com/stretchr/testify/assert" ) @@ -13,36 +15,36 @@ func TestLocker(t *testing.T) { rp := testsuite.RP() // acquire a lock, but have it expire in 5 seconds - v1, err := GrabLock(rp, "test", time.Second*5, time.Second) + v1, err := locker.GrabLock(rp, "test", time.Second*5, time.Second) assert.NoError(t, err) assert.NotZero(t, v1) // try to acquire the same lock, should fail - v2, err := GrabLock(rp, "test", time.Second*5, time.Second) + v2, err := locker.GrabLock(rp, "test", time.Second*5, time.Second) assert.NoError(t, err) assert.Zero(t, v2) // should succeed if we wait longer - v3, err := GrabLock(rp, "test", time.Second*5, time.Second*5) + v3, err := locker.GrabLock(rp, "test", time.Second*5, time.Second*5) assert.NoError(t, err) assert.NotZero(t, v3) assert.NotEqual(t, v1, v3) // extend the lock - err = ExtendLock(rp, "test", v3, time.Second*10) + err = locker.ExtendLock(rp, "test", v3, time.Second*10) assert.NoError(t, err) // trying to grab it should fail with a 5 second timeout - v4, err := GrabLock(rp, "test", time.Second*5, time.Second*5) + v4, err := locker.GrabLock(rp, "test", time.Second*5, time.Second*5) assert.NoError(t, err) assert.Zero(t, v4) // return the lock - err = ReleaseLock(rp, "test", v3) + err = locker.ReleaseLock(rp, "test", v3) assert.NoError(t, err) // new grab should work - v5, err := GrabLock(rp, "test", time.Second*5, time.Second*5) + v5, err := locker.GrabLock(rp, "test", time.Second*5, time.Second*5) assert.NoError(t, err) assert.NotZero(t, v5) } diff --git a/marker/marker.go b/utils/marker/marker.go similarity index 100% rename from marker/marker.go rename to utils/marker/marker.go diff --git a/marker/marker_test.go b/utils/marker/marker_test.go similarity index 76% rename from marker/marker_test.go rename to utils/marker/marker_test.go index 7cf85305c..f5bf1e18e 100644 --- a/marker/marker_test.go +++ b/utils/marker/marker_test.go @@ -1,9 +1,11 @@ -package marker +package marker_test import ( "testing" "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/utils/marker" + "github.com/stretchr/testify/assert" ) @@ -30,18 +32,18 @@ func TestMarker(t *testing.T) { for i, tc := range tcs { if tc.Action == "absent" { - present, err := HasTask(rc, tc.Group, tc.TaskID) + present, err := marker.HasTask(rc, tc.Group, tc.TaskID) assert.NoError(t, err) assert.False(t, present, "%d: %s:%s should be absent", i, tc.Group, tc.TaskID) } else if tc.Action == "present" { - present, err := HasTask(rc, tc.Group, tc.TaskID) + present, err := marker.HasTask(rc, tc.Group, tc.TaskID) assert.NoError(t, err) assert.True(t, present, "%d: %s:%s should be present", i, tc.Group, tc.TaskID) } else if tc.Action == "add" { - err := AddTask(rc, tc.Group, tc.TaskID) + err := marker.AddTask(rc, tc.Group, tc.TaskID) assert.NoError(t, err) } else if tc.Action == "remove" { - err := RemoveTask(rc, tc.Group, tc.TaskID) + err := marker.RemoveTask(rc, tc.Group, tc.TaskID) assert.NoError(t, err) } } diff --git a/utils/storage/base.go b/utils/storage/base.go new file mode 100644 index 000000000..c074046ec --- /dev/null +++ b/utils/storage/base.go @@ -0,0 +1,20 @@ +package storage + +import ( + "github.com/nyaruka/mailroom/config" +) + +// Storage is the interface that provides storage of atttachments etc +type Storage interface { + Name() string + Test() error + Put(path string, contentType string, contents []byte) (string, error) +} + +// New creates a new storage service +func New(cfg *config.Config) (Storage, error) { + if cfg.AWSAccessKeyID != "" && cfg.AWSSecretAccessKey != "" { + return NewS3(cfg) + } + return NewFS("_storage"), nil +} diff --git a/utils/storage/fs.go b/utils/storage/fs.go new file mode 100644 index 000000000..d6f492bda --- /dev/null +++ b/utils/storage/fs.go @@ -0,0 +1,47 @@ +package storage + +import ( + "io/ioutil" + "os" + "path/filepath" +) + +type fsStorage struct { + directory string + perms os.FileMode +} + +// NewFS creates a new file system storage service suitable for use in tests +func NewFS(directory string) Storage { + return &fsStorage{directory: directory, perms: 0766} +} + +func (s *fsStorage) Name() string { + return "file system" +} + +func (s *fsStorage) Test() error { + path, err := s.Put("test.txt", "text/plain", []byte(`test`)) + if err != nil { + return err + } + + os.Remove(path) + return nil +} + +func (s *fsStorage) Put(path string, contentType string, contents []byte) (string, error) { + fullPath := filepath.Join(s.directory, path) + + err := os.MkdirAll(filepath.Dir(fullPath), s.perms) + if err != nil { + return "", err + } + + err = ioutil.WriteFile(fullPath, contents, s.perms) + if err != nil { + return "", err + } + + return fullPath, nil +} diff --git a/utils/storage/fs_test.go b/utils/storage/fs_test.go new file mode 100644 index 000000000..af761f681 --- /dev/null +++ b/utils/storage/fs_test.go @@ -0,0 +1,34 @@ +package storage_test + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/nyaruka/mailroom/utils/storage" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFS(t *testing.T) { + s := storage.NewFS("_testing") + assert.NoError(t, s.Test()) + + // break our ability to write to that directory + require.NoError(t, os.Chmod("_testing", 0555)) + + assert.EqualError(t, s.Test(), "open _testing/test.txt: permission denied") + + require.NoError(t, os.Chmod("_testing", 0777)) + + url, err := s.Put("/foo/bar.txt", "text/plain", []byte(`hello world`)) + assert.NoError(t, err) + assert.Equal(t, "_testing/foo/bar.txt", url) + + data, err := ioutil.ReadFile(url) + assert.NoError(t, err) + assert.Equal(t, []byte(`hello world`), data) + + require.NoError(t, os.RemoveAll("_testing")) +} diff --git a/utils/storage/s3.go b/utils/storage/s3.go new file mode 100644 index 000000000..9930aeeb7 --- /dev/null +++ b/utils/storage/s3.go @@ -0,0 +1,73 @@ +package storage + +import ( + "bytes" + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3iface" + "github.com/nyaruka/mailroom/config" + "github.com/sirupsen/logrus" +) + +var s3BucketURL = "https://%s.s3.amazonaws.com%s" + +type s3Storage struct { + client s3iface.S3API + bucket string +} + +// NewS3 creates a new S3 storage service +func NewS3(cfg *config.Config) (Storage, error) { + s3Session, err := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials(cfg.AWSAccessKeyID, cfg.AWSSecretAccessKey, ""), + Endpoint: aws.String(cfg.S3Endpoint), + Region: aws.String(cfg.S3Region), + DisableSSL: aws.Bool(cfg.S3DisableSSL), + S3ForcePathStyle: aws.Bool(cfg.S3ForcePathStyle), + }) + if err != nil { + return nil, err + } + + return &s3Storage{ + client: s3.New(s3Session), + bucket: cfg.S3MediaBucket, + }, nil +} + +func (s *s3Storage) Name() string { + return "S3" +} + +// Test tests whether our S3 client is properly configured +func (s *s3Storage) Test() error { + params := &s3.HeadBucketInput{ + Bucket: aws.String(s.bucket), + } + _, err := s.client.HeadBucket(params) + return err +} + +// Put writes the passed in file to the bucket with the passed in content type +func (s *s3Storage) Put(path string, contentType string, contents []byte) (string, error) { + params := &s3.PutObjectInput{ + Bucket: aws.String(s.bucket), + Body: bytes.NewReader(contents), + Key: aws.String(path), + ContentType: aws.String(contentType), + ACL: aws.String(s3.BucketCannedACLPublicRead), + } + + logrus.WithField("path", path).Info("** uploading s3 file") + + _, err := s.client.PutObject(params) + if err != nil { + return "", err + } + + return fmt.Sprintf(s3BucketURL, s.bucket, path), nil +} diff --git a/web/contact/contact.go b/web/contact/contact.go index ac707ba0a..5b3ee550b 100644 --- a/web/contact/contact.go +++ b/web/contact/contact.go @@ -79,7 +79,7 @@ func handleSearch(ctx context.Context, s *web.Server, r *http.Request) (interfac } // grab our org assets - oa, err := models.GetOrgAssetsWithRefresh(s.CTX, s.DB, request.OrgID, models.RefreshFields) + oa, err := models.GetOrgAssetsWithRefresh(s.CTX, s.DB, request.OrgID, models.RefreshFields|models.RefreshGroups) if err != nil { return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load org assets") } @@ -171,13 +171,13 @@ func handleParseQuery(ctx context.Context, s *web.Server, r *http.Request) (inte } // grab our org assets - oa, err := models.GetOrgAssetsWithRefresh(s.CTX, s.DB, request.OrgID, models.RefreshFields) + oa, err := models.GetOrgAssetsWithRefresh(s.CTX, s.DB, request.OrgID, models.RefreshFields|models.RefreshGroups) if err != nil { return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load org assets") } env := oa.Env() - parsed, err := contactql.ParseQuery(request.Query, env.RedactionPolicy(), env.DefaultCountry(), oa.SessionAssets()) + parsed, err := contactql.ParseQuery(env, request.Query, oa.SessionAssets()) if err != nil { isQueryError, qerr := contactql.IsQueryError(err) @@ -203,10 +203,7 @@ func handleParseQuery(ctx context.Context, s *web.Server, r *http.Request) (inte allowAsGroup = metadata.AllowAsGroup } - eq, err := models.BuildElasticQuery(oa, request.GroupUUID, parsed) - if err != nil { - return nil, http.StatusInternalServerError, err - } + eq := models.BuildElasticQuery(oa, request.GroupUUID, parsed) eqj, err := eq.Source() if err != nil { return nil, http.StatusInternalServerError, err @@ -304,8 +301,9 @@ func handleModify(ctx context.Context, s *web.Server, r *http.Request) (interfac // create an environment instance with location support env := flows.NewEnvironment(oa.Env(), oa.SessionAssets().Locations()) - // create scenes for our contacts - scenes := make([]*models.Scene, 0, len(contacts)) + // gather up events for our contacts + contactEvents := make(map[*flows.Contact][]flows.Event, len(contacts)) + for _, contact := range contacts { flowContact, err := contact.FlowContact(oa) if err != nil { @@ -317,57 +315,18 @@ func handleModify(ctx context.Context, s *web.Server, r *http.Request) (interfac Events: make([]flows.Event, 0, len(mods)), } - scene := models.NewSceneForContact(flowContact) - // apply our modifiers for _, mod := range mods { mod.Apply(env, oa.SessionAssets(), flowContact, func(e flows.Event) { result.Events = append(result.Events, e) }) } results[contact.ID()] = result - scenes = append(scenes, scene) + contactEvents[flowContact] = result.Events } - // ok, commit all our events - tx, err := s.DB.BeginTxx(ctx, nil) + err = models.HandleAndCommitEvents(ctx, s.DB, s.RP, oa, contactEvents) if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "error starting transaction") - } - - // apply our events - for _, scene := range scenes { - err := models.HandleEvents(ctx, tx, s.RP, oa, scene, results[scene.ContactID()].Events) - if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "error applying events") - } - } - - // gather all our pre commit events, group them by hook and apply them - err = models.ApplyEventPreCommitHooks(ctx, tx, s.RP, oa, scenes) - if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "error applying pre commit hooks") - } - - // commit our transaction - err = tx.Commit() - if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "error committing pre commit hooks") - } - - tx, err = s.DB.BeginTxx(ctx, nil) - if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "error starting transaction for post commit") - } - - // then apply our post commit hooks - err = models.ApplyEventPostCommitHooks(ctx, tx, s.RP, oa, scenes) - if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "error applying pre commit hooks") - } - - err = tx.Commit() - if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "error committing pre commit hooks") + return nil, http.StatusInternalServerError, err } return results, http.StatusOK, nil diff --git a/web/contact/testdata/modify.json b/web/contact/testdata/modify.json index 31b23cad6..92aaa552f 100644 --- a/web/contact/testdata/modify.json +++ b/web/contact/testdata/modify.json @@ -160,6 +160,135 @@ } ] }, + { + "label": "set status to blocked", + "method": "POST", + "path": "/mr/contact/modify", + "body": { + "org_id": 1, + "user_id": 1, + "contact_ids": [ + 10000 + ], + "modifiers": [ + { + "type": "status", + "status": "blocked" + } + ] + }, + "status": 200, + "response": { + "10000": { + "contact": { + "uuid": "6393abc0-283d-4c9b-a1b3-641a035c34bf", + "id": 10000, + "status": "blocked", + "timezone": "America/Los_Angeles", + "created_on": "2018-07-06T12:30:00.123457Z" + }, + "events": [ + { + "type": "contact_status_changed", + "created_on": "2018-07-06T12:30:00.123456789Z", + "status": "blocked" + } + ] + } + }, + "db_assertions": [ + { + "query": "SELECT count(*) FROM contacts_contact WHERE id = 10000 AND status = 'B'", + "count": 1 + } + ] + }, + { + "label": "set status to archived", + "method": "POST", + "path": "/mr/contact/modify", + "body": { + "org_id": 1, + "user_id": 1, + "contact_ids": [ + 10000 + ], + "modifiers": [ + { + "type": "status", + "status": "archived" + } + ] + }, + "status": 200, + "response": { + "10000": { + "contact": { + "uuid": "6393abc0-283d-4c9b-a1b3-641a035c34bf", + "id": 10000, + "status": "archived", + "timezone": "America/Los_Angeles", + "created_on": "2018-07-06T12:30:00.123457Z" + }, + "events": [ + { + "type": "contact_status_changed", + "created_on": "2018-07-06T12:30:00.123456789Z", + "status": "archived" + } + ] + } + }, + "db_assertions": [ + { + "query": "SELECT count(*) FROM contacts_contact WHERE id = 10000 AND status = 'V'", + "count": 1 + } + ] + }, + { + "label": "set status to active", + "method": "POST", + "path": "/mr/contact/modify", + "body": { + "org_id": 1, + "user_id": 1, + "contact_ids": [ + 10000 + ], + "modifiers": [ + { + "type": "status", + "status": "active" + } + ] + }, + "status": 200, + "response": { + "10000": { + "contact": { + "uuid": "6393abc0-283d-4c9b-a1b3-641a035c34bf", + "id": 10000, + "status": "active", + "timezone": "America/Los_Angeles", + "created_on": "2018-07-06T12:30:00.123457Z" + }, + "events": [ + { + "type": "contact_status_changed", + "created_on": "2018-07-06T12:30:00.123456789Z", + "status": "active" + } + ] + } + }, + "db_assertions": [ + { + "query": "SELECT count(*) FROM contacts_contact WHERE id = 10000 AND status = 'A'", + "count": 1 + } + ] + }, { "label": "set text field with valid value", "method": "POST", diff --git a/web/contact/testdata/parse_query.json b/web/contact/testdata/parse_query.json index 48cd71ee7..19b5aed1d 100644 --- a/web/contact/testdata/parse_query.json +++ b/web/contact/testdata/parse_query.json @@ -10,7 +10,24 @@ } }, { - "label": "invalid body", + "label": "query that is syntactically invalid", + "method": "POST", + "path": "/mr/contact/parse_query", + "body": { + "org_id": 1, + "query": "$" + }, + "status": 400, + "response": { + "error": "mismatched input '$' expecting {'(', TEXT, STRING}", + "code": "unexpected_token", + "extra": { + "token": "$" + } + } + }, + { + "label": "query with invalid property", "method": "POST", "path": "/mr/contact/parse_query", "body": { @@ -19,7 +36,11 @@ }, "status": 400, "response": { - "error": "can't resolve 'birthday' to attribute, scheme or field" + "error": "can't resolve 'birthday' to attribute, scheme or field", + "code": "unknown_property", + "extra": { + "property": "birthday" + } } }, { diff --git a/web/errors.go b/web/errors.go new file mode 100644 index 000000000..a5bce23a6 --- /dev/null +++ b/web/errors.go @@ -0,0 +1,27 @@ +package web + +import ( + "github.com/nyaruka/goflow/utils" + + "github.com/pkg/errors" +) + +// ErrorResponse is the type for our error responses +type ErrorResponse struct { + Error string `json:"error"` + Code string `json:"code,omitempty"` + Extra map[string]string `json:"extra,omitempty"` +} + +// NewErrorResponse creates a new error response from the passed in error +func NewErrorResponse(err error) *ErrorResponse { + rich, isRich := errors.Cause(err).(utils.RichError) + if isRich { + return &ErrorResponse{ + Error: rich.Error(), + Code: rich.Code(), + Extra: rich.Extra(), + } + } + return &ErrorResponse{Error: err.Error()} +} diff --git a/web/errors_test.go b/web/errors_test.go new file mode 100644 index 000000000..56e6d706b --- /dev/null +++ b/web/errors_test.go @@ -0,0 +1,34 @@ +package web_test + +import ( + "testing" + + "github.com/nyaruka/gocommon/jsonx" + "github.com/nyaruka/goflow/contactql" + "github.com/nyaruka/goflow/envs" + "github.com/nyaruka/mailroom/web" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func TestErrorResponse(t *testing.T) { + // create a simple error + er1 := web.NewErrorResponse(errors.New("I'm an error!")) + assert.Equal(t, "I'm an error!", er1.Error) + + er1JSON, err := jsonx.Marshal(er1) + assert.NoError(t, err) + assert.JSONEq(t, `{"error": "I'm an error!"}`, string(er1JSON)) + + // create a rich error + _, err = contactql.ParseQuery(envs.NewBuilder().Build(), "$$", nil) + + er2 := web.NewErrorResponse(err) + assert.Equal(t, "mismatched input '$' expecting {'(', TEXT, STRING}", er2.Error) + assert.Equal(t, "unexpected_token", er2.Code) + + er2JSON, err := jsonx.Marshal(er2) + assert.NoError(t, err) + assert.JSONEq(t, `{"error": "mismatched input '$' expecting {'(', TEXT, STRING}", "code": "unexpected_token", "extra": {"token": "$"}}`, string(er2JSON)) +} diff --git a/web/ivr/ivr.go b/web/ivr/ivr.go index c67df53b1..fa0fe0612 100644 --- a/web/ivr/ivr.go +++ b/web/ivr/ivr.go @@ -316,10 +316,10 @@ func handleFlow(ctx context.Context, s *web.Server, r *http.Request, rawW http.R return client.WriteErrorResponse(w, errors.Wrapf(err, "no such contact")) } if len(contacts) == 0 { - return client.WriteErrorResponse(w, errors.Errorf("no contact width id: %d", conn.ContactID())) + return client.WriteErrorResponse(w, errors.Errorf("no contact with id: %d", conn.ContactID())) } - if contacts[0].IsStopped() || contacts[0].IsBlocked() { - return client.WriteErrorResponse(w, errors.Errorf("no contact width id: %d", conn.ContactID())) + if contacts[0].Status() != models.ContactStatusActive { + return client.WriteErrorResponse(w, errors.Errorf("no contact with id: %d", conn.ContactID())) } // load the URN for this connection @@ -352,7 +352,7 @@ func handleFlow(ctx context.Context, s *web.Server, r *http.Request, rawW http.R case actionResume: err = ivr.ResumeIVRFlow( - ctx, s.Config, s.DB, s.RP, s.S3Client, resumeURL, client, + ctx, s.Config, s.DB, s.RP, s.Storage, resumeURL, client, oa, channel, conn, contacts[0], urn, r, w, ) diff --git a/web/ivr/ivr_test.go b/web/ivr/ivr_test.go index b3316b938..cc4cc5c11 100644 --- a/web/ivr/ivr_test.go +++ b/web/ivr/ivr_test.go @@ -23,7 +23,6 @@ import ( "github.com/stretchr/testify/assert" _ "github.com/nyaruka/mailroom/hooks" - "github.com/nyaruka/mailroom/ivr" "github.com/nyaruka/mailroom/ivr/nexmo" "github.com/nyaruka/mailroom/ivr/twiml" ivr_tasks "github.com/nyaruka/mailroom/tasks/ivr" @@ -33,6 +32,7 @@ func TestTwilioIVR(t *testing.T) { ctx, db, rp := testsuite.Reset() rc := rp.Get() defer rc.Close() + defer testsuite.ResetStorage() // start test server ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -60,7 +60,7 @@ func TestTwilioIVR(t *testing.T) { twiml.IgnoreSignatures = true wg := &sync.WaitGroup{} - server := web.NewServer(ctx, config.Mailroom, db, rp, nil, nil, wg) + server := web.NewServer(ctx, config.Mailroom, db, rp, testsuite.Storage(), nil, wg) server.Start() defer server.Stop() @@ -327,6 +327,7 @@ func TestNexmoIVR(t *testing.T) { ctx, db, rp := testsuite.Reset() rc := rp.Get() defer rc.Close() + defer testsuite.ResetStorage() models.FlushCache() // deactivate our twilio channel @@ -343,7 +344,7 @@ func TestNexmoIVR(t *testing.T) { } else { type CallForm struct { To []struct { - Number int64 `json:"number` + Number int64 `json:"number"` } `json:"to"` } body, _ := ioutil.ReadAll(r.Body) @@ -364,11 +365,10 @@ func TestNexmoIVR(t *testing.T) { defer ts.Close() wg := &sync.WaitGroup{} - server := web.NewServer(ctx, config.Mailroom, db, rp, nil, nil, wg) + server := web.NewServer(ctx, config.Mailroom, db, rp, testsuite.Storage(), nil, wg) server.Start() defer server.Stop() - ivr.WriteAttachments = false nexmo.BaseURL = ts.URL nexmo.IgnoreSignatures = true diff --git a/web/server.go b/web/server.go index aba325c1c..2a3116f7a 100644 --- a/web/server.go +++ b/web/server.go @@ -8,10 +8,10 @@ import ( "sync" "time" - "github.com/nyaruka/goflow/utils/jsonx" + "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/mailroom/config" + "github.com/nyaruka/mailroom/utils/storage" - "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" "github.com/gomodule/redigo/redis" @@ -60,12 +60,12 @@ func RegisterRoute(method string, pattern string, handler Handler) { } // NewServer creates a new web server, it will need to be started after being created -func NewServer(ctx context.Context, config *config.Config, db *sqlx.DB, rp *redis.Pool, s3Client s3iface.S3API, elasticClient *elastic.Client, wg *sync.WaitGroup) *Server { +func NewServer(ctx context.Context, config *config.Config, db *sqlx.DB, rp *redis.Pool, store storage.Storage, elasticClient *elastic.Client, wg *sync.WaitGroup) *Server { s := &Server{ CTX: ctx, RP: rp, DB: db, - S3Client: s3Client, + Storage: store, ElasticClient: elasticClient, Config: config, @@ -212,7 +212,7 @@ type Server struct { CTX context.Context RP *redis.Pool DB *sqlx.DB - S3Client s3iface.S3API + Storage storage.Storage Config *config.Config ElasticClient *elastic.Client @@ -220,13 +220,3 @@ type Server struct { httpServer *http.Server } - -// ErrorResponse is the type for our error responses, it just contains a single error field -type ErrorResponse struct { - Error string `json:"error"` -} - -// NewErrorResponse creates a new error response from the passed in errro -func NewErrorResponse(err error) *ErrorResponse { - return &ErrorResponse{err.Error()} -} diff --git a/web/testing.go b/web/testing.go index 804cb2628..12e4b22bf 100644 --- a/web/testing.go +++ b/web/testing.go @@ -13,10 +13,10 @@ import ( "testing" "time" + "github.com/nyaruka/gocommon/dates" + "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/goflow/test" - "github.com/nyaruka/goflow/utils/dates" - "github.com/nyaruka/goflow/utils/httpx" - "github.com/nyaruka/goflow/utils/jsonx" "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/testsuite" diff --git a/web/wrappers_test.go b/web/wrappers_test.go index 53187d63b..3c426daf1 100644 --- a/web/wrappers_test.go +++ b/web/wrappers_test.go @@ -5,8 +5,8 @@ import ( "net/http" "testing" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/goflow/utils/httpx" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" "github.com/nyaruka/mailroom/web"