Update master-rp to v5.7.14

.github/workflows/ci.yml

@@ -57,11 +57,11 @@ jobs:
       uses: actions/checkout@v1
 
     - name: Fetch GoFlow docs
-      # for now just grab en_US/ docs and bundle as docs/
+      # for backward compatibility, English docs are copied to root of docs directory
       run: |
-        GOFLOW_VERSION=$(grep goflow go.mod | cut -d" " -f2)
-        curl https://codeload.github.com/nyaruka/goflow/tar.gz/$GOFLOW_VERSION | tar --wildcards --strip=2 -zx "*/docs/en_US/*"
-        mv en_US docs  
+        GOFLOW_VERSION=$(grep goflow go.mod | cut -d" " -f2 | cut -c2-)
+        curl https://codeload.github.com/nyaruka/goflow/tar.gz/v$GOFLOW_VERSION | tar --wildcards --strip=1 -zx "goflow-${GOFLOW_VERSION}/docs/*"
+        cp ./docs/en-us/*.* docs/
 
     - name: Install Go
       uses: actions/setup-go@v1

.gitignore

@@ -13,6 +13,7 @@ docs/*
 docs
 
 .DS_Store
+_storage/
 
 # Test binary, build with `go test -c`
 *.test

CHANGELOG.md

@@ -1,3 +1,67 @@
+v5.7.14
+----------
+ * Stop writing is_blocked and is_stopped
+
+v5.7.13
+----------
+ * Read from contact.status intead of is_stopped/is_blocked
+ * Implement saving of zendesk ticket files as attachments
+ * Abstract S3 code so tests and dev envs can use file storage
+
+v5.7.12
+----------
+ * Fix inserting channel logs and add test
+
+v5.7.11
+----------
+ * Always write contact.status when writing is_blocked or is_stopped
+ * Convert IVR code to use goflow's httpx package
+
+v5.7.10
+----------
+ * Tweak goreleaser config to include subdirectories inside docs folder
+
+v5.7.9
+----------
+ * Update to goflow v0.101.2
+ * Bundle localized goflow docs in release
+
+v5.7.8
+----------
+ * Recalculate event fires for campaign events based on last_seen_on
+
+v5.7.7
+----------
+ * Update to latest goflow v0.100.0
+
+v5.7.6
+----------
+ * Remove protection for overwriting last_seen_on with older values
+
+v5.7.5
+----------
+ * Update last_seen_on when handling certain channel events
+ * Update last_seen_on when we receive a message from a contact
+
+v5.7.4
+----------
+ * Fail outgoing messages for suspended orgs
+ * Refresh groups as well as fields for contact query parsing
+
+v5.7.3
+----------
+ * Update to goflow v0.99.0
+
+v5.7.2
+----------
+ * Update to latest goflow v0.98.0
+ * Render rich errors with code and extra field
+
+v5.7.1
+----------
+ * Update to latest goflow v0.96.0
+ * Add loop protection by passing session history to new flow action triggers
+
 v5.7.0
 ----------
  * Set user and origin on manual triggers

config/config.go

@@ -94,8 +94,8 @@ func NewMailroomConfig() *Config {
 		S3MediaPrefix:      "/media/",
 		S3DisableSSL:       false,
 		S3ForcePathStyle:   false,
-		AWSAccessKeyID:     "missing_aws_access_key_id",
-		AWSSecretAccessKey: "missing_aws_secret_access_key",
+		AWSAccessKeyID:     "",
+		AWSSecretAccessKey: "",
 
 		RetryPendingMessages: true,
 

courier/courier.go

@@ -52,8 +52,8 @@ func QueueMessages(rc redis.Conn, msgs []*models.Msg) error {
 	}
 
 	for _, msg := range msgs {
-		// no channel, continue
-		if msg.ChannelUUID() == "" {
+		// ignore any message without a channel or already marked as failed (maybe org is suspended)
+		if msg.ChannelUUID() == "" || msg.Status() == models.MsgStatusFailed {
 			continue
 		}
 

courier/courier_test.go

@@ -0,0 +1,168 @@
+package courier_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/nyaruka/gocommon/urns"
+	"github.com/nyaruka/goflow/assets"
+	"github.com/nyaruka/goflow/flows"
+	"github.com/nyaruka/mailroom/courier"
+	"github.com/nyaruka/mailroom/models"
+	"github.com/nyaruka/mailroom/testsuite"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+type msgSpec struct {
+	ChannelUUID assets.ChannelUUID
+	Text        string
+	ContactID   models.ContactID
+	URN         urns.URN
+	URNID       models.URNID
+	Failed      bool
+}
+
+func createMsg(t *testing.T, m msgSpec) *models.Msg {
+	ctx := testsuite.CTX()
+	db := testsuite.DB()
+	db.MustExec(`UPDATE orgs_org SET is_suspended = $1 WHERE id = $2`, m.Failed, models.Org1)
+
+	oa, err := models.GetOrgAssetsWithRefresh(ctx, db, models.Org1, models.RefreshOrg)
+	require.NoError(t, err)
+
+	channel := oa.ChannelByUUID(m.ChannelUUID)
+
+	flowMsg := flows.NewMsgOut(m.URN, channel.ChannelReference(), m.Text, nil, nil, nil, flows.NilMsgTopic)
+	msg, err := models.NewOutgoingMsg(oa.Org(), channel, m.ContactID, flowMsg, time.Now())
+	require.NoError(t, err)
+	return msg
+}
+
+func TestQueueMessages(t *testing.T) {
+	db := testsuite.DB()
+	rc := testsuite.RC()
+	testsuite.Reset()
+	models.FlushCache()
+
+	// convert the Twitter channel to be an Android channel
+	db.MustExec(`UPDATE channels_channel SET name = 'Android', channel_type = 'A' WHERE id = $1`, models.TwitterChannelID)
+	androidChannelUUID := models.TwitterChannelUUID
+
+	tests := []struct {
+		Description string
+		Msgs        []msgSpec
+		QueueSizes  map[string]int
+	}{
+		{
+			Description: "2 queueable messages",
+			Msgs: []msgSpec{
+				{
+					ChannelUUID: models.TwilioChannelUUID,
+					Text:        "normal msg 1",
+					ContactID:   models.CathyID,
+					URN:         urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)),
+					URNID:       models.CathyURNID,
+				},
+				{
+					ChannelUUID: models.TwilioChannelUUID,
+					Text:        "normal msg 2",
+					ContactID:   models.CathyID,
+					URN:         urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)),
+					URNID:       models.CathyURNID,
+				},
+			},
+			QueueSizes: map[string]int{
+				"msgs:74729f45-7f29-4868-9dc4-90e491e3c7d8|10/0": 2,
+			},
+		},
+		{
+			Description: "1 queueable message and 1 failed",
+			Msgs: []msgSpec{
+				{
+					ChannelUUID: models.TwilioChannelUUID,
+					Text:        "normal msg 1",
+					ContactID:   models.CathyID,
+					URN:         urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)),
+					URNID:       models.CathyURNID,
+					Failed:      true,
+				},
+				{
+					ChannelUUID: models.TwilioChannelUUID,
+					Text:        "normal msg 1",
+					ContactID:   models.CathyID,
+					URN:         urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)),
+					URNID:       models.CathyURNID,
+				},
+			},
+			QueueSizes: map[string]int{
+				"msgs:74729f45-7f29-4868-9dc4-90e491e3c7d8|10/0": 1,
+			},
+		},
+		{
+			Description: "1 queueable message and 1 Android",
+			Msgs: []msgSpec{
+				{
+					ChannelUUID: androidChannelUUID,
+					Text:        "normal msg 1",
+					ContactID:   models.CathyID,
+					URN:         urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)),
+					URNID:       models.CathyURNID,
+				},
+				{
+					ChannelUUID: models.TwilioChannelUUID,
+					Text:        "normal msg 1",
+					ContactID:   models.CathyID,
+					URN:         urns.URN(fmt.Sprintf("tel:+250700000001?id=%d", models.CathyURNID)),
+					URNID:       models.CathyURNID,
+				},
+			},
+			QueueSizes: map[string]int{
+				"msgs:74729f45-7f29-4868-9dc4-90e491e3c7d8|10/0": 1,
+			},
+		},
+		{
+			Description: "0 messages",
+			Msgs:        []msgSpec{},
+			QueueSizes: map[string]int{
+				"msgs:74729f45-7f29-4868-9dc4-90e491e3c7d8|10/0": 0,
+			},
+		},
+	}
+
+	for _, tc := range tests {
+		msgs := make([]*models.Msg, len(tc.Msgs))
+		for i := range tc.Msgs {
+			msgs[i] = createMsg(t, tc.Msgs[i])
+		}
+
+		rc.Do("FLUSHDB")
+		courier.QueueMessages(rc, msgs)
+
+		for queueKey, size := range tc.QueueSizes {
+			if size == 0 {
+				result, err := rc.Do("ZCARD", queueKey)
+				require.NoError(t, err)
+				assert.Equal(t, size, int(result.(int64)))
+			} else {
+				result, err := rc.Do("ZPOPMAX", queueKey)
+				require.NoError(t, err)
+
+				results := result.([]interface{})
+				assert.Equal(t, 2, len(results)) // result is (item, score)
+
+				batchJSON := results[0].([]byte)
+				var batch []map[string]interface{}
+				err = json.Unmarshal(batchJSON, &batch)
+				require.NoError(t, err)
+
+				assert.Equal(t, size, len(batch))
+			}
+		}
+	}
+
+	testsuite.Reset()
+}

docker/Dockerfile

@@ -11,8 +11,8 @@ RUN addgroup -S golang \
 
 COPY . .
 
-RUN curl https://codeload.github.com/nyaruka/goflow/tar.gz/$(grep goflow go.mod | cut -d" " -f2) | tar --wildcards --strip=1 -zx "*/docs/*"
-RUN cp docs/en_US/completion.json docs/completion.json && cp docs/en_US/functions.json docs/functions.json
+RUN export GOFLOW_VERSION=$(grep goflow go.mod | cut -d" " -f2 | cut -c2-) && curl https://codeload.github.com/nyaruka/goflow/tar.gz/v$GOFLOW_VERSION | tar --wildcards --strip=1 -zx "goflow-${GOFLOW_VERSION}/docs/*"
+RUN cp ./docs/en-us/*.* docs/
 
 RUN go install -v ./cmd/...
 

go.mod

@@ -18,8 +18,8 @@ require (
 	github.com/lib/pq v1.4.0
 	github.com/mattn/go-sqlite3 v1.10.0 // indirect
 	github.com/nyaruka/ezconf v0.2.1
-	github.com/nyaruka/gocommon v1.2.0
-	github.com/nyaruka/goflow v0.95.1
+	github.com/nyaruka/gocommon v1.3.0
+	github.com/nyaruka/goflow v0.102.1
 	github.com/nyaruka/librato v1.0.0
 	github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d
 	github.com/nyaruka/null v1.2.0
@@ -28,7 +28,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.9.1
-	github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24
+	github.com/shopspring/decimal v1.2.0
 	github.com/sirupsen/logrus v1.5.0
 	github.com/stretchr/testify v1.5.1
 	gopkg.in/go-playground/validator.v9 v9.31.0

go.sum

@@ -128,10 +128,10 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
 github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
 github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0=
 github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw=
-github.com/nyaruka/gocommon v1.2.0 h1:gCmVCXYZFwKDMqQj8R1jNlK+7a06khKFq3zX8fBBbzw=
-github.com/nyaruka/gocommon v1.2.0/go.mod h1:9Y21Fd6iZXDLHWTRiZAc6b4LQSCi6HEEQK4SB45Yav4=
-github.com/nyaruka/goflow v0.95.1 h1:FmYlZH4+mM+5Xft1BxUG97TIWx4eDqZk0Dbnh86Qqs4=
-github.com/nyaruka/goflow v0.95.1/go.mod h1:ttUI9hd9ufGU/3EqIst6Hdc68imMG1utlqcELbYrtOQ=
+github.com/nyaruka/gocommon v1.3.0 h1:IqaPT4KQ2oVq/2Ivp/c+RVCs8v71+RzPU2VhMoRrgpU=
+github.com/nyaruka/gocommon v1.3.0/go.mod h1:w7lKxIkm/qLAoO9Y3aI1LV7EiYogn6+1C8MTEjxTC9M=
+github.com/nyaruka/goflow v0.102.1 h1:7QX2jTwV7uIbaGnkkpmB+ao+E7Cmyar9g7sRQH4Bu3M=
+github.com/nyaruka/goflow v0.102.1/go.mod h1:wuvXZTs6a6S1rjSRLaQGVxDfKomDJ/1XQoLXCqFekK4=
 github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0=
 github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg=
 github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d h1:hyp9u36KIwbTCo2JAJ+TuJcJBc+UZzEig7RI/S5Dvkc=
@@ -170,9 +170,8 @@ github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
-github.com/shopspring/decimal v0.0.0-20180319170823-2df3e6ddaf6e/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
-github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE=
-github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4=
+github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ=
+github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=

goflow/engine_test.go

@@ -4,9 +4,9 @@ import (
 	"net/http"
 	"testing"
 
+	"github.com/nyaruka/gocommon/httpx"
 	"github.com/nyaruka/gocommon/urns"
 	"github.com/nyaruka/goflow/flows"
-	"github.com/nyaruka/goflow/utils/httpx"
 	"github.com/nyaruka/mailroom/goflow"
 	"github.com/nyaruka/mailroom/models"
 	"github.com/nyaruka/mailroom/testsuite"

goflow/http.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/nyaruka/goflow/utils/httpx"
+	"github.com/nyaruka/gocommon/httpx"
 	"github.com/nyaruka/mailroom/config"
 )
 

goflow/modifiers.go

@@ -5,7 +5,7 @@ import (
 
 	"github.com/nyaruka/goflow/assets"
 	"github.com/nyaruka/goflow/flows"
-	"github.com/nyaruka/goflow/flows/actions/modifiers"
+	"github.com/nyaruka/goflow/flows/modifiers"
 
 	"github.com/pkg/errors"
 )

goreleaser.yml

@@ -11,4 +11,4 @@ archives:
   - files:
     - LICENSE
     - README.md
-    - docs/*
+    - docs/**/*

hooks/airtime_transferred_test.go

@@ -4,9 +4,9 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/nyaruka/gocommon/httpx"
 	"github.com/nyaruka/goflow/flows"
 	"github.com/nyaruka/goflow/flows/actions"
-	"github.com/nyaruka/goflow/utils/httpx"
 	"github.com/nyaruka/mailroom/models"
 	"github.com/nyaruka/mailroom/testsuite"
 

hooks/campaigns.go

@@ -52,6 +52,10 @@ func (h *UpdateCampaignEventsHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *r
 					continue
 				}
 				fieldChanges[field.ID()] = true
+
+			case *events.MsgReceivedEvent:
+				field := oa.FieldByKey(models.LastSeenOnKey)
+				fieldChanges[field.ID()] = true
 			}
 		}