Merge branch 'main' into slog

CHANGELOG.md

@@ -1,3 +1,39 @@
+v8.3.19 (2023-09-14)
+-------------------------
+ * Fix stop contact task name
+
+v8.3.18 (2023-09-14)
+-------------------------
+ * Add support for optin/optout triggers and channel events
+
+v8.3.17 (2023-09-12)
+-------------------------
+ * Fix not supporting channel events with extra with non-string values
+ * Update test database based on https://github.com/nyaruka/rapidpro/pull/4819
+
+v8.3.16 (2023-09-12)
+-------------------------
+ * Stop reading ContactURN.auth and remove from model
+
+v8.3.15 (2023-09-11)
+-------------------------
+ * Start reading and writing ContactURN.auth_tokens
+
+v8.3.14 (2023-09-11)
+-------------------------
+ * Remove support for delegate channels
+
+v8.3.13 (2023-09-11)
+-------------------------
+ * Just noop if trying to sync an Android channel that doesn't have an FM ID
+
+v8.3.12 (2023-09-11)
+-------------------------
+ * Remove encoding URN priority in URN strings as it's not used
+ * Remove having auth as a URN param
+ * Rework message sending so that URNs are loaded before queueing
+ * Update to latest null library and use Map[string] for channel events extra
+
 v8.3.11 (2023-09-05)
 -------------------------
  * Update to latest goflow

Dockerfile

@@ -1,11 +1,20 @@
 FROM golang:1.20
 
+# copy our dev certs into the container
+# WORKDIR /usr/local/share/ca-certificates
+# COPY ./rootCA.pem /usr/local/share/ca-certificates/rootCA.crt
+# RUN /usr/sbin/update-ca-certificates
+
 WORKDIR /usr/src/app
 
 # pre-copy/cache go.mod for pre-downloading dependencies and only redownloading them in subsequent builds if they change
 COPY go.mod go.sum ./
 RUN go mod download && go mod verify
 
+# fetch our docs for our goflow version
+RUN grep goflow go.mod | cut -d" " -f2 | cut -c2- > /tmp/goflow_version
+RUN curl -L "https://github.com/nyaruka/goflow/releases/download/v`cat /tmp/goflow_version`/docs.tar.gz" | tar zxv
+
 COPY . .
 RUN go build -v -o /usr/local/bin/app github.com/nyaruka/mailroom/cmd/mailroom
 

core/models/channel_event.go

@@ -13,21 +13,25 @@ type ChannelEventID int64
 
 // channel event types
 const (
-	NewConversationEventType = ChannelEventType("new_conversation")
-	WelcomeMessageEventType  = ChannelEventType("welcome_message")
-	ReferralEventType        = ChannelEventType("referral")
-	MOMissEventType          = ChannelEventType("mo_miss")
-	MOCallEventType          = ChannelEventType("mo_call")
-	StopContactEventType     = ChannelEventType("stop_contact")
+	EventTypeNewConversation ChannelEventType = "new_conversation"
+	EventTypeWelcomeMessage  ChannelEventType = "welcome_message"
+	EventTypeReferral        ChannelEventType = "referral"
+	EventTypeMissedCall      ChannelEventType = "mo_miss"
+	EventTypeIncomingCall    ChannelEventType = "mo_call"
+	EventTypeStopContact     ChannelEventType = "stop_contact"
+	EventTypeOptIn           ChannelEventType = "optin"
+	EventTypeOptOut          ChannelEventType = "optout"
 )
 
 // ContactSeenEvents are those which count as the contact having been seen
 var ContactSeenEvents = map[ChannelEventType]bool{
-	NewConversationEventType: true,
-	ReferralEventType:        true,
-	MOMissEventType:          true,
-	MOCallEventType:          true,
-	StopContactEventType:     true,
+	EventTypeNewConversation: true,
+	EventTypeReferral:        true,
+	EventTypeMissedCall:      true,
+	EventTypeIncomingCall:    true,
+	EventTypeStopContact:     true,
+	EventTypeOptIn:           true,
+	EventTypeOptOut:          true,
 }
 
 // ChannelEvent represents an event that occurred associated with a channel, such as a referral, missed call, etc..
@@ -39,7 +43,7 @@ type ChannelEvent struct {
 		ChannelID  ChannelID        `json:"channel_id"   db:"channel_id"`
 		ContactID  ContactID        `json:"contact_id"   db:"contact_id"`
 		URNID      URNID            `json:"urn_id"       db:"contact_urn_id"`
-		Extra      null.Map[string] `json:"extra"        db:"extra"`
+		Extra      null.Map[any]    `json:"extra"        db:"extra"`
 		OccurredOn time.Time        `json:"occurred_on"  db:"occurred_on"`
 
 		// only in JSON representation
@@ -50,14 +54,21 @@ type ChannelEvent struct {
 	}
 }
 
-func (e *ChannelEvent) ID() ChannelEventID       { return e.e.ID }
-func (e *ChannelEvent) ContactID() ContactID     { return e.e.ContactID }
-func (e *ChannelEvent) URNID() URNID             { return e.e.URNID }
-func (e *ChannelEvent) OrgID() OrgID             { return e.e.OrgID }
-func (e *ChannelEvent) ChannelID() ChannelID     { return e.e.ChannelID }
-func (e *ChannelEvent) IsNewContact() bool       { return e.e.NewContact }
-func (e *ChannelEvent) OccurredOn() time.Time    { return e.e.OccurredOn }
-func (e *ChannelEvent) Extra() map[string]string { return e.e.Extra }
+func (e *ChannelEvent) ID() ChannelEventID    { return e.e.ID }
+func (e *ChannelEvent) ContactID() ContactID  { return e.e.ContactID }
+func (e *ChannelEvent) URNID() URNID          { return e.e.URNID }
+func (e *ChannelEvent) OrgID() OrgID          { return e.e.OrgID }
+func (e *ChannelEvent) ChannelID() ChannelID  { return e.e.ChannelID }
+func (e *ChannelEvent) IsNewContact() bool    { return e.e.NewContact }
+func (e *ChannelEvent) OccurredOn() time.Time { return e.e.OccurredOn }
+func (e *ChannelEvent) Extra() map[string]any { return e.e.Extra }
+func (e *ChannelEvent) ExtraString(key string) string {
+	asStr, ok := e.e.Extra[key].(string)
+	if ok {
+		return asStr
+	}
+	return ""
+}
 
 // MarshalJSON is our custom marshaller so that our inner struct get output
 func (e *ChannelEvent) MarshalJSON() ([]byte, error) {
@@ -81,7 +92,7 @@ func (e *ChannelEvent) Insert(ctx context.Context, db DBorTx) error {
 }
 
 // NewChannelEvent creates a new channel event for the passed in parameters, returning it
-func NewChannelEvent(eventType ChannelEventType, orgID OrgID, channelID ChannelID, contactID ContactID, urnID URNID, extra map[string]string, isNewContact bool) *ChannelEvent {
+func NewChannelEvent(eventType ChannelEventType, orgID OrgID, channelID ChannelID, contactID ContactID, urnID URNID, extra map[string]any, isNewContact bool) *ChannelEvent {
 	event := &ChannelEvent{}
 	e := &event.e
 
@@ -93,9 +104,9 @@ func NewChannelEvent(eventType ChannelEventType, orgID OrgID, channelID ChannelI
 	e.NewContact = isNewContact
 
 	if extra == nil {
-		e.Extra = null.Map[string]{}
+		e.Extra = null.Map[any]{}
 	} else {
-		e.Extra = null.Map[string](extra)
+		e.Extra = null.Map[any](extra)
 	}
 
 	now := time.Now()

core/models/channel_event_test.go

@@ -20,19 +20,21 @@ func TestChannelEvents(t *testing.T) {
 	start := time.Now()
 
 	// no extra
-	e := models.NewChannelEvent(models.MOMissEventType, testdata.Org1.ID, testdata.TwilioChannel.ID, testdata.Cathy.ID, testdata.Cathy.URNID, nil, false)
+	e := models.NewChannelEvent(models.EventTypeMissedCall, testdata.Org1.ID, testdata.TwilioChannel.ID, testdata.Cathy.ID, testdata.Cathy.URNID, nil, false)
 	err := e.Insert(ctx, rt.DB)
 	assert.NoError(t, err)
 	assert.NotZero(t, e.ID())
-	assert.Equal(t, map[string]string{}, e.Extra())
+	assert.Equal(t, map[string]any{}, e.Extra())
 	assert.True(t, e.OccurredOn().After(start))
 
 	// with extra
-	e2 := models.NewChannelEvent(models.MOMissEventType, testdata.Org1.ID, testdata.TwilioChannel.ID, testdata.Cathy.ID, testdata.Cathy.URNID, map[string]string{"referral_id": "foobar"}, false)
+	e2 := models.NewChannelEvent(models.EventTypeMissedCall, testdata.Org1.ID, testdata.TwilioChannel.ID, testdata.Cathy.ID, testdata.Cathy.URNID, map[string]any{"referral_id": "foobar"}, false)
 	err = e2.Insert(ctx, rt.DB)
 	assert.NoError(t, err)
 	assert.NotZero(t, e2.ID())
-	assert.Equal(t, map[string]string{"referral_id": "foobar"}, e2.Extra())
+	assert.Equal(t, map[string]any{"referral_id": "foobar"}, e2.Extra())
+	assert.Equal(t, "foobar", e2.ExtraString("referral_id"))
+	assert.Equal(t, "", e2.ExtraString("invalid"))
 
 	asJSON, err := json.Marshal(e2)
 	assert.NoError(t, err)

core/models/channels.go

@@ -42,21 +42,20 @@ const (
 type Channel struct {
 	// inner struct for privacy and so we don't collide with method names
 	c struct {
-		ID                 ChannelID                `json:"id"`
-		UUID               assets.ChannelUUID       `json:"uuid"`
-		OrgID              OrgID                    `json:"org_id"`
-		Parent             *assets.ChannelReference `json:"parent"`
-		Name               string                   `json:"name"`
-		Address            string                   `json:"address"`
-		ChannelType        ChannelType              `json:"channel_type"`
-		TPS                int                      `json:"tps"`
-		Country            null.String              `json:"country"`
-		Schemes            []string                 `json:"schemes"`
-		Roles              []assets.ChannelRole     `json:"roles"`
-		MatchPrefixes      []string                 `json:"match_prefixes"`
-		AllowInternational bool                     `json:"allow_international"`
-		MachineDetection   bool                     `json:"machine_detection"`
-		Config             map[string]any           `json:"config"`
+		ID                 ChannelID            `json:"id"`
+		UUID               assets.ChannelUUID   `json:"uuid"`
+		OrgID              OrgID                `json:"org_id"`
+		Name               string               `json:"name"`
+		Address            string               `json:"address"`
+		ChannelType        ChannelType          `json:"channel_type"`
+		TPS                int                  `json:"tps"`
+		Country            null.String          `json:"country"`
+		Schemes            []string             `json:"schemes"`
+		Roles              []assets.ChannelRole `json:"roles"`
+		MatchPrefixes      []string             `json:"match_prefixes"`
+		AllowInternational bool                 `json:"allow_international"`
+		MachineDetection   bool                 `json:"machine_detection"`
+		Config             map[string]any       `json:"config"`
 	}
 }
 
@@ -99,9 +98,6 @@ func (c *Channel) AllowInternational() bool { return c.c.AllowInternational }
 // MachineDetection returns whether this channel should do answering machine detection (only applies to IVR)
 func (c *Channel) MachineDetection() bool { return c.c.MachineDetection }
 
-// Parent returns a reference to the parent channel of this channel (if any)
-func (c *Channel) Parent() *assets.ChannelReference { return c.c.Parent }
-
 // Config returns the config for this channel
 func (c *Channel) Config() map[string]any { return c.c.Config }
 
@@ -197,7 +193,6 @@ SELECT ROW_TO_JSON(r) FROM (SELECT
 	c.id as id,
 	c.uuid as uuid,
 	c.org_id as org_id,
-	(SELECT ROW_TO_JSON(p) FROM (SELECT uuid, name FROM channels_channel cc where cc.id = c.parent_id) p) as parent,
 	c.name as name,
 	c.channel_type as channel_type,
 	COALESCE(c.tps, 10) as tps,

core/models/channels_test.go

@@ -19,9 +19,6 @@ func TestChannels(t *testing.T) {
 	// add some tel specific config to channel 2
 	rt.DB.MustExec(`UPDATE channels_channel SET config = '{"matching_prefixes": ["250", "251"], "allow_international": true}' WHERE id = $1`, testdata.VonageChannel.ID)
 
-	// make twitter channel have a parent of twilio channel
-	rt.DB.MustExec(`UPDATE channels_channel SET parent_id = $1 WHERE id = $2`, testdata.TwilioChannel.ID, testdata.TwitterChannel.ID)
-
 	oa, err := models.GetOrgAssetsWithRefresh(ctx, rt, 1, models.RefreshChannels)
 	require.NoError(t, err)
 
@@ -37,7 +34,6 @@ func TestChannels(t *testing.T) {
 		Roles              []assets.ChannelRole
 		Prefixes           []string
 		AllowInternational bool
-		Parent             *assets.ChannelReference
 	}{
 		{
 			testdata.TwilioChannel.ID,
@@ -48,7 +44,6 @@ func TestChannels(t *testing.T) {
 			[]assets.ChannelRole{"send", "receive", "call", "answer"},
 			nil,
 			false,
-			nil,
 		},
 		{
 			testdata.VonageChannel.ID,
@@ -59,7 +54,6 @@ func TestChannels(t *testing.T) {
 			[]assets.ChannelRole{"send", "receive"},
 			[]string{"250", "251"},
 			true,
-			nil,
 		},
 		{
 			testdata.TwitterChannel.ID,
@@ -70,7 +64,6 @@ func TestChannels(t *testing.T) {
 			[]assets.ChannelRole{"send", "receive"},
 			nil,
 			false,
-			assets.NewChannelReference(testdata.TwilioChannel.UUID, "Twilio"),
 		},
 	}
 
@@ -85,6 +78,5 @@ func TestChannels(t *testing.T) {
 		assert.Equal(t, tc.Schemes, channel.Schemes())
 		assert.Equal(t, tc.Prefixes, channel.MatchPrefixes())
 		assert.Equal(t, tc.AllowInternational, channel.AllowInternational())
-		assert.Equal(t, tc.Parent, channel.Parent())
 	}
 }

core/models/contacts.go

@@ -417,16 +417,16 @@ func queryContactIDs(ctx context.Context, db Queryer, query string, args ...any)
 }
 
 type ContactURN struct {
-	ID        URNID       `json:"id"          db:"id"`
-	OrgID     OrgID       `                   db:"org_id"`
-	ContactID ContactID   `                   db:"contact_id"`
-	Priority  int         `                   db:"priority"`
-	Identity  urns.URN    `json:"identity"    db:"identity"`
-	Scheme    string      `json:"scheme"      db:"scheme"`
-	Path      string      `json:"path"        db:"path"`
-	Display   null.String `json:"display"     db:"display"`
-	Auth      null.String `json:"auth"        db:"auth"`
-	ChannelID ChannelID   `json:"channel_id"  db:"channel_id"`
+	ID         URNID            `json:"id"          db:"id"`
+	OrgID      OrgID            `                   db:"org_id"`
+	ContactID  ContactID        `                   db:"contact_id"`
+	Priority   int              `                   db:"priority"`
+	Identity   urns.URN         `json:"identity"    db:"identity"`
+	Scheme     string           `json:"scheme"      db:"scheme"`
+	Path       string           `json:"path"        db:"path"`
+	Display    null.String      `json:"display"     db:"display"`
+	AuthTokens null.Map[string] `json:"auth_tokens" db:"auth_tokens"`
+	ChannelID  ChannelID        `json:"channel_id"  db:"channel_id"`
 }
 
 // AsURN returns a full URN representation including the query parameters needed by goflow and mailroom
@@ -871,7 +871,7 @@ func insertContactAndURNs(ctx context.Context, db DBorTx, orgID OrgID, userID Us
 func URNForURN(ctx context.Context, db Queryer, oa *OrgAssets, u urns.URN) (urns.URN, error) {
 	urn := &ContactURN{}
 	rows, err := db.QueryContext(ctx,
-		`SELECT row_to_json(r) FROM (SELECT id, scheme, path, display, auth, channel_id, priority FROM contacts_contacturn WHERE identity = $1 AND org_id = $2) r;`,
+		`SELECT row_to_json(r) FROM (SELECT id, scheme, path, display, auth_tokens, channel_id, priority FROM contacts_contacturn WHERE identity = $1 AND org_id = $2) r;`,
 		u.Identity(), oa.OrgID(),
 	)
 	if err != nil {
@@ -931,7 +931,7 @@ func GetOrCreateURN(ctx context.Context, db DBorTx, oa *OrgAssets, contactID Con
 func URNForID(ctx context.Context, db Queryer, oa *OrgAssets, urnID URNID) (urns.URN, error) {
 	urn := &ContactURN{}
 	rows, err := db.QueryContext(ctx,
-		`SELECT row_to_json(r) FROM (SELECT id, scheme, path, display, auth, channel_id, priority FROM contacts_contacturn WHERE id = $1) r;`,
+		`SELECT row_to_json(r) FROM (SELECT id, scheme, path, display, auth_tokens, channel_id, priority FROM contacts_contacturn WHERE id = $1) r;`,
 		urnID,
 	)
 	if err != nil {
@@ -1090,7 +1090,7 @@ UPDATE contacts_contact
  WHERE id = $1`
 
 const sqlSelectURNsByID = `
-SELECT id, org_id, contact_id, identity, priority, scheme, path, display, auth, channel_id 
+SELECT id, org_id, contact_id, identity, priority, scheme, path, display, auth_tokens, channel_id 
   FROM contacts_contacturn 
  WHERE id = ANY($1)`
 
@@ -1316,8 +1316,8 @@ WHERE
 
 const sqlInsertContactURN = `
 INSERT INTO
-	contacts_contacturn(contact_id, identity, path, display, auth, scheme, priority, org_id)
-				 VALUES(:contact_id, :identity, :path, :display, :auth, :scheme, :priority, :org_id)
+	contacts_contacturn(contact_id, identity, path, display, auth_tokens, scheme, priority, org_id)
+				 VALUES(:contact_id, :identity, :path, :display, :auth_tokens, :scheme, :priority, :org_id)
 ON CONFLICT(identity, org_id)
 DO 
 	UPDATE

core/models/triggers.go

@@ -36,6 +36,8 @@ const (
 	IncomingCallTriggerType    = TriggerType("V")
 	ScheduleTriggerType        = TriggerType("S")
 	TicketClosedTriggerType    = TriggerType("T")
+	OptInTriggerType           = TriggerType("I")
+	OptOutTriggerType          = TriggerType("O")
 )
 
 // match type constants
@@ -167,6 +169,20 @@ func FindMatchingNewConversationTrigger(oa *OrgAssets, channel *Channel) *Trigge
 	return findBestTriggerMatch(candidates, channel, nil)
 }
 
+// FindMatchingOptInTrigger finds the best match trigger for optin channel events
+func FindMatchingOptInTrigger(oa *OrgAssets, channel *Channel) *Trigger {
+	candidates := findTriggerCandidates(oa, OptInTriggerType, nil)
+
+	return findBestTriggerMatch(candidates, channel, nil)
+}
+
+// FindMatchingOptOutTrigger finds the best match trigger for optout channel events
+func FindMatchingOptOutTrigger(oa *OrgAssets, channel *Channel) *Trigger {
+	candidates := findTriggerCandidates(oa, OptOutTriggerType, nil)
+
+	return findBestTriggerMatch(candidates, channel, nil)
+}
+
 // FindMatchingReferralTrigger finds the best match trigger for referral click channel events
 func FindMatchingReferralTrigger(oa *OrgAssets, channel *Channel, referrerID string) *Trigger {
 	// first try to find matching referrer ID