Merge tag 'v7.4.0' into update/v7.4.0

 * Update README
 * Use analytics package from gocommon

.github/workflows/ci.yml

@@ -1,8 +1,8 @@
 name: CI
 on: [push, pull_request]
 env:
-  go-version: "1.17.7"
-  redis-version: "3.2.4"
+  go-version: "1.18.x"
+  redis-version: "5.0.6"
 jobs:
   test:
     name: Test
@@ -37,7 +37,7 @@ jobs:
 
       - name: Upload coverage
         if: success()
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v2
         with:
           fail_ci_if_error: true
 

CHANGELOG.md

@@ -1,3 +1,57 @@
+v7.4.0
+----------
+ * Update README
+ * Use analytics package from gocommon
+
+v7.3.10
+----------
+ * Make sure text are sent after audio attachments for WA channels
+
+v7.3.9
+----------
+ * Add arm64 as a build target
+ * Add support for WA Cloud API
+ * Refactor FBA tests
+
+v7.3.8
+----------
+ * Add log to status first when handling telegram opt outs
+
+v7.3.7
+----------
+ * Fix to not stop contact for other errors
+
+v7.3.6
+----------
+ * Update to go 1.18 and latest gocommon/phonenumbers/jsonparser
+
+v7.3.5
+----------
+ * Update Start Mobile send URL
+
+v7.3.4
+----------
+ * Update WhatsApp handler so that we update the URN if the returned ID doesn't match
+ * Stop Telegram contact that have blocked the channel bot
+
+v7.3.3
+----------
+ * Quick fix to stop JSON content being omitted in logs
+
+v7.3.2
+----------
+ * Update to latest gocommon and start using httpx.DetectContentType
+ * Add link preview attribute for sending whatsapp
+ * Update golang.org/x/sys
+
+v7.3.1
+----------
+ * Fix handling stops via status callbacks on Twilio
+
+v7.3.0
+----------
+ * Support stopping contacts when we get stop events on status callbacks
+
 v7.2.0
 ----------
  * CI testing with go 1.17.7

README.md

@@ -4,28 +4,26 @@
 [![codecov](https://codecov.io/gh/nyaruka/courier/branch/main/graph/badge.svg)](https://codecov.io/gh/nyaruka/courier)
 [![Go Report Card](https://goreportcard.com/badge/github.com/nyaruka/courier)](https://goreportcard.com/report/github.com/nyaruka/courier)
 
-# About 
-
 Courier is a messaging gateway for text-based messaging channels. It abstracts out various different
 texting mediums and providers, allowing applications to focus on the creation and processing of those messages.
 
-Current courier supports over 50 different channel types, ranging for SMS aggregators like Twilio to
+Currently it supports over 50 different channel types, ranging for SMS aggregators like Twilio to
 IP channels like Facebook and Telegram messenger. The goal is for Courier to support every popular
 messaging channels and aggregator and we are happy to accept pull requests to help accomplish that.
 
 Courier is currently used to power [RapidPro](https://rapidpro.io) and [TextIt](https://textit.com)
 but the backend is pluggable, so you can add your own backend to read and write messages.
 
-# Deploying
+## Deploying
 
-As courier is a go application, it compiles to a binary and that binary along with the config file is all
+As a Go application, it compiles to a binary and that binary along with the config file is all
 you need to run it on your server. You can find bundles for each platform in the
-[releases directory](https://github.com/nyaruka/courier/releases). We recommend running Courier
+[releases directory](https://github.com/nyaruka/courier/releases). We recommend running it
 behind a reverse proxy such as nginx or Elastic Load Balancer that provides HTTPs encryption.
 
-# Configuration
+## Configuration
 
-Courier uses a tiered configuration system, each option takes precendence over the ones above it:
+The service uses a tiered configuration system, each option takes precendence over the ones above it:
 
  1. The configuration file
  2. Environment variables starting with `COURIER_` 
@@ -35,7 +33,7 @@ We recommend running courier with no changes to the configuration and no paramet
 environment variables to configure it. You can use `% courier --help` to see a list of the
 environment variables and parameters and for more details on each option.
 
-# RapidPro Configuration
+### RapidPro
 
 For use with RapidPro, you will want to configure these settings:
 
@@ -58,9 +56,9 @@ Recommended settings for error and performance monitoring:
  * `COURIER_LIBRATO_TOKEN`: The token to use for logging of events to Librato
  * `COURIER_SENTRY_DSN`: The DSN to use when logging errors to Sentry
 
-# Development
+## Development
 
-Once you've checked out the code, you can build Courier with:
+Once you've checked out the code, you can build it with:
 
 ```
 go install github.com/nyaruka/courier/cmd/courier

backends/rapidpro/backend.go

@@ -18,10 +18,10 @@ import (
 	"github.com/nyaruka/courier/batch"
 	"github.com/nyaruka/courier/queue"
 	"github.com/nyaruka/courier/utils"
+	"github.com/nyaruka/gocommon/analytics"
 	"github.com/nyaruka/gocommon/dbutil"
 	"github.com/nyaruka/gocommon/storage"
 	"github.com/nyaruka/gocommon/urns"
-	"github.com/nyaruka/librato"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
@@ -479,14 +479,14 @@ func (b *backend) Heartbeat() error {
 	b.redisWaitDuration = redisStats.WaitDuration
 	b.redisWaitCount = redisStats.WaitCount
 
-	librato.Gauge("courier.db_busy", float64(dbStats.InUse))
-	librato.Gauge("courier.db_idle", float64(dbStats.Idle))
-	librato.Gauge("courier.db_wait_ms", float64(dbWaitDurationInPeriod/time.Millisecond))
-	librato.Gauge("courier.db_wait_count", float64(dbWaitCountInPeriod))
-	librato.Gauge("courier.redis_wait_ms", float64(redisWaitDurationInPeriod/time.Millisecond))
-	librato.Gauge("courier.redis_wait_count", float64(redisWaitCountInPeriod))
-	librato.Gauge("courier.bulk_queue", float64(bulkSize))
-	librato.Gauge("courier.priority_queue", float64(prioritySize))
+	analytics.Gauge("courier.db_busy", float64(dbStats.InUse))
+	analytics.Gauge("courier.db_idle", float64(dbStats.Idle))
+	analytics.Gauge("courier.db_wait_ms", float64(dbWaitDurationInPeriod/time.Millisecond))
+	analytics.Gauge("courier.db_wait_count", float64(dbWaitCountInPeriod))
+	analytics.Gauge("courier.redis_wait_ms", float64(redisWaitDurationInPeriod/time.Millisecond))
+	analytics.Gauge("courier.redis_wait_count", float64(redisWaitCountInPeriod))
+	analytics.Gauge("courier.bulk_queue", float64(bulkSize))
+	analytics.Gauge("courier.priority_queue", float64(prioritySize))
 
 	logrus.WithFields(logrus.Fields{
 		"db_busy":          dbStats.InUse,

backends/rapidpro/contact.go

@@ -9,10 +9,10 @@ import (
 	"unicode/utf8"
 
 	"github.com/nyaruka/courier"
+	"github.com/nyaruka/gocommon/analytics"
 	"github.com/nyaruka/gocommon/dbutil"
 	"github.com/nyaruka/gocommon/urns"
 	"github.com/nyaruka/gocommon/uuids"
-	"github.com/nyaruka/librato"
 	"github.com/nyaruka/null"
 	"github.com/pkg/errors"
 
@@ -212,7 +212,7 @@ func contactForURN(ctx context.Context, b *backend, org OrgID, channel *DBChanne
 	contact.URNID_ = contactURN.ID
 
 	// log that we created a new contact to librato
-	librato.Gauge("courier.new_contact", float64(1))
+	analytics.Gauge("courier.new_contact", float64(1))
 
 	// and return it
 	return contact, nil

channel_log.go

@@ -2,11 +2,11 @@ package courier
 
 import (
 	"fmt"
-	"net/http"
 	"strings"
 	"time"
 
 	"github.com/nyaruka/courier/utils"
+	"github.com/nyaruka/gocommon/httpx"
 )
 
 // NilStatusCode is used when we have an error before even sending anything
@@ -42,10 +42,10 @@ func sanitizeBody(body string) string {
 		return body
 	}
 
-	ct := http.DetectContentType([]byte(parts[1]))
+	ct := httpx.DetectContentType([]byte(parts[1]))
 
 	// if this isn't text, replace with placeholder
-	if !strings.HasPrefix(ct, "text") {
+	if !strings.HasPrefix(ct, "text") && !strings.HasPrefix(ct, "application/json") {
 		return fmt.Sprintf("%s\r\n\r\nOmitting non text body of type: %s", parts[0], ct)
 	}
 

go.mod

@@ -1,56 +1,56 @@
 module github.com/nyaruka/courier
 
+go 1.18
+
 require (
 	github.com/antchfx/xmlquery v0.0.0-20181223105952-355641961c92
-	github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect
-	github.com/aws/aws-sdk-go v1.40.56 // indirect
-	github.com/buger/jsonparser v0.0.0-20180318095312-2cac668e8456
-	github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect
+	github.com/buger/jsonparser v1.1.1
 	github.com/dghubble/oauth1 v0.4.0
 	github.com/evalphobia/logrus_sentry v0.4.6
-	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/go-errors/errors v1.0.1
-	github.com/go-playground/locales v0.14.0 // indirect
-	github.com/go-playground/universal-translator v0.18.0 // indirect
-	github.com/gofrs/uuid v3.3.0+incompatible
-	github.com/gomodule/redigo v2.0.0+incompatible
-	github.com/gorilla/schema v1.0.2
-	github.com/jmoiron/sqlx v1.3.4
-	github.com/kr/pretty v0.1.0 // indirect
-	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect
-	github.com/lib/pq v1.10.4
 	github.com/mattn/go-sqlite3 v1.14.10 // indirect
+	github.com/gofrs/uuid v4.2.0+incompatible
+	github.com/gomodule/redigo v1.8.8
+	github.com/gorilla/schema v1.0.2
+	github.com/jmoiron/sqlx v1.3.5
+	github.com/lib/pq v1.10.6
 	github.com/nyaruka/ezconf v0.2.1
-	github.com/nyaruka/librato v1.0.0
+	github.com/nyaruka/gocommon v1.22.2
 	github.com/nyaruka/null v1.1.1
 	github.com/pkg/errors v0.9.1
-	github.com/sirupsen/logrus v1.4.2
-	github.com/stretchr/testify v1.7.1
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	github.com/sirupsen/logrus v1.8.1
+	github.com/stretchr/testify v1.7.2
+	golang.org/x/mod v0.4.2
 	gopkg.in/go-playground/validator.v9 v9.31.0
 	gopkg.in/h2non/filetype.v1 v1.0.5
 )
 
 require (
-	github.com/gabriel-vasile/mimetype v1.4.1
 	github.com/golang-jwt/jwt/v4 v4.4.1
 	github.com/lestrrat-go/jwx v1.2.25
-	github.com/nyaruka/gocommon v1.5.3
 	github.com/nyaruka/redisx v0.2.2
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	golang.org/x/mod v0.5.1
 	gopkg.in/go-playground/assert.v1 v1.2.1
 )
 
 require (
+	github.com/antchfx/xpath v0.0.0-20181208024549-4bbdf6db12aa // indirect
+	github.com/aws/aws-sdk-go v1.44.34 // indirect
+	github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d // indirect
 	github.com/fatih/structs v1.0.0 // indirect
 	github.com/goccy/go-json v0.9.7 // indirect
-	github.com/golang/protobuf v1.3.2 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.0 // indirect
+	github.com/getsentry/raven-go v0.0.0-20180517221441-ed7bcb39ff10 // indirect
+	github.com/go-playground/locales v0.14.0 // indirect
+	github.com/go-playground/universal-translator v0.18.0 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.0 // indirect
@@ -59,18 +59,19 @@ require (
 	github.com/lestrrat-go/option v1.0.0 // indirect
 	github.com/naoina/go-stringutil v0.1.0 // indirect
 	github.com/naoina/toml v0.1.1 // indirect
-	github.com/nyaruka/phonenumbers v1.0.71 // indirect
+	github.com/nyaruka/librato v1.0.0 // indirect
+	github.com/nyaruka/phonenumbers v1.0.75 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/shopspring/decimal v1.2.0 // indirect
 	golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f // indirect
-	golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e // indirect
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
+	github.com/shopspring/decimal v1.3.1 // indirect
+	golang.org/x/net v0.0.0-20220614195744-fb05da6f9022 // indirect
+	golang.org/x/sys v0.0.0-20220614162138-6c1b26c55098 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-go 1.17
-
 replace github.com/nyaruka/gocommon => github.com/Ilhasoft/gocommon v1.16.2-weni
 
 replace github.com/gomodule/redigo => github.com/gomodule/redigo v1.8.8