diff --git a/CHANGELOG.md b/CHANGELOG.md index 38ce20750..2c2d6965f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,52 @@ +v5.7.24 +---------- + * Add SessionStatus to messages queued to courier + +v5.7.23 +---------- + * Make defining new task types easier and drier + * Better locking when handling + * Fix and simplify creation of channel logs in IVR handlers + +v5.7.22 +---------- + * Update to latest goflow v0.104.1 + +v5.7.21 +---------- + * Simplify test-smtp cmd using smtpx package + * Create new dbutil package with generic DB stuff + * Add task to calculate fires for new campaign event + +v5.7.20 +---------- + * Fix incoming attachments from Zendesk + +v5.7.19 +---------- + * Update to latest goflow + * Empty contact names and languages should be saved as NULL + * Delete no longer used utils/celery package + +v5.7.18 +---------- + * Update to latest goflow + * Add support for incoming attachments on ticketing services + +v5.7.17 +---------- + * Use status for elastic queries that need to filter out non-active contacts + +v5.7.16 +---------- + * Add support for excluding contacts from searches by ids + * Rework utils/storage to be generic and moveable to gocommon + +v5.7.15 +---------- + * Add create contact endpoint which uses modifiers to add fields and groups to contacts + * Rework contact creation functions to support creation with multiple URNs + v5.7.14 ---------- * Stop writing is_blocked and is_stopped diff --git a/cmd/mailroom/main.go b/cmd/mailroom/main.go index 47a2bc485..28f84ef8b 100644 --- a/cmd/mailroom/main.go +++ b/cmd/mailroom/main.go @@ -6,15 +6,17 @@ import ( "runtime" "syscall" - _ "github.com/lib/pq" "github.com/nyaruka/ezconf" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/logrus_sentry" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/config" - "github.com/sirupsen/logrus" _ "github.com/nyaruka/mailroom/hooks" + _ "github.com/nyaruka/mailroom/ivr/nexmo" + _ "github.com/nyaruka/mailroom/ivr/twiml" + _ "github.com/nyaruka/mailroom/services/tickets/mailgun" + _ "github.com/nyaruka/mailroom/services/tickets/zendesk" _ "github.com/nyaruka/mailroom/tasks/broadcasts" _ "github.com/nyaruka/mailroom/tasks/campaigns" _ "github.com/nyaruka/mailroom/tasks/expirations" @@ -25,7 +27,6 @@ import ( _ "github.com/nyaruka/mailroom/tasks/starts" _ "github.com/nyaruka/mailroom/tasks/stats" _ "github.com/nyaruka/mailroom/tasks/timeouts" - _ "github.com/nyaruka/mailroom/web/contact" _ "github.com/nyaruka/mailroom/web/docs" _ "github.com/nyaruka/mailroom/web/expression" @@ -37,11 +38,8 @@ import ( _ "github.com/nyaruka/mailroom/web/surveyor" _ "github.com/nyaruka/mailroom/web/ticket" - _ "github.com/nyaruka/mailroom/services/tickets/mailgun" - _ "github.com/nyaruka/mailroom/services/tickets/zendesk" - - _ "github.com/nyaruka/mailroom/ivr/nexmo" - _ "github.com/nyaruka/mailroom/ivr/twiml" + _ "github.com/lib/pq" + "github.com/sirupsen/logrus" ) var version = "Dev" diff --git a/cmd/test-smtp/main.go b/cmd/test-smtp/main.go index 83a9a7395..be108dcc3 100644 --- a/cmd/test-smtp/main.go +++ b/cmd/test-smtp/main.go @@ -1,15 +1,13 @@ package main import ( - "net/url" - "strconv" - "github.com/nyaruka/ezconf" + "github.com/nyaruka/goflow/utils/smtpx" + "github.com/sirupsen/logrus" - "gopkg.in/mail.v2" ) -type Config struct { +type config struct { URL string `help:"the SMTP formatted URL to use to test sending"` To string `help:"the email address to send to"` Subject string `help:"the email subject to send"` @@ -18,7 +16,7 @@ type Config struct { func main() { // get our smtp server config - options := &Config{ + options := &config{ URL: "smtp://foo%40zap.com:opensesame@smtp.gmail.com:587/?from=foo%40zap.com&tls=true", To: "test@temba.io", Subject: "Test Email", @@ -31,52 +29,14 @@ func main() { ) loader.MustLoad() - // parse it - url, err := url.Parse(options.URL) + client, err := smtpx.NewClientFromURL(options.URL) if err != nil { logrus.WithError(err).Fatalf("unable to parse smtp config: %s", options.URL) } - // figure out our port - sPort := url.Port() - if sPort == "" { - sPort = "25" - } - port, err := strconv.Atoi(sPort) - if err != nil { - logrus.WithError(err).Fatalf("invalid port configuration: %s", options.URL) - } - - // and our user and password - if url.User == nil { - logrus.Fatalf("no user set for smtp server: %s", options.URL) - } - password, _ := url.User.Password() - - // get our from - from := url.Query()["from"] - if len(from) == 0 { - from = []string{url.User.Username()} - } - - // create our dialer for our org - d := mail.NewDialer(url.Hostname(), port, url.User.Username(), password) - - // send each of our emails, errors are logged but don't stop us from trying to send our other emails - m := mail.NewMessage() - m.SetHeader("From", from[0]) - m.SetHeader("To", options.To) - m.SetHeader("Subject", options.Subject) - m.SetBody("text/plain", options.Body) - - logrus.WithFields(logrus.Fields{ - "hostname": url.Hostname(), - "port": port, - "username": url.User.Username(), - "password": password, - }).Info("attempting to send email") + m := smtpx.NewMessage([]string{options.To}, options.Subject, options.Body, "") - err = d.DialAndSend(m) + err = smtpx.Send(client, m) if err != nil { logrus.WithError(err).Fatal("error sending email") } diff --git a/go.mod b/go.mod index 4f315d351..66d87dc45 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,12 @@ module github.com/nyaruka/mailroom require ( github.com/Masterminds/semver v1.5.0 github.com/apex/log v1.1.4 - github.com/aws/aws-sdk-go v1.30.17 github.com/buger/jsonparser v0.0.0-20200322175846-f7e751efca13 github.com/certifi/gocertifi v0.0.0-20200211180108-c7c1fbc02894 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/edganiukov/fcm v0.4.0 github.com/getsentry/raven-go v0.1.2-0.20190125112653-238ebd86338d // indirect - github.com/go-chi/chi v3.3.3+incompatible + github.com/go-chi/chi v4.1.2+incompatible github.com/golang/protobuf v1.4.0 github.com/gomodule/redigo v2.0.0+incompatible github.com/gorilla/schema v1.1.0 @@ -18,8 +17,8 @@ require ( github.com/lib/pq v1.4.0 github.com/mattn/go-sqlite3 v1.10.0 // indirect github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.3.0 - github.com/nyaruka/goflow v0.102.1 + github.com/nyaruka/gocommon v1.5.1 + github.com/nyaruka/goflow v0.104.1 github.com/nyaruka/librato v1.0.0 github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d github.com/nyaruka/null v1.2.0 @@ -32,7 +31,6 @@ require ( github.com/sirupsen/logrus v1.5.0 github.com/stretchr/testify v1.5.1 gopkg.in/go-playground/validator.v9 v9.31.0 - gopkg.in/mail.v2 v2.3.1 ) go 1.14 diff --git a/go.sum b/go.sum index dc6cc1e92..ac54f6daf 100644 --- a/go.sum +++ b/go.sum @@ -13,8 +13,8 @@ github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/aws/aws-sdk-go v1.20.6 h1:kmy4Gvdlyez1fV4kw5RYxZzWKVyuHZHgPWeU/YvRsV4= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.30.17 h1:Y8cyVjc7RWSJwt9uymwnsKZI4qnmamMkfYJJ806wHtA= -github.com/aws/aws-sdk-go v1.30.17/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.34.17 h1:9OzUgRrLmYm2mbfFx4v+2nBEg+Cvape1cvn9C3RNWTE= +github.com/aws/aws-sdk-go v1.34.17/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -39,8 +39,8 @@ github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHqu github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getsentry/raven-go v0.1.2-0.20190125112653-238ebd86338d h1:CIp8WnfXz70wJVQ0ytr3dswFYGoJbAxWgNvaLpiu3sY= github.com/getsentry/raven-go v0.1.2-0.20190125112653-238ebd86338d/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= -github.com/go-chi/chi v3.3.3+incompatible h1:KHkmBEMNkwKuK4FdQL7N2wOeB9jnIx7jR5wsuSBEFI8= -github.com/go-chi/chi v3.3.3+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= +github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= +github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -55,8 +55,8 @@ github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gofrs/uuid v3.2.0+incompatible h1:y12jRkkFxsd7GpqdSZ+/KCs/fJbqpEXSGd4+jfEaewE= -github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -128,19 +128,18 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.3.0 h1:IqaPT4KQ2oVq/2Ivp/c+RVCs8v71+RzPU2VhMoRrgpU= -github.com/nyaruka/gocommon v1.3.0/go.mod h1:w7lKxIkm/qLAoO9Y3aI1LV7EiYogn6+1C8MTEjxTC9M= -github.com/nyaruka/goflow v0.102.1 h1:7QX2jTwV7uIbaGnkkpmB+ao+E7Cmyar9g7sRQH4Bu3M= -github.com/nyaruka/goflow v0.102.1/go.mod h1:wuvXZTs6a6S1rjSRLaQGVxDfKomDJ/1XQoLXCqFekK4= +github.com/nyaruka/gocommon v1.5.1 h1:2R6uo6EVSTHOerupAmVm6h5fyufO189dlv/5gwHj3lM= +github.com/nyaruka/gocommon v1.5.1/go.mod h1:6XoaOsVk6z+294hM6pZxX3fDgT2IyLV8hFU4FoQz9Aw= +github.com/nyaruka/goflow v0.104.1 h1:uFmB4dDJwuVJxgcJFEnbXzFOYrqNiiJLnlxd0t6yXxg= +github.com/nyaruka/goflow v0.104.1/go.mod h1:dZBsFXFQ9EzcDlEupM7rcbMdDpIGUNOCMndSCRo7Ofo= github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d h1:hyp9u36KIwbTCo2JAJ+TuJcJBc+UZzEig7RI/S5Dvkc= github.com/nyaruka/logrus_sentry v0.8.2-0.20190129182604-c2962b80ba7d/go.mod h1:FGdPJVDTNqbRAD+2RvnK9YoO2HcEW7ogSMPzc90b638= github.com/nyaruka/null v1.2.0 h1:uEbkyy4Z+zPB2Pr3ryQh/0N2965I9kEsXq/cGpyJ7PA= github.com/nyaruka/null v1.2.0/go.mod h1:HSAFbLNOaEhHnoU0VCveCPz0GDtJ3GEtFWhvnBNkhPE= -github.com/nyaruka/phonenumbers v1.0.34/go.mod h1:GQ0cTHlrxPrhoLwyQ1blyN1hO794ygt6FTHWrFB5SSc= -github.com/nyaruka/phonenumbers v1.0.55 h1:bj0nTO88Y68KeUQ/n3Lo2KgK7lM1hF7L9NFuwcCl3yg= -github.com/nyaruka/phonenumbers v1.0.55/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= +github.com/nyaruka/phonenumbers v1.0.57 h1:V4FNPs061PSUOEzQaLH0+pfzEdqoiMH/QJWryx/0hfs= +github.com/nyaruka/phonenumbers v1.0.57/go.mod h1:sDaTZ/KPX5f8qyV9qN+hIm+4ZBARJrupC6LuhshJq1U= github.com/olivere/elastic v6.2.33+incompatible h1:SRPB2w2OhJ7iULftDEHsNPRoL2GLREqPMRalVmbZaEw= github.com/olivere/elastic v6.2.33+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -198,7 +197,6 @@ golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180921000356-2f5d2388922f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -223,6 +221,8 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/goflow/engine.go b/goflow/engine.go index f01100613..1d750d015 100644 --- a/goflow/engine.go +++ b/goflow/engine.go @@ -4,10 +4,10 @@ import ( "sync" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/engine" "github.com/nyaruka/goflow/services/webhooks" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/config" "github.com/shopspring/decimal" diff --git a/goflow/flows.go b/goflow/flows.go index 9fd785d6a..7d1b56619 100644 --- a/goflow/flows.go +++ b/goflow/flows.go @@ -4,10 +4,10 @@ import ( "encoding/json" "sync" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/definition" "github.com/nyaruka/goflow/flows/definition/migrations" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/config" "github.com/Masterminds/semver" diff --git a/goflow/flows_test.go b/goflow/flows_test.go index f64ef0cb2..2d792046f 100644 --- a/goflow/flows_test.go +++ b/goflow/flows_test.go @@ -3,11 +3,11 @@ package goflow_test import ( "testing" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/test" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/goflow" "github.com/Masterminds/semver" diff --git a/hooks/campaigns_test.go b/hooks/campaigns_test.go index 634a8696b..3e2703826 100644 --- a/hooks/campaigns_test.go +++ b/hooks/campaigns_test.go @@ -3,10 +3,10 @@ package hooks import ( "testing" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/actions" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" ) diff --git a/hooks/contact_field_changed.go b/hooks/contact_field_changed.go index 239468364..f094c5223 100644 --- a/hooks/contact_field_changed.go +++ b/hooks/contact_field_changed.go @@ -72,7 +72,7 @@ func (h *CommitFieldChangesHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *red // first apply our deletes // in pg9.6 we need to do this as one query per field type, in pg10 we can rewrite this to be a single query for _, fds := range fieldDeletes { - err := models.BulkSQL(ctx, "deleting contact field values", tx, deleteContactFieldsSQL, fds) + err := models.BulkQuery(ctx, "deleting contact field values", tx, deleteContactFieldsSQL, fds) if err != nil { return errors.Wrapf(err, "error deleting contact fields") } @@ -80,7 +80,7 @@ func (h *CommitFieldChangesHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *red // then our updates if len(fieldUpdates) > 0 { - err := models.BulkSQL(ctx, "updating contact field values", tx, updateContactFieldsSQL, fieldUpdates) + err := models.BulkQuery(ctx, "updating contact field values", tx, updateContactFieldsSQL, fieldUpdates) if err != nil { return errors.Wrapf(err, "error updating contact fields") } diff --git a/hooks/contact_language_changed.go b/hooks/contact_language_changed.go index 344ebf6a1..38776c0a3 100644 --- a/hooks/contact_language_changed.go +++ b/hooks/contact_language_changed.go @@ -8,6 +8,7 @@ import ( "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/null" "github.com/sirupsen/logrus" ) @@ -27,11 +28,11 @@ func (h *CommitLanguageChangesHook) Apply(ctx context.Context, tx *sqlx.Tx, rp * for s, e := range scenes { // we only care about the last name change event := e[len(e)-1].(*events.ContactLanguageChangedEvent) - updates = append(updates, &languageUpdate{s.ContactID(), event.Language}) + updates = append(updates, &languageUpdate{s.ContactID(), null.String(event.Language)}) } // do our update - return models.BulkSQL(ctx, "updating contact language", tx, updateContactLanguageSQL, updates) + return models.BulkQuery(ctx, "updating contact language", tx, updateContactLanguageSQL, updates) } // handleContactLanguageChanged is called when we process a contact language change @@ -50,7 +51,7 @@ func handleContactLanguageChanged(ctx context.Context, tx *sqlx.Tx, rp *redis.Po // struct used for our bulk update type languageUpdate struct { ContactID models.ContactID `db:"id"` - Language string `db:"language"` + Language null.String `db:"language"` } const updateContactLanguageSQL = ` diff --git a/hooks/contact_language_changed_test.go b/hooks/contact_language_changed_test.go index 0c5fec1d8..f60a8f171 100644 --- a/hooks/contact_language_changed_test.go +++ b/hooks/contact_language_changed_test.go @@ -10,7 +10,7 @@ import ( func TestContactLanguageChanged(t *testing.T) { tcs := []HookTestCase{ - HookTestCase{ + { Actions: ContactActionMap{ models.CathyID: []flows.Action{ actions.NewSetContactLanguage(newActionUUID(), "fra"), @@ -19,6 +19,9 @@ func TestContactLanguageChanged(t *testing.T) { models.GeorgeID: []flows.Action{ actions.NewSetContactLanguage(newActionUUID(), "spa"), }, + models.AlexandriaID: []flows.Action{ + actions.NewSetContactLanguage(newActionUUID(), ""), + }, }, SQLAssertions: []SQLAssertion{ { @@ -36,6 +39,11 @@ func TestContactLanguageChanged(t *testing.T) { Args: []interface{}{models.BobID}, Count: 1, }, + { + SQL: "select count(*) from contacts_contact where id = $1 and language is NULL;", + Args: []interface{}{models.AlexandriaID}, + Count: 1, + }, }, }, } diff --git a/hooks/contact_name_changed.go b/hooks/contact_name_changed.go index b51c75be8..2b35009f7 100644 --- a/hooks/contact_name_changed.go +++ b/hooks/contact_name_changed.go @@ -9,6 +9,7 @@ import ( "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/null" "github.com/sirupsen/logrus" ) @@ -28,11 +29,11 @@ func (h *CommitNameChangesHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redi for s, e := range scenes { // we only care about the last name change event := e[len(e)-1].(*events.ContactNameChangedEvent) - updates = append(updates, &nameUpdate{s.ContactID(), fmt.Sprintf("%.128s", event.Name)}) + updates = append(updates, &nameUpdate{s.ContactID(), null.String(fmt.Sprintf("%.128s", event.Name))}) } // do our update - return models.BulkSQL(ctx, "updating contact name", tx, updateContactNameSQL, updates) + return models.BulkQuery(ctx, "updating contact name", tx, updateContactNameSQL, updates) } // handleContactNameChanged changes the name of the contact @@ -51,7 +52,7 @@ func handleContactNameChanged(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, // struct used for our bulk insert type nameUpdate struct { ContactID models.ContactID `db:"id"` - Name string `db:"name"` + Name null.String `db:"name"` } const updateContactNameSQL = ` diff --git a/hooks/contact_name_changed_test.go b/hooks/contact_name_changed_test.go index 97be952b1..a0395c016 100644 --- a/hooks/contact_name_changed_test.go +++ b/hooks/contact_name_changed_test.go @@ -10,7 +10,7 @@ import ( func TestContactNameChanged(t *testing.T) { tcs := []HookTestCase{ - HookTestCase{ + { Actions: ContactActionMap{ models.CathyID: []flows.Action{ actions.NewSetContactName(newActionUUID(), "Fred"), @@ -19,6 +19,9 @@ func TestContactNameChanged(t *testing.T) { models.GeorgeID: []flows.Action{ actions.NewSetContactName(newActionUUID(), "Geoff Newman"), }, + models.BobID: []flows.Action{ + actions.NewSetContactName(newActionUUID(), ""), + }, models.AlexandriaID: []flows.Action{ actions.NewSetContactName(newActionUUID(), "😃234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890"), }, @@ -34,7 +37,7 @@ func TestContactNameChanged(t *testing.T) { Count: 1, }, { - SQL: "select count(*) from contacts_contact where name = 'Bob' and id = $1", + SQL: "select count(*) from contacts_contact where name IS NULL and id = $1", Args: []interface{}{models.BobID}, Count: 1, }, diff --git a/hooks/hooks_test.go b/hooks/hooks_test.go index 0a3a77f63..4dd6f0892 100644 --- a/hooks/hooks_test.go +++ b/hooks/hooks_test.go @@ -9,13 +9,13 @@ import ( "time" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/definition" "github.com/nyaruka/goflow/flows/routers" "github.com/nyaruka/goflow/flows/triggers" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/runner" "github.com/nyaruka/mailroom/testsuite" diff --git a/hooks/msg_created.go b/hooks/msg_created.go index a758f372a..a04bc5118 100644 --- a/hooks/msg_created.go +++ b/hooks/msg_created.go @@ -63,7 +63,7 @@ func (h *SendMessagesHook) Apply(ctx context.Context, tx *sqlx.Tx, rp *redis.Poo if len(courierMsgs) > 0 { // if our scene has a timeout, set it on our last message if s.Session().Timeout() != nil && s.Session().WaitStartedOn() != nil { - courierMsgs[len(courierMsgs)-1].SetTimeout(s.SessionID(), *s.Session().WaitStartedOn(), *s.Session().Timeout()) + courierMsgs[len(courierMsgs)-1].SetTimeout(*s.Session().WaitStartedOn(), *s.Session().Timeout()) } log := log.WithField("messages", courierMsgs).WithField("scene", s.SessionID) @@ -220,6 +220,9 @@ func handleMsgCreated(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, oa *mode return errors.Wrapf(err, "error creating outgoing message to %s", event.Msg.URN()) } + // include some information about the session + msg.SetSession(scene.Session().ID(), scene.Session().Status()) + // set our reply to as well (will be noop in cases when there is no incoming message) msg.SetResponseTo(scene.Session().IncomingMsgID(), scene.Session().IncomingMsgExternalID()) diff --git a/hooks/msg_created_test.go b/hooks/msg_created_test.go index 08c242f49..d77097ddf 100644 --- a/hooks/msg_created_test.go +++ b/hooks/msg_created_test.go @@ -10,10 +10,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/gomodule/redigo/redis" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/actions" - "github.com/nyaruka/goflow/utils/uuids" ) func TestMsgCreated(t *testing.T) { diff --git a/hooks/msg_received_test.go b/hooks/msg_received_test.go index f38cd3b0e..0bbe7a3de 100644 --- a/hooks/msg_received_test.go +++ b/hooks/msg_received_test.go @@ -5,9 +5,9 @@ import ( "time" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/actions" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" ) diff --git a/hooks/session_triggered_test.go b/hooks/session_triggered_test.go index 8dc571186..d052b16f1 100644 --- a/hooks/session_triggered_test.go +++ b/hooks/session_triggered_test.go @@ -4,10 +4,10 @@ import ( "encoding/json" "testing" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/actions" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" "github.com/nyaruka/mailroom/testsuite" diff --git a/hooks/ticket_opened_test.go b/hooks/ticket_opened_test.go index 3d777c615..a81ecdd8c 100644 --- a/hooks/ticket_opened_test.go +++ b/hooks/ticket_opened_test.go @@ -4,10 +4,10 @@ import ( "testing" "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/actions" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" diff --git a/ivr/ivr.go b/ivr/ivr.go index b8628ad7a..1bf363bc8 100644 --- a/ivr/ivr.go +++ b/ivr/ivr.go @@ -11,18 +11,18 @@ import ( "time" "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/excellent/types" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/resumes" "github.com/nyaruka/goflow/flows/triggers" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/runner" - "github.com/nyaruka/mailroom/utils/storage" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" @@ -481,7 +481,7 @@ func ResumeIVRFlow( // filename is based on our org id and msg UUID filename := string(msgUUID) + path.Ext(attachment.URL()) - attachment, err = oa.Org().StoreAttachment(store, config.S3MediaPrefix, filename, body) + attachment, err = oa.Org().StoreAttachment(store, filename, body) if err != nil { return errors.Wrapf(err, "unable to store IVR attachment") } diff --git a/ivr/nexmo/nexmo.go b/ivr/nexmo/nexmo.go index bc1da095a..8bf4e1b4e 100644 --- a/ivr/nexmo/nexmo.go +++ b/ivr/nexmo/nexmo.go @@ -20,12 +20,12 @@ import ( "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/goflow/flows/routers/waits" "github.com/nyaruka/goflow/flows/routers/waits/hints" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/ivr" "github.com/nyaruka/mailroom/models" diff --git a/ivr/nexmo/nexmo_test.go b/ivr/nexmo/nexmo_test.go index 561809ddf..76ffbdbdf 100644 --- a/ivr/nexmo/nexmo_test.go +++ b/ivr/nexmo/nexmo_test.go @@ -5,13 +5,13 @@ import ( "testing" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/goflow/flows/routers/waits" "github.com/nyaruka/goflow/flows/routers/waits/hints" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" diff --git a/ivr/twiml/twiml_test.go b/ivr/twiml/twiml_test.go index 8a90e997a..4e925b89e 100644 --- a/ivr/twiml/twiml_test.go +++ b/ivr/twiml/twiml_test.go @@ -5,12 +5,12 @@ import ( "testing" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/goflow/flows/routers/waits" "github.com/nyaruka/goflow/flows/routers/waits/hints" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/config" "github.com/nyaruka/goflow/flows" diff --git a/mailroom.go b/mailroom.go index cfda2408f..4ff2abca1 100644 --- a/mailroom.go +++ b/mailroom.go @@ -9,9 +9,9 @@ import ( "sync" "time" + "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/queue" - "github.com/nyaruka/mailroom/utils/storage" "github.com/nyaruka/mailroom/web" "github.com/gomodule/redigo/redis" @@ -159,9 +159,21 @@ func (mr *Mailroom) Start() error { } // create our storage (S3 or file system) - mr.Storage, err = storage.New(mr.Config) - if err != nil { - return err + if mr.Config.AWSAccessKeyID != "" { + s3Client, err := storage.NewS3Client(&storage.S3Options{ + AWSAccessKeyID: mr.Config.AWSAccessKeyID, + AWSSecretAccessKey: mr.Config.AWSSecretAccessKey, + Endpoint: mr.Config.S3Endpoint, + Region: mr.Config.S3Region, + DisableSSL: mr.Config.S3DisableSSL, + ForcePathStyle: mr.Config.S3ForcePathStyle, + }) + if err != nil { + return err + } + mr.Storage = storage.NewS3(s3Client, mr.Config.S3MediaBucket) + } else { + mr.Storage = storage.NewFS("_storage") } // test our storage diff --git a/mailroom_test.dump b/mailroom_test.dump index 8e1f40120..6ba4a08e9 100644 Binary files a/mailroom_test.dump and b/mailroom_test.dump differ diff --git a/models/airtime.go b/models/airtime.go index 47ca1629e..f73c7c124 100644 --- a/models/airtime.go +++ b/models/airtime.go @@ -86,7 +86,7 @@ func InsertAirtimeTransfers(ctx context.Context, tx Queryer, transfers []*Airtim ts[i] = &transfers[i].t } - return BulkSQL(ctx, "inserted airtime transfers", tx, insertAirtimeTransfersSQL, ts) + return BulkQuery(ctx, "inserted airtime transfers", tx, insertAirtimeTransfersSQL, ts) } // MarshalJSON marshals into JSON. 0 values will become null diff --git a/models/campaigns.go b/models/campaigns.go index c043b1b7f..e51dd7caa 100644 --- a/models/campaigns.go +++ b/models/campaigns.go @@ -2,15 +2,18 @@ package models import ( "context" + "database/sql" "encoding/json" "time" - "github.com/jmoiron/sqlx" - "github.com/lib/pq" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/nyaruka/null" + + "github.com/jmoiron/sqlx" + "github.com/lib/pq" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -266,7 +269,7 @@ func loadCampaigns(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]*Campai campaigns := make([]*Campaign, 0, 2) for rows.Next() { campaign := &Campaign{} - err := readJSONRow(rows, &campaign.c) + err := dbutil.ReadJSONRow(rows, &campaign.c) if err != nil { return nil, errors.Wrapf(err, "error unmarshalling campaign") } @@ -337,7 +340,7 @@ func MarkEventsFired(ctx context.Context, tx Queryer, fires []*EventFire, fired updates = append(updates, f) } - return BulkSQL(ctx, "mark events fired", tx, markEventsFired, updates) + return BulkQuery(ctx, "mark events fired", tx, markEventsFired, updates) } const markEventsFired = ` @@ -455,7 +458,7 @@ func DeleteUnfiredEventFires(ctx context.Context, tx Queryer, removes []*FireDel for i := range removes { is[i] = removes[i] } - return BulkSQL(ctx, "removing campaign event fires", tx, removeUnfiredFiresSQL, is) + return BulkQuery(ctx, "removing campaign event fires", tx, removeUnfiredFiresSQL, is) } const removeUnfiredFiresSQL = ` @@ -490,6 +493,18 @@ func DeleteUnfiredContactEvents(ctx context.Context, tx Queryer, contactID Conta return nil } +const insertEventFiresSQL = ` +INSERT INTO campaigns_eventfire(contact_id, event_id, scheduled) + VALUES(:contact_id, :event_id, :scheduled) +ON CONFLICT DO NOTHING +` + +type FireAdd struct { + ContactID ContactID `db:"contact_id"` + EventID CampaignEventID `db:"event_id"` + Scheduled time.Time `db:"scheduled"` +} + // AddEventFires adds the passed in event fires to our db func AddEventFires(ctx context.Context, tx Queryer, adds []*FireAdd) error { if len(adds) == 0 { @@ -501,20 +516,7 @@ func AddEventFires(ctx context.Context, tx Queryer, adds []*FireAdd) error { for i := range adds { is[i] = adds[i] } - return BulkSQL(ctx, "adding campaign event fires", tx, insertEventFiresSQL, is) -} - -const insertEventFiresSQL = ` - INSERT INTO - campaigns_eventfire - (contact_id, event_id, scheduled) - VALUES(:contact_id, :event_id, :scheduled) -` - -type FireAdd struct { - ContactID ContactID `db:"contact_id"` - EventID CampaignEventID `db:"event_id"` - Scheduled time.Time `db:"scheduled"` + return BulkQueryBatches(ctx, "adding campaign event fires", tx, insertEventFiresSQL, 1000, is) } // DeleteUnfiredEventsForGroupRemoval deletes any unfired events for all campaigns that are @@ -586,3 +588,127 @@ func AddCampaignEventsForGroupAddition(ctx context.Context, tx Queryer, org *Org // add all our new event fires return AddEventFires(ctx, tx, fas) } + +// ScheduleCampaignEvent calculates event fires for a new campaign event +func ScheduleCampaignEvent(ctx context.Context, db *sqlx.DB, orgID OrgID, eventID CampaignEventID) error { + oa, err := GetOrgAssetsWithRefresh(ctx, db, orgID, RefreshCampaigns) + if err != nil { + return errors.Wrapf(err, "unable to load org: %d", orgID) + } + + event := oa.CampaignEventByID(eventID) + if event == nil { + return errors.Errorf("can't find campaign event with id %d", eventID) + } + + field := oa.FieldByKey(event.RelativeToKey()) + if field == nil { + return errors.Errorf("can't find field with key %s", event.RelativeToKey()) + } + + eligible, err := campaignEventEligibleContacts(ctx, db, event.campaign.GroupID(), field) + if err != nil { + return errors.Wrapf(err, "unable to calculate eligible contacts for event %d", eventID) + } + + fas := make([]*FireAdd, 0, len(eligible)) + tz := oa.Env().Timezone() + + for _, el := range eligible { + if el.RelToValue != nil { + start := *el.RelToValue + + // calculate next fire for this contact + scheduled, err := event.ScheduleForTime(tz, time.Now(), start) + if err != nil { + return errors.Wrapf(err, "error calculating offset for start: %s and event: %d", start, eventID) + } + + if scheduled != nil { + fas = append(fas, &FireAdd{ContactID: el.ContactID, EventID: eventID, Scheduled: *scheduled}) + } + } + } + + // add all our new event fires + return AddEventFires(ctx, db, fas) +} + +type eligibleContact struct { + ContactID ContactID `db:"contact_id"` + RelToValue *time.Time `db:"rel_to_value"` +} + +const eligibleContactsForCreatedOnSQL = ` +SELECT + c.id AS contact_id, + c.created_on AS rel_to_value +FROM + contacts_contact c +INNER JOIN + contacts_contactgroup_contacts gc ON gc.contact_id = c.id +WHERE + gc.contactgroup_id = $1 AND c.is_active = TRUE +` + +const eligibleContactsForLastSeenOnSQL = ` +SELECT + c.id AS contact_id, + c.last_seen_on AS rel_to_value +FROM + contacts_contact c +INNER JOIN + contacts_contactgroup_contacts gc ON gc.contact_id = c.id +WHERE + gc.contactgroup_id = $1 AND c.is_active = TRUE AND c.last_seen_on IS NOT NULL +` + +const eligibleContactsForFieldSQL = ` +SELECT + c.id AS contact_id, + (c.fields->$2->>'datetime')::timestamptz AS rel_to_value +FROM + contacts_contact c +INNER JOIN + contacts_contactgroup_contacts gc ON gc.contact_id = c.id +WHERE + gc.contactgroup_id = $1 AND c.is_active = TRUE AND ARRAY[$2]::text[] <@ (extract_jsonb_keys(c.fields)) IS NOT NULL +` + +func campaignEventEligibleContacts(ctx context.Context, db *sqlx.DB, groupID GroupID, field *Field) ([]*eligibleContact, error) { + var query string + var params []interface{} + + switch field.Key() { + case CreatedOnKey: + query = eligibleContactsForCreatedOnSQL + params = []interface{}{groupID} + case LastSeenOnKey: + query = eligibleContactsForLastSeenOnSQL + params = []interface{}{groupID} + default: + query = eligibleContactsForFieldSQL + params = []interface{}{groupID, field.UUID()} + } + + rows, err := db.QueryxContext(ctx, query, params...) + if err != nil && err != sql.ErrNoRows { + return nil, errors.Wrapf(err, "error querying for eligible contacts") + } + defer rows.Close() + + contacts := make([]*eligibleContact, 0, 100) + + for rows.Next() { + contact := &eligibleContact{} + + err := rows.StructScan(&contact) + if err != nil { + return nil, errors.Wrapf(err, "error scanning eligible contact result") + } + + contacts = append(contacts, contact) + } + + return contacts, nil +} diff --git a/models/campaigns_test.go b/models/campaigns_test.go index a750190c3..309c62eba 100644 --- a/models/campaigns_test.go +++ b/models/campaigns_test.go @@ -1,19 +1,25 @@ -package models +package models_test import ( + "encoding/json" + "fmt" "testing" "time" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/testsuite" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) -func testCampaignSchedule(t *testing.T) { +func TestCampaignSchedule(t *testing.T) { eastern, _ := time.LoadLocation("US/Eastern") nilDate := time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC) tcs := []struct { Offset int - Unit OffsetUnit + Unit models.OffsetUnit DeliveryHour int Timezone *time.Location @@ -25,30 +31,30 @@ func testCampaignSchedule(t *testing.T) { Delta time.Duration }{ // this crosses a DST boundary, so two days is really 49 hours (fall back) - {2, OffsetDay, NilDeliveryHour, eastern, time.Now(), time.Date(2029, 11, 3, 0, 30, 0, 0, eastern), + {2, models.OffsetDay, models.NilDeliveryHour, eastern, time.Now(), time.Date(2029, 11, 3, 0, 30, 0, 0, eastern), false, time.Date(2029, 11, 5, 0, 30, 0, 0, eastern), time.Hour * 49}, // this also crosses a boundary but in the other direction - {2, OffsetDay, NilDeliveryHour, eastern, time.Now(), time.Date(2029, 3, 10, 2, 30, 0, 0, eastern), + {2, models.OffsetDay, models.NilDeliveryHour, eastern, time.Now(), time.Date(2029, 3, 10, 2, 30, 0, 0, eastern), false, time.Date(2029, 3, 12, 2, 30, 0, 0, eastern), time.Hour * 47}, // this event is in the past, no schedule - {2, OffsetDay, NilDeliveryHour, eastern, time.Date(2018, 10, 31, 0, 0, 0, 0, eastern), time.Date(2018, 10, 15, 0, 0, 0, 0, eastern), + {2, models.OffsetDay, models.NilDeliveryHour, eastern, time.Date(2018, 10, 31, 0, 0, 0, 0, eastern), time.Date(2018, 10, 15, 0, 0, 0, 0, eastern), false, nilDate, 0}, - {2, OffsetMinute, NilDeliveryHour, eastern, time.Now(), time.Date(2029, 1, 1, 2, 58, 0, 0, eastern), + {2, models.OffsetMinute, models.NilDeliveryHour, eastern, time.Now(), time.Date(2029, 1, 1, 2, 58, 0, 0, eastern), false, time.Date(2029, 1, 1, 3, 0, 0, 0, eastern), time.Minute * 2}, - {2, OffsetMinute, NilDeliveryHour, eastern, time.Now(), time.Date(2029, 1, 1, 2, 57, 32, 0, eastern), + {2, models.OffsetMinute, models.NilDeliveryHour, eastern, time.Now(), time.Date(2029, 1, 1, 2, 57, 32, 0, eastern), false, time.Date(2029, 1, 1, 3, 0, 0, 0, eastern), time.Minute*2 + time.Second*28}, - {-2, OffsetHour, NilDeliveryHour, eastern, time.Now(), time.Date(2029, 1, 2, 1, 58, 0, 0, eastern), + {-2, models.OffsetHour, models.NilDeliveryHour, eastern, time.Now(), time.Date(2029, 1, 2, 1, 58, 0, 0, eastern), false, time.Date(2029, 1, 1, 23, 58, 0, 0, eastern), time.Hour * -2}, - {2, OffsetWeek, NilDeliveryHour, eastern, time.Now(), time.Date(2029, 1, 20, 1, 58, 0, 0, eastern), + {2, models.OffsetWeek, models.NilDeliveryHour, eastern, time.Now(), time.Date(2029, 1, 20, 1, 58, 0, 0, eastern), false, time.Date(2029, 2, 3, 1, 58, 0, 0, eastern), time.Hour * 24 * 14}, - {2, OffsetWeek, 14, eastern, time.Now(), time.Date(2029, 1, 20, 1, 58, 0, 0, eastern), + {2, models.OffsetWeek, 14, eastern, time.Now(), time.Date(2029, 1, 20, 1, 58, 0, 0, eastern), false, time.Date(2029, 2, 3, 14, 0, 0, 0, eastern), time.Hour*24*14 + 13*time.Hour - 58*time.Minute}, {2, "L", 14, eastern, time.Now(), time.Date(2029, 1, 20, 1, 58, 0, 0, eastern), @@ -56,10 +62,10 @@ func testCampaignSchedule(t *testing.T) { } for i, tc := range tcs { - evt := &CampaignEvent{} - evt.e.Offset = tc.Offset - evt.e.Unit = tc.Unit - evt.e.DeliveryHour = tc.DeliveryHour + evtJSON := fmt.Sprintf(`{"offset": %d, "unit": "%s", "delivery_hour": %d}`, tc.Offset, tc.Unit, tc.DeliveryHour) + evt := &models.CampaignEvent{} + err := json.Unmarshal([]byte(evtJSON), evt) + require.NoError(t, err) scheduled, err := evt.ScheduleForTime(tc.Timezone, tc.Now, tc.Start) @@ -73,3 +79,37 @@ func testCampaignSchedule(t *testing.T) { } } } + +func TestAddEventFires(t *testing.T) { + ctx := testsuite.CTX() + db := testsuite.DB() + testsuite.Reset() + + scheduled1 := time.Date(2020, 9, 8, 14, 38, 30, 123456789, time.UTC) + + err := models.AddEventFires(ctx, db, []*models.FireAdd{ + {ContactID: models.CathyID, EventID: models.RemindersEvent1ID, Scheduled: scheduled1}, + {ContactID: models.BobID, EventID: models.RemindersEvent1ID, Scheduled: scheduled1}, + {ContactID: models.BobID, EventID: models.RemindersEvent2ID, Scheduled: scheduled1}, + }) + require.NoError(t, err) + + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM campaigns_eventfire`, nil, 3) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM campaigns_eventfire WHERE contact_id = $1 AND event_id = $2`, []interface{}{models.CathyID, models.RemindersEvent1ID}, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM campaigns_eventfire WHERE contact_id = $1 AND event_id = $2`, []interface{}{models.BobID, models.RemindersEvent1ID}, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM campaigns_eventfire WHERE contact_id = $1 AND event_id = $2`, []interface{}{models.BobID, models.RemindersEvent2ID}, 1) + + db.MustExec(`UPDATE campaigns_eventfire SET fired = NOW() WHERE contact_id = $1`, models.CathyID) + + scheduled2 := time.Date(2020, 9, 8, 14, 38, 30, 123456789, time.UTC) + + err = models.AddEventFires(ctx, db, []*models.FireAdd{ + {ContactID: models.CathyID, EventID: models.RemindersEvent1ID, Scheduled: scheduled2}, // fine because previous one now has non-null fired + {ContactID: models.BobID, EventID: models.RemindersEvent1ID, Scheduled: scheduled2}, // won't be added due to conflict + }) + require.NoError(t, err) + + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM campaigns_eventfire`, nil, 4) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM campaigns_eventfire WHERE contact_id = $1 AND event_id = $2`, []interface{}{models.CathyID, models.RemindersEvent1ID}, 2) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM campaigns_eventfire WHERE contact_id = $1`, []interface{}{models.BobID}, 2) +} diff --git a/models/channel_event.go b/models/channel_event.go index 2997c49c8..45fc2a3d2 100644 --- a/models/channel_event.go +++ b/models/channel_event.go @@ -90,7 +90,7 @@ RETURNING // Insert inserts this channel event to our DB. The ID of the channel event will be // set if no error is returned func (e *ChannelEvent) Insert(ctx context.Context, db *sqlx.DB) error { - return BulkSQL(ctx, "insert channel event", db, insertChannelEventSQL, []interface{}{&e.e}) + return BulkQuery(ctx, "insert channel event", db, insertChannelEventSQL, []interface{}{&e.e}) } // NewChannelEvent creates a new channel event for the passed in parameters, returning it diff --git a/models/channel_logs.go b/models/channel_logs.go index 7f8aa931c..a8899811a 100644 --- a/models/channel_logs.go +++ b/models/channel_logs.go @@ -2,6 +2,8 @@ package models import ( "context" + "fmt" + "net/http" "time" "github.com/jmoiron/sqlx" @@ -52,9 +54,12 @@ func NewChannelLog(trace *httpx.Trace, isError bool, desc string, channel *Chann statusCode = trace.Response.StatusCode } + // if URL was rewritten (by nginx for example), we want to log the original request + url := originalURL(trace.Request) + l.Description = desc l.IsError = isError - l.URL = trace.Request.URL.String() + l.URL = url l.Method = trace.Request.Method l.Request = string(trace.RequestTrace) l.Response = trace.ResponseTraceUTF8("...") @@ -68,6 +73,14 @@ func NewChannelLog(trace *httpx.Trace, isError bool, desc string, channel *Chann return log } +func originalURL(r *http.Request) string { + proxyPath := r.Header.Get("X-Forwarded-Path") + if proxyPath != "" { + return fmt.Sprintf("https://%s%s", r.Host, proxyPath) + } + return r.URL.String() +} + // InsertChannelLogs writes the given channel logs to the db func InsertChannelLogs(ctx context.Context, db *sqlx.DB, logs []*ChannelLog) error { ls := make([]interface{}, len(logs)) @@ -75,7 +88,7 @@ func InsertChannelLogs(ctx context.Context, db *sqlx.DB, logs []*ChannelLog) err ls[i] = &logs[i].l } - err := BulkSQL(ctx, "insert channel log", db, insertChannelLogSQL, ls) + err := BulkQuery(ctx, "insert channel log", db, insertChannelLogSQL, ls) if err != nil { return errors.Wrapf(err, "error inserting channel log") } @@ -105,7 +118,7 @@ func InsertChannelLog(ctx context.Context, db *sqlx.DB, l.ConnectionID = conn.ID() } - err := BulkSQL(ctx, "insert channel log", db, insertChannelLogSQL, []interface{}{l}) + err := BulkQuery(ctx, "insert channel log", db, insertChannelLogSQL, []interface{}{l}) if err != nil { return nil, errors.Wrapf(err, "error inserting channel log") } diff --git a/models/channel_logs_test.go b/models/channel_logs_test.go index d444c64a5..a75b43e36 100644 --- a/models/channel_logs_test.go +++ b/models/channel_logs_test.go @@ -21,6 +21,7 @@ func TestChannelLogs(t *testing.T) { httpx.SetRequestor(httpx.NewMockRequestor(map[string][]httpx.MockResponse{ "http://rapidpro.io": {httpx.NewMockResponse(200, nil, "OK")}, "http://rapidpro.io/bad": {httpx.NewMockResponse(400, nil, "Oops")}, + "http://rapidpro.io/new": {httpx.NewMockResponse(200, nil, "OK")}, })) oa, err := models.GetOrgAssets(ctx, db, models.Org1) @@ -36,10 +37,15 @@ func TestChannelLogs(t *testing.T) { trace2, err := httpx.DoTrace(http.DefaultClient, req2, nil, nil, -1) log2 := models.NewChannelLog(trace2, true, "test request", channel, nil) - err = models.InsertChannelLogs(ctx, db, []*models.ChannelLog{log1, log2}) + req3, _ := httpx.NewRequest("GET", "http://rapidpro.io/new", nil, map[string]string{"X-Forwarded-Path": "/old"}) + trace3, err := httpx.DoTrace(http.DefaultClient, req3, nil, nil, -1) + log3 := models.NewChannelLog(trace3, false, "test request", channel, nil) + + err = models.InsertChannelLogs(ctx, db, []*models.ChannelLog{log1, log2, log3}) require.NoError(t, err) - testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM channels_channellog`, nil, 2) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM channels_channellog`, nil, 3) testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM channels_channellog WHERE url = 'http://rapidpro.io' AND is_error = FALSE AND channel_id = $1`, []interface{}{channel.ID()}, 1) testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM channels_channellog WHERE url = 'http://rapidpro.io/bad' AND is_error = TRUE AND channel_id = $1`, []interface{}{channel.ID()}, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM channels_channellog WHERE url = 'https://rapidpro.io/old' AND is_error = FALSE AND channel_id = $1`, []interface{}{channel.ID()}, 1) } diff --git a/models/channels.go b/models/channels.go index 70bd86108..3809662ab 100644 --- a/models/channels.go +++ b/models/channels.go @@ -9,6 +9,7 @@ import ( "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/envs" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/nyaruka/null" "github.com/jmoiron/sqlx" @@ -132,7 +133,7 @@ func loadChannels(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets.C channels := make([]assets.Channel, 0, 2) for rows.Next() { channel := &Channel{} - err := readJSONRow(rows, &channel.c) + err := dbutil.ReadJSONRow(rows, &channel.c) if err != nil { return nil, errors.Wrapf(err, "error unmarshalling channel") } diff --git a/models/classifiers.go b/models/classifiers.go index 1c98d7882..b90d56e64 100644 --- a/models/classifiers.go +++ b/models/classifiers.go @@ -5,14 +5,16 @@ import ( "database/sql/driver" "time" - "github.com/jmoiron/sqlx" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/services/classification/bothub" "github.com/nyaruka/goflow/services/classification/luis" "github.com/nyaruka/goflow/services/classification/wit" "github.com/nyaruka/mailroom/goflow" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/nyaruka/null" + + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -133,7 +135,7 @@ func loadClassifiers(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]asset classifiers := make([]assets.Classifier, 0, 2) for rows.Next() { classifier := &Classifier{} - err := readJSONRow(rows, &classifier.c) + err := dbutil.ReadJSONRow(rows, &classifier.c) if err != nil { return nil, errors.Wrapf(err, "error unmarshalling classifier") } diff --git a/models/contacts.go b/models/contacts.go index cb0b81e75..fcb3151f1 100644 --- a/models/contacts.go +++ b/models/contacts.go @@ -10,14 +10,16 @@ import ( "strconv" "time" + "github.com/nyaruka/gocommon/dates" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/contactql" "github.com/nyaruka/goflow/contactql/es" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/excellent/types" "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/nyaruka/null" "github.com/jmoiron/sqlx" @@ -44,6 +46,9 @@ const ( // ContactStatus is the type for contact statuses type ContactStatus string +// NilContactStatus is an unset contact status +const NilContactStatus ContactStatus = "" + // possible contact statuses const ( ContactStatusActive = "A" @@ -91,7 +96,7 @@ func LoadContacts(ctx context.Context, db Queryer, org *OrgAssets, ids []Contact contacts := make([]*Contact, 0, len(ids)) for rows.Next() { e := &contactEnvelope{} - err := readJSONRow(rows, e) + err := dbutil.ReadJSONRow(rows, e) if err != nil { return nil, errors.Wrap(err, "error scanning contact json") } @@ -208,7 +213,7 @@ func ContactIDsFromReferences(ctx context.Context, tx Queryer, org *OrgAssets, r } // BuildElasticQuery turns the passed in contact ql query into an elastic query -func BuildElasticQuery(org *OrgAssets, group assets.GroupUUID, query *contactql.ContactQuery) elastic.Query { +func BuildElasticQuery(org *OrgAssets, group assets.GroupUUID, status ContactStatus, excludeIDs []ContactID, query *contactql.ContactQuery) elastic.Query { // filter by org and active contacts eq := elastic.NewBoolQuery().Must( elastic.NewTermQuery("org_id", org.OrgID()), @@ -220,6 +225,20 @@ func BuildElasticQuery(org *OrgAssets, group assets.GroupUUID, query *contactql. eq = eq.Must(elastic.NewTermQuery("groups", group)) } + // our status is present + if status != NilContactStatus { + eq = eq.Must(elastic.NewTermQuery("status", status)) + } + + // exclude ids if present + if len(excludeIDs) > 0 { + ids := make([]string, len(excludeIDs)) + for i := range excludeIDs { + ids[i] = fmt.Sprintf("%d", excludeIDs[i]) + } + eq = eq.MustNot(elastic.NewIdsQuery("_doc").Ids(ids...)) + } + // and by our query if present if query != nil { q := es.ToElasticQuery(org.Env(), query) @@ -230,7 +249,7 @@ func BuildElasticQuery(org *OrgAssets, group assets.GroupUUID, query *contactql. } // ContactIDsForQueryPage returns the ids of the contacts for the passed in query page -func ContactIDsForQueryPage(ctx context.Context, client *elastic.Client, org *OrgAssets, group assets.GroupUUID, query string, sort string, offset int, pageSize int) (*contactql.ContactQuery, []ContactID, int64, error) { +func ContactIDsForQueryPage(ctx context.Context, client *elastic.Client, org *OrgAssets, group assets.GroupUUID, excludeIDs []ContactID, query string, sort string, offset int, pageSize int) (*contactql.ContactQuery, []ContactID, int64, error) { env := org.Env() start := time.Now() var parsed *contactql.ContactQuery @@ -247,7 +266,7 @@ func ContactIDsForQueryPage(ctx context.Context, client *elastic.Client, org *Or } } - eq := BuildElasticQuery(org, group, parsed) + eq := BuildElasticQuery(org, group, NilContactStatus, excludeIDs, parsed) fieldSort, err := es.ToElasticFieldSort(sort, org.SessionAssets()) if err != nil { @@ -305,14 +324,7 @@ func ContactIDsForQuery(ctx context.Context, client *elastic.Client, org *OrgAss return nil, errors.Wrapf(err, "error parsing query: %s", query) } - eq := BuildElasticQuery(org, "", parsed) - - // only include unblocked and unstopped contacts - eq = elastic.NewBoolQuery().Must( - eq, - elastic.NewTermQuery("is_blocked", false), - elastic.NewTermQuery("is_stopped", false), - ) + eq := BuildElasticQuery(org, "", ContactStatusActive, nil, parsed) ids := make([]ContactID, 0, 100) @@ -599,7 +611,7 @@ func ContactIDsFromURNs(ctx context.Context, db *sqlx.DB, org *OrgAssets, us []u // create the contacts that are missing for _, u := range us { if urnMap[u] == NilContactID { - id, err := CreateContact(ctx, db, org, u) + contact, _, err := CreateContact(ctx, db, org, NilUserID, "", envs.NilLanguage, []urns.URN{u}) if err != nil { return nil, errors.Wrapf(err, "error while creating contact") } @@ -608,7 +620,7 @@ func ContactIDsFromURNs(ctx context.Context, db *sqlx.DB, org *OrgAssets, us []u if !found { return nil, errors.Wrapf(err, "unable to find original URN from identity") } - urnMap[original] = ContactID(id) + urnMap[original] = contact.ID() } } } @@ -618,8 +630,82 @@ func ContactIDsFromURNs(ctx context.Context, db *sqlx.DB, org *OrgAssets, us []u } // CreateContact creates a new contact for the passed in org with the passed in URNs -func CreateContact(ctx context.Context, db *sqlx.DB, org *OrgAssets, urn urns.URN) (ContactID, error) { - // we have a URN, first try to look up the URN +func CreateContact(ctx context.Context, db *sqlx.DB, org *OrgAssets, userID UserID, name string, language envs.Language, urnz []urns.URN) (*Contact, *flows.Contact, error) { + contactID, err := insertContactAndURNs(ctx, db, org, userID, name, language, urnz) + if err != nil { + if dbutil.IsUniqueViolation(err) { + return nil, nil, errors.New("URNs in use by other contacts") + } + return nil, nil, err + } + + // load a full contact so that we can calculate dynamic groups + contacts, err := LoadContacts(ctx, db, org, []ContactID{contactID}) + if err != nil { + return nil, nil, errors.Wrapf(err, "error loading new contact") + } + contact := contacts[0] + + flowContact, err := contact.FlowContact(org) + if err != nil { + return nil, nil, errors.Wrapf(err, "error creating flow contact") + } + + err = CalculateDynamicGroups(ctx, db, org, flowContact) + if err != nil { + return nil, nil, errors.Wrapf(err, "error calculating dynamic groups") + } + + return contact, flowContact, nil +} + +// GetOrCreateContact creates a new contact for the passed in org with the passed in URNs +func GetOrCreateContact(ctx context.Context, db *sqlx.DB, org *OrgAssets, urn urns.URN) (*Contact, *flows.Contact, error) { + created := true + + contactID, err := insertContactAndURNs(ctx, db, org, UserID(1), "", envs.NilLanguage, []urns.URN{urn}) + if err != nil { + if dbutil.IsUniqueViolation(err) { + // if this was a duplicate URN, we should be able to fetch this contact instead + err := db.GetContext(ctx, &contactID, `SELECT contact_id FROM contacts_contacturn WHERE org_id = $1 AND identity = $2`, org.OrgID(), urn.Identity()) + if err != nil { + return nil, nil, errors.Wrapf(err, "unable to load contact") + } + created = false + } else { + return nil, nil, err + } + } + + // load a full contact so that we can calculate dynamic groups + contacts, err := LoadContacts(ctx, db, org, []ContactID{contactID}) + if err != nil { + return nil, nil, errors.Wrapf(err, "error loading new contact") + } + contact := contacts[0] + + flowContact, err := contact.FlowContact(org) + if err != nil { + return nil, nil, errors.Wrapf(err, "error creating flow contact") + } + + // calculate dynamic groups if contact was created + if created { + err := CalculateDynamicGroups(ctx, db, org, flowContact) + if err != nil { + return nil, nil, errors.Wrapf(err, "error calculating dynamic groups") + } + } + + return contact, flowContact, nil +} + +// tries to create a new contact for the passed in org with the passed in URNs +func insertContactAndURNs(ctx context.Context, db *sqlx.DB, org *OrgAssets, userID UserID, name string, language envs.Language, urnz []urns.URN) (ContactID, error) { + if userID == NilUserID { + userID = UserID(1) + } + tx, err := db.BeginTxx(ctx, nil) if err != nil { return NilContactID, errors.Wrapf(err, "unable to start transaction") @@ -628,13 +714,12 @@ func CreateContact(ctx context.Context, db *sqlx.DB, org *OrgAssets, urn urns.UR // first insert our contact var contactID ContactID err = tx.GetContext(ctx, &contactID, - `INSERT INTO - contacts_contact - (org_id, is_active, status, uuid, created_on, modified_on, created_by_id, modified_by_id, name) + `INSERT INTO contacts_contact + (org_id, is_active, status, uuid, name, language, created_on, modified_on, created_by_id, modified_by_id) VALUES - ($1, TRUE, 'A', $2, NOW(), NOW(), 1, 1, '') + ($1, TRUE, 'A', $2, $3, $4, $5, $5, $6, $6) RETURNING id`, - org.OrgID(), uuids.New(), + org.OrgID(), uuids.New(), null.String(name), null.String(string(language)), dates.Now(), userID, ) if err != nil { @@ -642,24 +727,23 @@ func CreateContact(ctx context.Context, db *sqlx.DB, org *OrgAssets, urn urns.UR return NilContactID, errors.Wrapf(err, "error inserting new contact") } - // handler for when we insert the URN or commit, we try to look the contact up instead - handleURNError := func(err error) (ContactID, error) { - if pqErr, ok := err.(*pq.Error); ok { - // if this was a duplicate URN, we should be able to select this contact instead - if pqErr.Code.Name() == "unique_violation" { - ids, err := ContactIDsFromURNs(ctx, db, org, []urns.URN{urn}) - if err != nil || len(ids) == 0 { - return NilContactID, errors.Wrapf(err, "unable to load contact for urn: %s", urn) - } - return ids[urn], nil - } + var urnsToAttach []URNID + + // now try to insert the URNs + for _, urn := range urnz { + // look for a URN with this identity that already exists but doesn't have a contact so could be attached + var orphanURNID URNID + err = tx.GetContext(ctx, &orphanURNID, `SELECT id FROM contacts_contacturn WHERE org_id = $1 AND identity = $2 AND contact_id IS NULL`, org.OrgID(), urn.Identity()) + if err != nil && err != sql.ErrNoRows { + return NilContactID, err + } + if orphanURNID > 0 { + urnsToAttach = append(urnsToAttach, orphanURNID) + continue } - return NilContactID, errors.Wrapf(err, "error creating new contact") - } - // now try to insert our URN if we have one - if urn != urns.NilURN { - _, err := tx.Exec( + _, err := tx.ExecContext( + ctx, `INSERT INTO contacts_contacturn (org_id, identity, path, scheme, display, auth, priority, channel_id, contact_id) @@ -670,36 +754,24 @@ func CreateContact(ctx context.Context, db *sqlx.DB, org *OrgAssets, urn urns.UR if err != nil { tx.Rollback() - return handleURNError(err) + return NilContactID, err } } - // load a full contact so that we can calculate dynamic groups - contacts, err := LoadContacts(ctx, tx, org, []ContactID{contactID}) - if err != nil { - tx.Rollback() - return NilContactID, errors.Wrapf(err, "error loading new contact") - } - - flowContact, err := contacts[0].FlowContact(org) - if err != nil { - tx.Rollback() - return NilContactID, errors.Wrapf(err, "error creating flow contact") - } - - // now calculate dynamic groups - err = CalculateDynamicGroups(ctx, tx, org, flowContact) - if err != nil { - tx.Rollback() - return NilContactID, errors.Wrapf(err, "error calculating dynamic groups") + // attach URNs + if len(urnsToAttach) > 0 { + _, err := tx.ExecContext(ctx, `UPDATE contacts_contacturn SET contact_id = $3 WHERE org_id = $1 AND id = ANY($2)`, org.OrgID(), pq.Array(urnsToAttach), contactID) + if err != nil { + tx.Rollback() + return NilContactID, errors.Wrapf(err, "error attaching existing URNs to new contact") + } } // try to commit err = tx.Commit() - if err != nil { tx.Rollback() - return handleURNError(err) + return NilContactID, err } return contactID, nil @@ -722,7 +794,7 @@ func URNForURN(ctx context.Context, tx Queryer, org *OrgAssets, u urns.URN) (urn return urns.NilURN, errors.Errorf("no urn with identity: %s", u.Identity()) } - err = readJSONRow(rows, urn) + err = dbutil.ReadJSONRow(rows, urn) if err != nil { return urns.NilURN, errors.Wrapf(err, "error loading contact urn") } @@ -783,7 +855,7 @@ func URNForID(ctx context.Context, tx Queryer, org *OrgAssets, urnID URNID) (urn return urns.NilURN, errors.Errorf("no urn with id: %d", urnID) } - err = readJSONRow(rows, urn) + err = dbutil.ReadJSONRow(rows, urn) if err != nil { return urns.NilURN, errors.Wrapf(err, "error loading contact urn") } @@ -794,7 +866,7 @@ func URNForID(ctx context.Context, tx Queryer, org *OrgAssets, urnID URNID) (urn // CalculateDynamicGroups recalculates all the dynamic groups for the passed in contact, recalculating // campaigns as necessary based on those group changes. func CalculateDynamicGroups(ctx context.Context, tx Queryer, org *OrgAssets, contact *flows.Contact) error { - added, removed, errs := contact.ReevaluateDynamicGroups(org.Env()) + added, removed, errs := contact.ReevaluateQueryBasedGroups(org.Env()) if len(errs) > 0 { return errors.Wrapf(errs[0], "error calculating dynamic groups") } @@ -1156,7 +1228,7 @@ func UpdateContactURNs(ctx context.Context, tx Queryer, org *OrgAssets, changes } // first update existing URNs - err := BulkSQL(ctx, "updating contact urns", tx, updateContactURNsSQL, updates) + err := BulkQuery(ctx, "updating contact urns", tx, updateContactURNsSQL, updates) if err != nil { return errors.Wrapf(err, "error updating urns") } @@ -1362,7 +1434,7 @@ func UpdateContactStatus(ctx context.Context, tx Queryer, changes []*ContactStat } // do our status update - err = BulkSQL(ctx, "updating contact statuses", tx, updateContactStatusSQL, statusUpdates) + err = BulkQuery(ctx, "updating contact statuses", tx, updateContactStatusSQL, statusUpdates) if err != nil { return errors.Wrapf(err, "error updating contact statuses") } diff --git a/models/contacts_test.go b/models/contacts_test.go index c82a6c53c..04d2f9048 100644 --- a/models/contacts_test.go +++ b/models/contacts_test.go @@ -6,7 +6,9 @@ import ( "time" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/goflow/test" "github.com/nyaruka/mailroom/testsuite" "github.com/olivere/elastic" @@ -15,7 +17,7 @@ import ( "github.com/stretchr/testify/require" ) -func TestElasticContacts(t *testing.T) { +func TestContactIDsForQueryPage(t *testing.T) { testsuite.Reset() ctx := testsuite.CTX() db := testsuite.DB() @@ -28,44 +30,289 @@ func TestElasticContacts(t *testing.T) { elastic.SetHealthcheck(false), elastic.SetSniff(false), ) - assert.NoError(t, err) + require.NoError(t, err) - org, err := GetOrgAssets(ctx, db, 1) - assert.NoError(t, err) + oa, err := GetOrgAssets(ctx, db, 1) + require.NoError(t, err) tcs := []struct { - Query string - Request string - Response string - Contacts []ContactID - Error bool + Group assets.GroupUUID + ExcludeIDs []ContactID + Query string + Sort string + ExpectedESRequest string + MockedESResponse string + ExpectedContacts []ContactID + ExpectedTotal int64 + ExpectedError string }{ { + Group: AllContactsGroupUUID, Query: "george", - Request: `{ - "_source":false, - "query":{ - "bool":{ - "must":[ - { "bool":{ - "must":[ - {"term":{"org_id":1}}, - {"term":{"is_active":true}}, - {"match":{"name":{"query":"george"}}} + ExpectedESRequest: `{ + "_source": false, + "from": 0, + "query": { + "bool": { + "must": [ + { + "term": { + "org_id": 1 + } + }, + { + "term": { + "is_active": true + } + }, + { + "term": { + "groups": "d1ee73f0-bdb5-47ce-99dd-0c95d4ebf008" + } + }, + { + "match": { + "name": { + "query": "george" + } + } + } + ] + } + }, + "size": 50, + "sort": [ + { + "id": { + "order": "desc" + } + } + ] + }`, + MockedESResponse: fmt.Sprintf(`{ + "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", + "took": 2, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": null, + "hits": [ + { + "_index": "contacts", + "_type": "_doc", + "_id": "%d", + "_score": null, + "_routing": "1", + "sort": [ + 15124352 + ] + } + ] + } + }`, GeorgeID), + ExpectedContacts: []ContactID{GeorgeID}, + ExpectedTotal: 1, + }, + { + Group: BlockedContactsGroupUUID, + ExcludeIDs: []ContactID{BobID, CathyID}, + Query: "age > 32", + Sort: "-age", + ExpectedESRequest: `{ + "_source": false, + "from": 0, + "query": { + "bool": { + "must": [ + { + "term": { + "org_id": 1 + } + }, + { + "term": { + "is_active": true + } + }, + { + "term": { + "groups": "9295ebab-5c2d-4eb1-86f9-7c15ed2f3219" + } + }, + { + "nested": { + "path": "fields", + "query": { + "bool": { + "must": [ + { + "term": { + "fields.field": "903f51da-2717-47c7-a0d3-f2f32877013d" + } + }, + { + "range": { + "fields.number": { + "from": 32, + "include_lower": false, + "include_upper": true, + "to": null + } + } + } + ] + } + } + } + } + ], + "must_not": { + "ids": { + "type": "_doc", + "values": [ + "10001", + "10000" ] - }}, - { "term":{ - "is_blocked":false - }}, - {"term": - {"is_stopped":false - }} + } + } + } + }, + "size": 50, + "sort": [ + { + "fields.number": { + "nested": { + "filter": { + "term": { + "fields.field": "903f51da-2717-47c7-a0d3-f2f32877013d" + } + }, + "path": "fields" + }, + "order": "desc" + } + } + ] + }`, + MockedESResponse: fmt.Sprintf(`{ + "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", + "took": 2, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": null, + "hits": [ + { + "_index": "contacts", + "_type": "_doc", + "_id": "%d", + "_score": null, + "_routing": "1", + "sort": [ + 15124352 + ] + } + ] + } + }`, GeorgeID), + ExpectedContacts: []ContactID{GeorgeID}, + ExpectedTotal: 1, + }, + { + Query: "goats > 2", // no such contact field + ExpectedError: "error parsing query: goats > 2: can't resolve 'goats' to attribute, scheme or field", + }, + } + + for i, tc := range tcs { + es.NextResponse = tc.MockedESResponse + + _, ids, total, err := ContactIDsForQueryPage(ctx, client, oa, tc.Group, tc.ExcludeIDs, tc.Query, tc.Sort, 0, 50) + + if tc.ExpectedError != "" { + assert.EqualError(t, err, tc.ExpectedError) + } else { + assert.NoError(t, err, "%d: error encountered performing query", i) + assert.Equal(t, tc.ExpectedContacts, ids, "%d: ids mismatch", i) + assert.Equal(t, tc.ExpectedTotal, total, "%d: total mismatch", i) + + test.AssertEqualJSON(t, []byte(tc.ExpectedESRequest), []byte(es.LastBody), "%d: ES request mismatch", i) + } + } +} + +func TestContactIDsForQuery(t *testing.T) { + testsuite.Reset() + ctx := testsuite.CTX() + db := testsuite.DB() + + es := testsuite.NewMockElasticServer() + defer es.Close() + + client, err := elastic.NewClient( + elastic.SetURL(es.URL()), + elastic.SetHealthcheck(false), + elastic.SetSniff(false), + ) + require.NoError(t, err) + + oa, err := GetOrgAssets(ctx, db, 1) + require.NoError(t, err) + + tcs := []struct { + Query string + ExpectedESRequest string + MockedESResponse string + ExpectedContacts []ContactID + ExpectedError string + }{ + { + Query: "george", + ExpectedESRequest: `{ + "_source":false, + "query": { + "bool": { + "must": [ + { + "term": { + "org_id": 1 + } + }, + { + "term": { + "is_active": true + } + }, + { + "term": { + "status": "A" + } + }, + { + "match": { + "name": { + "query": "george" + } + } + } ] } }, "sort":["_doc"] }`, - Response: fmt.Sprintf(`{ + MockedESResponse: fmt.Sprintf(`{ "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", "took": 2, "timed_out": false, @@ -92,29 +339,42 @@ func TestElasticContacts(t *testing.T) { ] } }`, GeorgeID), - Contacts: []ContactID{GeorgeID}, + ExpectedContacts: []ContactID{GeorgeID}, }, { Query: "nobody", - Request: `{ + ExpectedESRequest: `{ "_source":false, - "query":{ - "bool":{ - "must":[ - {"bool": - {"must":[ - {"term":{"org_id":1}}, - {"term":{"is_active":true}}, - {"match":{"name":{"query":"nobody"}}} - ]} + "query": { + "bool": { + "must": [ + { + "term": { + "org_id": 1 + } + }, + { + "term": { + "is_active": true + } }, - {"term":{"is_blocked":false}}, - {"term":{"is_stopped":false}} + { + "term": { + "status": "A" + } + }, + { + "match": { + "name": { + "query": "nobody" + } + } + } ] } }, "sort":["_doc"] }`, - Response: `{ + MockedESResponse: `{ "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", "took": 2, "timed_out": false, @@ -130,24 +390,26 @@ func TestElasticContacts(t *testing.T) { "hits": [] } }`, - Contacts: []ContactID{}, - }, { - Query: "goats > 2", // no such contact field - Error: true, + ExpectedContacts: []ContactID{}, + }, + { + Query: "goats > 2", // no such contact field + ExpectedError: "error parsing query: goats > 2: can't resolve 'goats' to attribute, scheme or field", }, } for i, tc := range tcs { - es.NextResponse = tc.Response + es.NextResponse = tc.MockedESResponse - ids, err := ContactIDsForQuery(ctx, client, org, tc.Query) + ids, err := ContactIDsForQuery(ctx, client, oa, tc.Query) - if tc.Error { - assert.Error(t, err) + if tc.ExpectedError != "" { + assert.EqualError(t, err, tc.ExpectedError) } else { assert.NoError(t, err, "%d: error encountered performing query", i) - assert.JSONEq(t, tc.Request, es.LastBody, "%d: request mismatch, got: %s", i, es.LastBody) - assert.Equal(t, tc.Contacts, ids, "%d: ids mismatch", i) + assert.Equal(t, tc.ExpectedContacts, ids, "%d: ids mismatch", i) + + test.AssertEqualJSON(t, []byte(tc.ExpectedESRequest), []byte(es.LastBody), "%d: request mismatch", i) } } } @@ -162,7 +424,7 @@ func TestContacts(t *testing.T) { db.MustExec( `INSERT INTO contacts_contacturn(org_id, contact_id, scheme, path, identity, priority) - VALUES(1, $1, 'whatsapp', '250788373373', 'whatsapp:250788373373', 100)`, BobID) + VALUES(1, $1, 'whatsapp', '250788373373', 'whatsapp:250788373373', 999)`, BobID) db.MustExec(`DELETE FROM contacts_contacturn WHERE contact_id = $1`, GeorgeID) db.MustExec(`DELETE FROM contacts_contactgroup_contacts WHERE contact_id = $1`, GeorgeID) @@ -182,7 +444,7 @@ func TestContacts(t *testing.T) { if len(contacts) == 3 { assert.Equal(t, "Cathy", contacts[0].Name()) assert.Equal(t, len(contacts[0].URNs()), 1) - assert.Equal(t, contacts[0].URNs()[0].String(), "tel:+16055741111?id=10000&priority=50") + assert.Equal(t, contacts[0].URNs()[0].String(), "tel:+16055741111?id=10000&priority=1000") assert.Equal(t, 1, contacts[0].Groups().Count()) assert.Equal(t, "Yobe", contacts[0].Fields()["state"].QueryValue()) @@ -193,8 +455,8 @@ func TestContacts(t *testing.T) { assert.Equal(t, "Bob", contacts[1].Name()) assert.NotNil(t, contacts[1].Fields()["joined"].QueryValue()) assert.Equal(t, 2, len(contacts[1].URNs())) - assert.Equal(t, contacts[1].URNs()[0].String(), "whatsapp:250788373373?id=20121&priority=100") - assert.Equal(t, contacts[1].URNs()[1].String(), "tel:+16055742222?id=10001&priority=50") + assert.Equal(t, contacts[1].URNs()[0].String(), "tel:+16055742222?id=10001&priority=1000") + assert.Equal(t, contacts[1].URNs()[1].String(), "whatsapp:250788373373?id=20121&priority=999") assert.Equal(t, 0, contacts[1].Groups().Count()) assert.Equal(t, "George", contacts[2].Name()) @@ -287,7 +549,7 @@ func TestContactsFromURN(t *testing.T) { } } -func TestCreateContact(t *testing.T) { +func TestGetOrCreateContact(t *testing.T) { ctx := testsuite.CTX() db := testsuite.DB() testsuite.Reset() @@ -304,16 +566,15 @@ func TestCreateContact(t *testing.T) { {Org1, urns.URN(CathyURN.String() + "?foo=bar"), CathyID}, {Org1, urns.URN("telegram:12345678"), ContactID(maxContactID + 3)}, {Org1, urns.URN("telegram:12345678"), ContactID(maxContactID + 3)}, - {Org1, urns.NilURN, ContactID(maxContactID + 5)}, } org, err := GetOrgAssets(ctx, db, Org1) assert.NoError(t, err) for i, tc := range tcs { - id, err := CreateContact(ctx, db, org, tc.URN) + contact, _, err := GetOrCreateContact(ctx, db, org, tc.URN) assert.NoError(t, err, "%d: error creating contact", i) - assert.Equal(t, tc.ContactID, id, "%d: mismatch in contact id", i) + assert.Equal(t, tc.ContactID, contact.ID(), "%d: mismatch in contact id", i) } } diff --git a/models/events.go b/models/events.go index 3ddb1c23e..cc54b5656 100644 --- a/models/events.go +++ b/models/events.go @@ -3,9 +3,10 @@ package models import ( "context" + "github.com/nyaruka/goflow/flows" + "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" - "github.com/nyaruka/goflow/flows" "github.com/pkg/errors" ) diff --git a/models/fields.go b/models/fields.go index 1f1b199b2..2468023ea 100644 --- a/models/fields.go +++ b/models/fields.go @@ -4,8 +4,10 @@ import ( "context" "time" - "github.com/jmoiron/sqlx" "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/mailroom/utils/dbutil" + + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -58,7 +60,7 @@ func loadFields(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets.Fie for rows.Next() { field := &Field{} - err = readJSONRow(rows, &field.f) + err = dbutil.ReadJSONRow(rows, &field.f) if err != nil { return nil, nil, errors.Wrap(err, "error reading field") } diff --git a/models/flows.go b/models/flows.go index d2639d89c..2ad796925 100644 --- a/models/flows.go +++ b/models/flows.go @@ -8,6 +8,7 @@ import ( "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/nyaruka/null" "github.com/jmoiron/sqlx" @@ -128,7 +129,7 @@ func loadFlow(ctx context.Context, db *sqlx.DB, sql string, orgID OrgID, arg int return nil, nil } - err = readJSONRow(rows, &flow.f) + err = dbutil.ReadJSONRow(rows, &flow.f) if err != nil { return nil, errors.Wrapf(err, "error reading flow definition by: %s", arg) } diff --git a/models/globals.go b/models/globals.go index a0420d8f2..18baad2cb 100644 --- a/models/globals.go +++ b/models/globals.go @@ -6,6 +6,7 @@ import ( "time" "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/jmoiron/sqlx" "github.com/pkg/errors" @@ -43,7 +44,7 @@ func loadGlobals(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets.Gl globals := make([]assets.Global, 0) for rows.Next() { global := &Global{} - err = readJSONRow(rows, &global.g) + err = dbutil.ReadJSONRow(rows, &global.g) if err != nil { return nil, errors.Wrap(err, "error reading global row") } diff --git a/models/groups.go b/models/groups.go index 15369cdeb..c41c9da94 100644 --- a/models/groups.go +++ b/models/groups.go @@ -4,10 +4,12 @@ import ( "context" "time" - "github.com/jmoiron/sqlx" - "github.com/lib/pq" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/mailroom/utils/dbutil" + + "github.com/jmoiron/sqlx" + "github.com/lib/pq" "github.com/olivere/elastic" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -60,7 +62,7 @@ func loadGroups(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets.Gro groups := make([]assets.Group, 0, 10) for rows.Next() { group := &Group{} - err = readJSONRow(rows, &group.g) + err = dbutil.ReadJSONRow(rows, &group.g) if err != nil { return nil, errors.Wrap(err, "error reading group row") } @@ -101,7 +103,7 @@ func RemoveContactsFromGroups(ctx context.Context, tx Queryer, removals []*Group for i := range removals { is[i] = removals[i] } - return BulkSQL(ctx, "removing contacts from groups", tx, removeContactsFromGroupsSQL, is) + return BulkQuery(ctx, "removing contacts from groups", tx, removeContactsFromGroupsSQL, is) } // GroupRemove is our struct to track group removals @@ -137,7 +139,7 @@ func AddContactsToGroups(ctx context.Context, tx Queryer, adds []*GroupAdd) erro for i := range adds { is[i] = adds[i] } - return BulkSQL(ctx, "adding contacts to groups", tx, addContactsToGroupsSQL, is) + return BulkQuery(ctx, "adding contacts to groups", tx, addContactsToGroupsSQL, is) } // GroupAdd is our struct to track a final group additions diff --git a/models/groups_test.go b/models/groups_test.go index 91049450c..d78f6a96b 100644 --- a/models/groups_test.go +++ b/models/groups_test.go @@ -4,8 +4,8 @@ import ( "fmt" "testing" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/testsuite" "github.com/lib/pq" diff --git a/models/http_logs.go b/models/http_logs.go index 1c80559d1..f89558859 100644 --- a/models/http_logs.go +++ b/models/http_logs.go @@ -101,7 +101,7 @@ func InsertHTTPLogs(ctx context.Context, tx Queryer, logs []*HTTPLog) error { ls[i] = &logs[i].h } - return BulkSQL(ctx, "inserted http logs", tx, insertHTTPLogsSQL, ls) + return BulkQuery(ctx, "inserted http logs", tx, insertHTTPLogsSQL, ls) } // MarshalJSON marshals into JSON. 0 values will become null diff --git a/models/labels.go b/models/labels.go index bf9cd9165..d301f149c 100644 --- a/models/labels.go +++ b/models/labels.go @@ -4,8 +4,10 @@ import ( "context" "time" - "github.com/jmoiron/sqlx" "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/mailroom/utils/dbutil" + + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -43,7 +45,7 @@ func loadLabels(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets.Lab labels := make([]assets.Label, 0, 10) for rows.Next() { label := &Label{} - err = readJSONRow(rows, &label.l) + err = dbutil.ReadJSONRow(rows, &label.l) if err != nil { return nil, errors.Wrap(err, "error scanning label row") } @@ -78,7 +80,7 @@ func AddMsgLabels(ctx context.Context, tx *sqlx.Tx, adds []*MsgLabelAdd) error { is[i] = adds[i] } - err := BulkSQL(ctx, "inserting msg labels", tx, insertMsgLabelsSQL, is) + err := BulkQuery(ctx, "inserting msg labels", tx, insertMsgLabelsSQL, is) if err != nil { return errors.Wrapf(err, "error inserting new msg labels") } diff --git a/models/msgs.go b/models/msgs.go index d64b989e2..d4b3d89d1 100644 --- a/models/msgs.go +++ b/models/msgs.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/nyaruka/gocommon/gsm7" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/envs" @@ -19,7 +20,6 @@ import ( "github.com/nyaruka/goflow/flows/events" "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/config" - "github.com/nyaruka/mailroom/utils/gsm7" "github.com/nyaruka/null" "github.com/gomodule/redigo/redis" @@ -124,10 +124,12 @@ type Msg struct { OrgID OrgID `db:"org_id" json:"org_id"` TopupID TopupID `db:"topup_id"` - // These three fields are set on the last outgoing message in a session's sprint. In the case + SessionID SessionID `json:"session_id,omitempty"` + SessionStatus SessionStatus `json:"session_status,omitempty"` + + // These fields are set on the last outgoing message in a session's sprint. In the case // of the session being at a wait with a timeout then the timeout will be set. It is up to // Courier to update the session's timeout appropriately after sending the message. - SessionID SessionID `json:"session_id,omitempty"` SessionWaitStartedOn *time.Time `json:"session_wait_started_on,omitempty"` SessionTimeout int `json:"session_timeout,omitempty"` } @@ -400,9 +402,13 @@ func NormalizeAttachment(attachment utils.Attachment) utils.Attachment { return utils.Attachment(fmt.Sprintf("%s:%s", attachment.ContentType(), url)) } -// SetTimeout sets the timeout for this message -func (m *Msg) SetTimeout(id SessionID, start time.Time, timeout time.Duration) { +func (m *Msg) SetSession(id SessionID, status SessionStatus) { m.m.SessionID = id + m.m.SessionStatus = status +} + +// SetTimeout sets the timeout for this message +func (m *Msg) SetTimeout(start time.Time, timeout time.Duration) { m.m.SessionWaitStartedOn = &start m.m.SessionTimeout = int(timeout / time.Second) } @@ -414,7 +420,7 @@ func InsertMessages(ctx context.Context, tx Queryer, msgs []*Msg) error { is[i] = &msgs[i].m } - return BulkSQL(ctx, "insert messages", tx, insertMsgSQL, is) + return BulkQuery(ctx, "insert messages", tx, insertMsgSQL, is) } const insertMsgSQL = ` @@ -472,7 +478,7 @@ func updateMessageStatus(ctx context.Context, tx *sqlx.Tx, msgs []*Msg, status M is[i] = m } - return BulkSQL(ctx, "updating message status", tx, updateMsgStatusSQL, is) + return BulkQuery(ctx, "updating message status", tx, updateMsgStatusSQL, is) } const updateMsgStatusSQL = ` @@ -566,7 +572,7 @@ func InsertChildBroadcast(ctx context.Context, db Queryer, parent *Broadcast) (* } // insert our broadcast - err := BulkSQL(ctx, "inserting broadcast", db, insertBroadcastSQL, []interface{}{&child.b}) + err := BulkQuery(ctx, "inserting broadcast", db, insertBroadcastSQL, []interface{}{&child.b}) if err != nil { return nil, errors.Wrapf(err, "error inserting child broadcast for broadcast: %d", parent.BroadcastID()) } @@ -581,7 +587,7 @@ func InsertChildBroadcast(ctx context.Context, db Queryer, parent *Broadcast) (* } // insert our contacts - err = BulkSQL(ctx, "inserting broadcast contacts", db, insertBroadcastContactsSQL, contacts) + err = BulkQuery(ctx, "inserting broadcast contacts", db, insertBroadcastContactsSQL, contacts) if err != nil { return nil, errors.Wrapf(err, "error inserting contacts for broadcast") } @@ -596,7 +602,7 @@ func InsertChildBroadcast(ctx context.Context, db Queryer, parent *Broadcast) (* } // insert our groups - err = BulkSQL(ctx, "inserting broadcast groups", db, insertBroadcastGroupsSQL, groups) + err = BulkQuery(ctx, "inserting broadcast groups", db, insertBroadcastGroupsSQL, groups) if err != nil { return nil, errors.Wrapf(err, "error inserting groups for broadcast") } @@ -615,7 +621,7 @@ func InsertChildBroadcast(ctx context.Context, db Queryer, parent *Broadcast) (* } // insert our urns - err = BulkSQL(ctx, "inserting broadcast urns", db, insertBroadcastURNsSQL, urns) + err = BulkQuery(ctx, "inserting broadcast urns", db, insertBroadcastURNsSQL, urns) if err != nil { return nil, errors.Wrapf(err, "error inserting URNs for broadcast") } diff --git a/models/orgs.go b/models/orgs.go index a7f3bf63f..bda7b10e4 100644 --- a/models/orgs.go +++ b/models/orgs.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "mime" "net/http" "path/filepath" "strings" @@ -11,6 +12,7 @@ import ( "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/gocommon/jsonx" + "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/services/airtime/dtone" @@ -18,7 +20,7 @@ import ( "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/goflow" - "github.com/nyaruka/mailroom/utils/storage" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/nyaruka/null" "github.com/jmoiron/sqlx" @@ -154,7 +156,7 @@ func (o *Org) EmailService(httpClient *http.Client) (flows.EmailService, error) if connectionURL == "" { return nil, errors.New("missing SMTP configuration") } - return smtp.NewServiceFromURL(connectionURL) + return smtp.NewService(connectionURL) } // AirtimeService returns the airtime service for this org if one is configured @@ -170,8 +172,11 @@ func (o *Org) AirtimeService(httpClient *http.Client, httpRetries *httpx.RetryCo } // StoreAttachment saves an attachment to storage -func (o *Org) StoreAttachment(s storage.Storage, prefix string, filename string, content []byte) (utils.Attachment, error) { +func (o *Org) StoreAttachment(s storage.Storage, filename string, content []byte) (utils.Attachment, error) { + prefix := config.Mailroom.S3MediaPrefix + contentType := http.DetectContentType(content) + contentType, _, _ = mime.ParseMediaType(contentType) path := o.attachmentPath(prefix, filename) @@ -226,7 +231,7 @@ func loadOrg(ctx context.Context, db sqlx.Queryer, orgID OrgID) (*Org, error) { return nil, errors.Errorf("no org with id: %d", orgID) } - err = readJSONRow(rows, org) + err = dbutil.ReadJSONRow(rows, org) if err != nil { return nil, errors.Wrapf(err, "error unmarshalling org") } diff --git a/models/orgs_test.go b/models/orgs_test.go index bba3ca08e..fa7c79cab 100644 --- a/models/orgs_test.go +++ b/models/orgs_test.go @@ -70,7 +70,7 @@ func TestStoreAttachment(t *testing.T) { org, err := loadOrg(ctx, db, Org1) assert.NoError(t, err) - attachment, err := org.StoreAttachment(store, "media", "668383ba-387c-49bc-b164-1213ac0ea7aa.jpg", image) + attachment, err := org.StoreAttachment(store, "668383ba-387c-49bc-b164-1213ac0ea7aa.jpg", image) require.NoError(t, err) assert.Equal(t, utils.Attachment("image/jpeg:_test_storage/media/1/6683/83ba/668383ba-387c-49bc-b164-1213ac0ea7aa.jpg"), attachment) diff --git a/models/resthooks.go b/models/resthooks.go index bbfe31aa8..e5a9419fb 100644 --- a/models/resthooks.go +++ b/models/resthooks.go @@ -4,8 +4,10 @@ import ( "context" "time" - "github.com/jmoiron/sqlx" "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/mailroom/utils/dbutil" + + "github.com/jmoiron/sqlx" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -44,7 +46,7 @@ func loadResthooks(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets. resthooks := make([]assets.Resthook, 0, 10) for rows.Next() { resthook := &Resthook{} - err = readJSONRow(rows, &resthook.r) + err = dbutil.ReadJSONRow(rows, &resthook.r) if err != nil { return nil, errors.Wrap(err, "error scanning resthook row") } @@ -89,7 +91,7 @@ func UnsubscribeResthooks(ctx context.Context, tx *sqlx.Tx, unsubs []*ResthookUn is[i] = unsubs[i] } - err := BulkSQL(ctx, "unsubscribing resthooks", tx, unsubscribeResthooksSQL, is) + err := BulkQuery(ctx, "unsubscribing resthooks", tx, unsubscribeResthooksSQL, is) if err != nil { return errors.Wrapf(err, "error unsubscribing from resthooks") } diff --git a/models/runs.go b/models/runs.go index 4681396a4..eb520f1aa 100644 --- a/models/runs.go +++ b/models/runs.go @@ -8,11 +8,11 @@ import ( "fmt" "time" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/events" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/goflow" "github.com/nyaruka/null" @@ -567,14 +567,14 @@ func (s *Session) WriteUpdatedSession(ctx context.Context, tx *sqlx.Tx, rp *redi } // update all modified runs at once - err = BulkSQL(ctx, "update runs", tx, updateRunSQL, updatedRuns) + err = BulkQuery(ctx, "update runs", tx, updateRunSQL, updatedRuns) if err != nil { logrus.WithError(err).WithField("session", string(output)).Error("error while updating runs for session") return errors.Wrapf(err, "error updating runs") } // insert all new runs at once - err = BulkSQL(ctx, "insert runs", tx, insertRunSQL, newRuns) + err = BulkQuery(ctx, "insert runs", tx, insertRunSQL, newRuns) if err != nil { return errors.Wrapf(err, "error writing runs") } @@ -682,7 +682,7 @@ func WriteSessions(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, org *OrgAss } // insert our complete sessions first - err := BulkSQL(ctx, "insert completed sessions", tx, insertCompleteSessionSQL, completeSessionsI) + err := BulkQuery(ctx, "insert completed sessions", tx, insertCompleteSessionSQL, completeSessionsI) if err != nil { return nil, errors.Wrapf(err, "error inserting completed sessions") } @@ -694,7 +694,7 @@ func WriteSessions(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, org *OrgAss } // insert incomplete sessions - err = BulkSQL(ctx, "insert incomplete sessions", tx, insertIncompleteSessionSQL, incompleteSessionsI) + err = BulkQuery(ctx, "insert incomplete sessions", tx, insertIncompleteSessionSQL, incompleteSessionsI) if err != nil { return nil, errors.Wrapf(err, "error inserting incomplete sessions") } @@ -711,7 +711,7 @@ func WriteSessions(ctx context.Context, tx *sqlx.Tx, rp *redis.Pool, org *OrgAss } // insert all runs - err = BulkSQL(ctx, "insert runs", tx, insertRunSQL, runs) + err = BulkQuery(ctx, "insert runs", tx, insertRunSQL, runs) if err != nil { return nil, errors.Wrapf(err, "error writing runs") } diff --git a/models/schedules.go b/models/schedules.go index 2b7204a8d..07f67b994 100644 --- a/models/schedules.go +++ b/models/schedules.go @@ -6,8 +6,10 @@ import ( "fmt" "time" - "github.com/jmoiron/sqlx" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/nyaruka/null" + + "github.com/jmoiron/sqlx" "github.com/pkg/errors" ) @@ -283,7 +285,7 @@ func GetUnfiredSchedules(ctx context.Context, db *sqlx.DB) ([]*Schedule, error) unfired := make([]*Schedule, 0, 10) for rows.Next() { s := &Schedule{} - err := readJSONRow(rows, &s.s) + err := dbutil.ReadJSONRow(rows, &s.s) if err != nil { return nil, errors.Wrapf(err, "error reading schedule") } diff --git a/models/starts.go b/models/starts.go index a4d8a78f6..5205fe697 100644 --- a/models/starts.go +++ b/models/starts.go @@ -8,8 +8,8 @@ import ( "github.com/jmoiron/sqlx" "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/null" "github.com/pkg/errors" ) @@ -257,7 +257,7 @@ func InsertFlowStarts(ctx context.Context, db Queryer, starts []*FlowStart) erro } // insert our starts - err := BulkSQL(ctx, "inserting flow start", db, insertStartSQL, is) + err := BulkQuery(ctx, "inserting flow start", db, insertStartSQL, is) if err != nil { return errors.Wrapf(err, "error inserting flow starts") } @@ -274,7 +274,7 @@ func InsertFlowStarts(ctx context.Context, db Queryer, starts []*FlowStart) erro } // insert our contacts - err = BulkSQL(ctx, "inserting flow start contacts", db, insertStartContactsSQL, contacts) + err = BulkQuery(ctx, "inserting flow start contacts", db, insertStartContactsSQL, contacts) if err != nil { return errors.Wrapf(err, "error inserting flow start contacts for flow") } @@ -291,7 +291,7 @@ func InsertFlowStarts(ctx context.Context, db Queryer, starts []*FlowStart) erro } // insert our groups - err = BulkSQL(ctx, "inserting flow start groups", db, insertStartGroupsSQL, groups) + err = BulkQuery(ctx, "inserting flow start groups", db, insertStartGroupsSQL, groups) if err != nil { return errors.Wrapf(err, "error inserting flow start groups for flow") } diff --git a/models/templates.go b/models/templates.go index f8825563e..580e08374 100644 --- a/models/templates.go +++ b/models/templates.go @@ -7,6 +7,7 @@ import ( "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/envs" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/nyaruka/null" "github.com/jmoiron/sqlx" @@ -73,7 +74,7 @@ func loadTemplates(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets. templates := make([]assets.Template, 0) for rows.Next() { template := &Template{} - err = readJSONRow(rows, &template.t) + err = dbutil.ReadJSONRow(rows, &template.t) if err != nil { return nil, errors.Wrap(err, "error reading group row") } diff --git a/models/test_constants.go b/models/test_constants.go index 4285f0207..d1e099343 100644 --- a/models/test_constants.go +++ b/models/test_constants.go @@ -2,9 +2,9 @@ package models import ( "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/goflow/utils/uuids" ) // Constants used in tests, these are tied to the DB created by the @@ -83,7 +83,10 @@ var DoctorsGroupID = GroupID(10000) var DoctorsGroupUUID = assets.GroupUUID("c153e265-f7c9-4539-9dbc-9b358714b638") var AllContactsGroupID = GroupID(1) -var AllContactsGroupUUID = assets.GroupUUID("bc268217-9ffa-49e0-883e-e4e09c252a5a") +var AllContactsGroupUUID = assets.GroupUUID("d1ee73f0-bdb5-47ce-99dd-0c95d4ebf008") + +var BlockedContactsGroupID = GroupID(2) +var BlockedContactsGroupUUID = assets.GroupUUID("9295ebab-5c2d-4eb1-86f9-7c15ed2f3219") var TestersGroupID = GroupID(10001) var TestersGroupUUID = assets.GroupUUID("5e9d8fab-5e7e-4f51-b533-261af5dea70d") diff --git a/models/tickets.go b/models/tickets.go index b9f81edd5..b71e79341 100644 --- a/models/tickets.go +++ b/models/tickets.go @@ -13,6 +13,7 @@ import ( "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/goflow" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/nyaruka/null" "github.com/jmoiron/sqlx" @@ -98,7 +99,7 @@ func (t *Ticket) ForwardIncoming(ctx context.Context, db *sqlx.DB, org *OrgAsset } logger := &HTTPLogger{} - err = service.Forward(t, msgUUID, text, logger.Ticketer(ticketer)) + err = service.Forward(t, msgUUID, text, attachments, logger.Ticketer(ticketer)) return logger.Insert(ctx, db) } @@ -270,7 +271,7 @@ func InsertTickets(ctx context.Context, tx Queryer, tickets []*Ticket) error { ts[i] = &tickets[i].t } - return BulkSQL(ctx, "inserted tickets", tx, insertTicketSQL, ts) + return BulkQuery(ctx, "inserted tickets", tx, insertTicketSQL, ts) } const updateTicketExternalIDSQL = ` @@ -477,7 +478,7 @@ func (t *Ticketer) UpdateConfig(ctx context.Context, db *sqlx.DB, add map[string type TicketService interface { flows.TicketService - Forward(*Ticket, flows.MsgUUID, string, flows.HTTPLogCallback) error + Forward(*Ticket, flows.MsgUUID, string, []utils.Attachment, flows.HTTPLogCallback) error Close([]*Ticket, flows.HTTPLogCallback) error Reopen([]*Ticket, flows.HTTPLogCallback) error } @@ -521,7 +522,7 @@ func LookupTicketerByUUID(ctx context.Context, db Queryer, uuid assets.TicketerU } ticketer := &Ticketer{} - err = readJSONRow(rows, &ticketer.t) + err = dbutil.ReadJSONRow(rows, &ticketer.t) if err != nil { return nil, errors.Wrapf(err, "error unmarshalling ticketer") } @@ -560,7 +561,7 @@ func loadTicketers(ctx context.Context, db sqlx.Queryer, orgID OrgID) ([]assets. ticketers := make([]assets.Ticketer, 0, 2) for rows.Next() { ticketer := &Ticketer{} - err := readJSONRow(rows, &ticketer.t) + err := dbutil.ReadJSONRow(rows, &ticketer.t) if err != nil { return nil, errors.Wrapf(err, "error unmarshalling ticketer") } diff --git a/models/tokens.go b/models/tokens.go index e08c40ae5..750c178b8 100644 --- a/models/tokens.go +++ b/models/tokens.go @@ -4,7 +4,7 @@ import ( "context" "database/sql" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/gocommon/uuids" ) // OrgReference is just a reference for an org, containing the id, uuid and name for the org diff --git a/models/triggers.go b/models/triggers.go index 8e64e4180..24d1f2c6a 100644 --- a/models/triggers.go +++ b/models/triggers.go @@ -8,6 +8,7 @@ import ( "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/triggers" "github.com/nyaruka/goflow/utils" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/jmoiron/sqlx" "github.com/lib/pq" @@ -101,7 +102,7 @@ func loadTriggers(ctx context.Context, db *sqlx.DB, orgID OrgID) ([]*Trigger, er triggers := make([]*Trigger, 0, 10) for rows.Next() { trigger := &Trigger{} - err = readJSONRow(rows, &trigger.t) + err = dbutil.ReadJSONRow(rows, &trigger.t) if err != nil { return nil, errors.Wrap(err, "error scanning label row") } diff --git a/models/utils.go b/models/utils.go index 0b9d1f5ba..4ed3bb324 100644 --- a/models/utils.go +++ b/models/utils.go @@ -3,73 +3,17 @@ package models import ( "context" "database/sql" - "encoding/json" "fmt" - "strings" "time" - "github.com/jmoiron/sqlx" + "github.com/nyaruka/mailroom/utils/dbutil" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "gopkg.in/go-playground/validator.v9" ) -var validate = validator.New() - -func readJSONRow(rows *sqlx.Rows, destination interface{}) error { - var jsonBlob string - err := rows.Scan(&jsonBlob) - if err != nil { - return errors.Wrap(err, "error scanning row json") - } - - err = json.Unmarshal([]byte(jsonBlob), destination) - if err != nil { - return errors.Wrap(err, "error unmarshalling row json") - } - - // validate our final struct - err = validate.Struct(destination) - if err != nil { - return errors.Wrapf(err, "failed validation for json: %s", jsonBlob) - } - - return nil -} - -// extractValues is just a simple utility method that extracts the portion between `VALUE(` -// and `)` in the passed in string. (leaving VALUE but not the parentheses) -func extractValues(sql string) (string, error) { - startValues := strings.Index(sql, "VALUES(") - if startValues <= 0 { - return "", errors.Errorf("unable to find VALUES( in bulk insert SQL: %s", sql) - } - - // find the matching end parentheses, we need to count balances here - openCount := 1 - endValues := -1 - for i, r := range sql[startValues+7:] { - if r == '(' { - openCount++ - } else if r == ')' { - openCount-- - if openCount == 0 { - endValues = i + startValues + 7 - break - } - } - } - - if endValues <= 0 { - return "", errors.Errorf("unable to find end of VALUES() in bulk insert sql: %s", sql) - } - - return sql[startValues+6 : endValues+1], nil -} - type Queryer interface { - Rebind(query string) string - QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) + dbutil.Queryer + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) NamedExecContext(ctx context.Context, query string, arg interface{}) (sql.Result, error) GetContext(ctx context.Context, value interface{}, query string, args ...interface{}) error @@ -89,78 +33,51 @@ func Exec(ctx context.Context, label string, tx Queryer, sql string, args ...int return nil } -// BulkSQL runs the SQL passed in for the passed in interfaces, replacing any variables in the SQL as needed -func BulkSQL(ctx context.Context, label string, tx Queryer, sql string, vs []interface{}) error { +// BulkQuery runs the given query as a bulk operation +func BulkQuery(ctx context.Context, label string, tx Queryer, sql string, structs []interface{}) error { // no values, nothing to do - if len(vs) == 0 { + if len(structs) == 0 { return nil } start := time.Now() - // this will be our SQL placeholders for values in our final query, built dynamically - values := strings.Builder{} - values.Grow(7 * len(vs)) + err := dbutil.BulkQuery(ctx, tx, sql, structs) + if err != nil { + return errors.Wrap(err, "error making bulk query") + } - // this will be each of the arguments to match the positional values above - args := make([]interface{}, 0, len(vs)*5) + logrus.WithField("elapsed", time.Since(start)).WithField("rows", len(structs)).Infof("%s bulk sql complete", label) - // for each value we build a bound SQL statement, then extract the values clause - for i, value := range vs { - valueSQL, valueArgs, err := sqlx.Named(sql, value) - if err != nil { - return errors.Wrapf(err, "error converting bulk insert args") - } + return nil +} - args = append(args, valueArgs...) - argValues, err := extractValues(valueSQL) - if err != nil { - return errors.Wrapf(err, "error extracting values from sql: %s", valueSQL) - } +// BulkQueryBatches runs the given query as a bulk operation, in batches of the given size +func BulkQueryBatches(ctx context.Context, label string, tx Queryer, sql string, batchSize int, structs []interface{}) error { + start := time.Now() - // append to our global values, adding comma if necessary - values.WriteString(argValues) - if i+1 < len(vs) { - values.WriteString(",") + batches := chunkSlice(structs, batchSize) + for i, batch := range batches { + err := dbutil.BulkQuery(ctx, tx, sql, batch) + if err != nil { + return errors.Wrap(err, "error making bulk batch query") } - } - valuesSQL, err := extractValues(sql) - if err != nil { - return errors.Wrapf(err, "error extracting values from sql: %s", sql) + logrus.WithField("elapsed", time.Since(start)).WithField("rows", len(batch)).WithField("batch", i+1).Infof("%s bulk sql batch complete", label) } - bulkQuery := tx.Rebind(strings.Replace(sql, valuesSQL, values.String(), -1)) - rows, err := tx.QueryxContext(ctx, bulkQuery, args...) - if err != nil { - return errors.Wrapf(err, "error during bulk query") - } - defer rows.Close() - - // if have a returning clause, read them back and try to map them - if strings.Contains(strings.ToUpper(sql), "RETURNING") { - for _, v := range vs { - if !rows.Next() { - return errors.Errorf("did not receive expected number of rows on insert") - } - - err = rows.StructScan(v) - if err != nil { - return errors.Wrap(err, "error scanning for insert id") - } - } - } + return nil +} - // iterate our remaining rows - for rows.Next() { - } +func chunkSlice(slice []interface{}, size int) [][]interface{} { + chunks := make([][]interface{}, 0, len(slice)/size+1) - // check for any error - if rows.Err() != nil { - return errors.Wrapf(rows.Err(), "error in row cursor") + for i := 0; i < len(slice); i += size { + end := i + size + if end > len(slice) { + end = len(slice) + } + chunks = append(chunks, slice[i:end]) } - - logrus.WithField("elapsed", time.Since(start)).WithField("rows", len(vs)).Infof("%s bulk sql complete", label) - - return nil + return chunks } diff --git a/models/utils_test.go b/models/utils_test.go new file mode 100644 index 000000000..e22ce29b5 --- /dev/null +++ b/models/utils_test.go @@ -0,0 +1,60 @@ +package models_test + +import ( + "testing" + + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/testsuite" + + "github.com/stretchr/testify/assert" +) + +func TestBulkQueryBatches(t *testing.T) { + ctx := testsuite.CTX() + db := testsuite.DB() + defer testsuite.Reset() + + db.MustExec(`CREATE TABLE foo (id serial NOT NULL PRIMARY KEY, name TEXT, age INT)`) + + type foo struct { + ID int `db:"id"` + Name string `db:"name"` + Age int `db:"age"` + } + + sql := `INSERT INTO foo (name, age) VALUES(:name, :age) RETURNING id` + + // noop with zero structs + err := models.BulkQueryBatches(ctx, "foo inserts", db, sql, 10, nil) + assert.NoError(t, err) + + // test when structs fit into one batch + foo1 := &foo{Name: "A", Age: 30} + foo2 := &foo{Name: "B", Age: 31} + err = models.BulkQueryBatches(ctx, "foo inserts", db, sql, 2, []interface{}{foo1, foo2}) + assert.NoError(t, err) + assert.Equal(t, 1, foo1.ID) + assert.Equal(t, 2, foo2.ID) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM foo WHERE name = 'A' AND age = 30`, nil, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM foo WHERE name = 'B' AND age = 31`, nil, 1) + + // test when multiple batches are required + foo3 := &foo{Name: "C", Age: 32} + foo4 := &foo{Name: "D", Age: 33} + foo5 := &foo{Name: "E", Age: 34} + foo6 := &foo{Name: "F", Age: 35} + foo7 := &foo{Name: "G", Age: 36} + err = models.BulkQueryBatches(ctx, "foo inserts", db, sql, 2, []interface{}{foo3, foo4, foo5, foo6, foo7}) + assert.NoError(t, err) + assert.Equal(t, 3, foo3.ID) + assert.Equal(t, 4, foo4.ID) + assert.Equal(t, 5, foo5.ID) + assert.Equal(t, 6, foo6.ID) + assert.Equal(t, 7, foo7.ID) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM foo WHERE name = 'C' AND age = 32`, nil, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM foo WHERE name = 'D' AND age = 33`, nil, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM foo WHERE name = 'E' AND age = 34`, nil, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM foo WHERE name = 'F' AND age = 35`, nil, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM foo WHERE name = 'G' AND age = 36`, nil, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM foo `, nil, 7) +} diff --git a/models/webhook_event.go b/models/webhook_event.go index a20043a56..e11ded7ad 100644 --- a/models/webhook_event.go +++ b/models/webhook_event.go @@ -50,5 +50,5 @@ func InsertWebhookEvents(ctx context.Context, db Queryer, events []*WebhookEvent is[i] = &events[i].e } - return BulkSQL(ctx, "inserted webhook events", db, insertWebhookEventsSQL, is) + return BulkQuery(ctx, "inserted webhook events", db, insertWebhookEventsSQL, is) } diff --git a/models/webhook_results.go b/models/webhook_results.go index 338474a8a..2b7be4e9a 100644 --- a/models/webhook_results.go +++ b/models/webhook_results.go @@ -52,7 +52,7 @@ func InsertWebhookResults(ctx context.Context, db Queryer, results []*WebhookRes is[i] = &results[i].r } - return BulkSQL(ctx, "inserting webhook results", db, insertWebhookResultsSQL, is) + return BulkQuery(ctx, "inserting webhook results", db, insertWebhookResultsSQL, is) } const insertWebhookResultsSQL = ` diff --git a/queue/queue.go b/queue/queue.go index aac3467d9..3ce06bfad 100644 --- a/queue/queue.go +++ b/queue/queue.go @@ -47,9 +47,6 @@ const ( // SendBroadcastBatch is our type for sending a broadcast batch SendBroadcastBatch = "send_broadcast_batch" - // FireCampaignEvent is our type for firing a campaign event - FireCampaignEvent = "fire_campaign_event" - // HandleContactEvent is our task for event handling HandleContactEvent = "handle_contact_event" @@ -61,12 +58,6 @@ const ( // StartIVRFlowBatch is our task for starting an ivr batch StartIVRFlowBatch = "start_ivr_flow_batch" - - // InterruptSessions is our task type to interrupt a set of sessions - InterruptSessions = "interrupt_sessions" - - // PopulateDynamicGroup is our task to populate the contacts for a dynamic group - PopulateDynamicGroup = "populate_dynamic_group" ) // Size returns the number of tasks for the passed in queue diff --git a/runner/runner_test.go b/runner/runner_test.go index 778696f29..3efa06216 100644 --- a/runner/runner_test.go +++ b/runner/runner_test.go @@ -6,10 +6,10 @@ import ( "time" "github.com/lib/pq" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/resumes" "github.com/nyaruka/goflow/flows/triggers" - "github.com/nyaruka/goflow/utils/uuids" _ "github.com/nyaruka/mailroom/hooks" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" diff --git a/services/tickets/mailgun/client.go b/services/tickets/mailgun/client.go index b64a7a554..f68ade3a4 100644 --- a/services/tickets/mailgun/client.go +++ b/services/tickets/mailgun/client.go @@ -9,8 +9,8 @@ import ( "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/gocommon/jsonx" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/uuids" "github.com/pkg/errors" ) @@ -43,15 +43,25 @@ type messageResponse struct { ID string `json:"id"` } +type File struct { + Filename string + Data []byte +} + // SendMessage sends a new email message and returns the ID // see https://documentation.mailgun.com/en/latest/api-sending.html -func (c *Client) SendMessage(from, to, subject, text string, headers map[string]string) (string, *httpx.Trace, error) { +func (c *Client) SendMessage(from, to, subject, text string, attachments []File, headers map[string]string) (string, *httpx.Trace, error) { writeBody := func(w *multipart.Writer) { w.WriteField("from", from) w.WriteField("to", to) w.WriteField("subject", subject) w.WriteField("text", text) + for _, attachment := range attachments { + fw, _ := w.CreateFormFile("attachment", attachment.Filename) + fw.Write(attachment.Data) + } + // for the sake of tests, we want to output headers in consistent order headerKeys := make([]string, 0, len(headers)) for k := range headers { diff --git a/services/tickets/mailgun/client_test.go b/services/tickets/mailgun/client_test.go index cb9909d0f..7d468001f 100644 --- a/services/tickets/mailgun/client_test.go +++ b/services/tickets/mailgun/client_test.go @@ -5,7 +5,8 @@ import ( "testing" "github.com/nyaruka/gocommon/httpx" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/gocommon/uuids" + "github.com/nyaruka/goflow/test" "github.com/nyaruka/mailroom/services/tickets/mailgun" "github.com/stretchr/testify/assert" @@ -30,18 +31,26 @@ func TestSendMessage(t *testing.T) { client := mailgun.NewClient(http.DefaultClient, nil, "tickets.rapidpro.io", "123456789") - _, _, err := client.SendMessage("Bob ", "support@acme.com", "Need help", "Where are my cookies?", nil) + _, _, err := client.SendMessage("Bob ", "support@acme.com", "Need help", "Where are my cookies?", nil, nil) assert.EqualError(t, err, "unable to connect to server") - _, _, err = client.SendMessage("Bob ", "support@acme.com", "Need help", "Where are my cookies?", nil) + _, _, err = client.SendMessage("Bob ", "support@acme.com", "Need help", "Where are my cookies?", nil, nil) assert.EqualError(t, err, "Something went wrong") - _, _, err = client.SendMessage("Bob ", "support@acme.com", "Need help", "Where are my cookies?", nil) + _, _, err = client.SendMessage("Bob ", "support@acme.com", "Need help", "Where are my cookies?", nil, nil) assert.EqualError(t, err, "invalid character 'x' looking for beginning of value") - msgID, trace, err := client.SendMessage("Bob ", "support@acme.com", "Need help", "Where are my cookies?", map[string]string{"In-Reply-To": "12415"}) + msgID, trace, err := client.SendMessage( + "Bob ", + "support@acme.com", + "Need help", + "Where are my cookies?", + []mailgun.File{{"test.jpg", []byte(`IMANIMAGE`)}, {"test.mp4", []byte(`IMAVIDEO`)}}, + map[string]string{"In-Reply-To": "12415"}, + ) assert.NoError(t, err) assert.Equal(t, "<20200426161758.1.590432020254B2BF@tickets.rapidpro.io>", msgID) - assert.Equal(t, "POST /v3/tickets.rapidpro.io/messages HTTP/1.1\r\nHost: api.mailgun.net\r\nUser-Agent: Go-http-client/1.1\r\nContent-Length: 586\r\nAuthorization: Basic YXBpOjEyMzQ1Njc4OQ==\r\nContent-Type: multipart/form-data; boundary=9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nAccept-Encoding: gzip\r\n\r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nContent-Disposition: form-data; name=\"from\"\r\n\r\nBob \r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nContent-Disposition: form-data; name=\"to\"\r\n\r\nsupport@acme.com\r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nContent-Disposition: form-data; name=\"subject\"\r\n\r\nNeed help\r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nContent-Disposition: form-data; name=\"text\"\r\n\r\nWhere are my cookies?\r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d\r\nContent-Disposition: form-data; name=\"h:In-Reply-To\"\r\n\r\n12415\r\n--9688d21d-95aa-4bed-afc7-f31b35731a3d--\r\n", string(trace.RequestTrace)) assert.Equal(t, "HTTP/1.0 200 OK\r\nContent-Length: 111\r\n\r\n", string(trace.ResponseTrace)) + + test.AssertSnapshot(t, "mailgun_request", string(trace.RequestTrace)) } diff --git a/services/tickets/mailgun/service.go b/services/tickets/mailgun/service.go index de04e137a..0a8f2aa23 100644 --- a/services/tickets/mailgun/service.go +++ b/services/tickets/mailgun/service.go @@ -8,9 +8,9 @@ import ( "text/template" "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/services/tickets" @@ -115,7 +115,7 @@ func (s *service) Open(session flows.Session, subject, body string, logHTTP flow context := s.templateContext(subject, body, "", string(session.Contact().UUID()), contactDisplay) fullBody := evaluateTemplate(openBodyTemplate, context) - msgID, trace, err := s.client.SendMessage(from, s.toAddress, subject, fullBody, nil) + msgID, trace, err := s.client.SendMessage(from, s.toAddress, subject, fullBody, nil, nil) if trace != nil { logHTTP(flows.NewHTTPLog(trace, flows.HTTPStatusFromCode, s.redactor)) } @@ -126,11 +126,11 @@ func (s *service) Open(session flows.Session, subject, body string, logHTTP flow return flows.NewTicket(ticketUUID, s.ticketer.Reference(), subject, body, msgID), nil } -func (s *service) Forward(ticket *models.Ticket, msgUUID flows.MsgUUID, text string, logHTTP flows.HTTPLogCallback) error { +func (s *service) Forward(ticket *models.Ticket, msgUUID flows.MsgUUID, text string, attachments []utils.Attachment, logHTTP flows.HTTPLogCallback) error { context := s.templateContext(ticket.Subject(), ticket.Body(), text, ticket.Config(ticketConfigContactUUID), ticket.Config(ticketConfigContactDisplay)) body := evaluateTemplate(forwardBodyTemplate, context) - _, err := s.sendInTicket(ticket, body, logHTTP) + _, err := s.sendInTicket(ticket, body, attachments, logHTTP) return err } @@ -139,7 +139,7 @@ func (s *service) Close(tickets []*models.Ticket, logHTTP flows.HTTPLogCallback) context := s.templateContext(ticket.Subject(), ticket.Body(), "", ticket.Config(ticketConfigContactUUID), ticket.Config(ticketConfigContactDisplay)) body := evaluateTemplate(closedBodyTemplate, context) - _, err := s.sendInTicket(ticket, body, logHTTP) + _, err := s.sendInTicket(ticket, body, nil, logHTTP) if err != nil { return err } @@ -152,7 +152,7 @@ func (s *service) Reopen(tickets []*models.Ticket, logHTTP flows.HTTPLogCallback context := s.templateContext(ticket.Subject(), ticket.Body(), "", ticket.Config(ticketConfigContactUUID), ticket.Config(ticketConfigContactDisplay)) body := evaluateTemplate(reopenedBodyTemplate, context) - _, err := s.sendInTicket(ticket, body, logHTTP) + _, err := s.sendInTicket(ticket, body, nil, logHTTP) if err != nil { return err } @@ -161,7 +161,7 @@ func (s *service) Reopen(tickets []*models.Ticket, logHTTP flows.HTTPLogCallback } // sends an email as part of the thread for the given ticket -func (s *service) sendInTicket(ticket *models.Ticket, text string, logHTTP flows.HTTPLogCallback) (string, error) { +func (s *service) sendInTicket(ticket *models.Ticket, text string, attachments []utils.Attachment, logHTTP flows.HTTPLogCallback) (string, error) { contactDisplay := ticket.Config(ticketConfigContactDisplay) lastMessageID := ticket.Config(ticketConfigLastMessageID) if lastMessageID == "" { @@ -173,11 +173,14 @@ func (s *service) sendInTicket(ticket *models.Ticket, text string, logHTTP flows } from := s.ticketAddress(contactDisplay, ticket.UUID()) - return s.send(from, s.toAddress, ticket.Subject(), text, headers, logHTTP) + return s.send(from, s.toAddress, ticket.Subject(), text, attachments, headers, logHTTP) } -func (s *service) send(from, to, subject, text string, headers map[string]string, logHTTP flows.HTTPLogCallback) (string, error) { - msgID, trace, err := s.client.SendMessage(from, to, subject, text, headers) +func (s *service) send(from, to, subject, text string, attachments []utils.Attachment, headers map[string]string, logHTTP flows.HTTPLogCallback) (string, error) { + // TODO fetch attachments to send + var files []File + + msgID, trace, err := s.client.SendMessage(from, to, subject, text, files, headers) if trace != nil { logHTTP(flows.NewHTTPLog(trace, flows.HTTPStatusFromCode, s.redactor)) } diff --git a/services/tickets/mailgun/service_test.go b/services/tickets/mailgun/service_test.go index 2be0f4370..21896f72b 100644 --- a/services/tickets/mailgun/service_test.go +++ b/services/tickets/mailgun/service_test.go @@ -7,12 +7,12 @@ import ( "github.com/nyaruka/gocommon/dates" "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/assets/static/types" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/test" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/services/tickets/mailgun" @@ -94,7 +94,7 @@ func TestOpenAndForward(t *testing.T) { }) logger = &flows.HTTPLogger{} - err = svc.Forward(dbTicket, flows.MsgUUID("ca5607f0-cba8-4c94-9cd5-c4fbc24aa767"), "It's urgent", logger.Log) + err = svc.Forward(dbTicket, flows.MsgUUID("ca5607f0-cba8-4c94-9cd5-c4fbc24aa767"), "It's urgent", nil, logger.Log) assert.NoError(t, err) assert.Equal(t, 1, len(logger.Logs)) diff --git a/services/tickets/mailgun/testdata/TestSendMessage_mailgun_request.snap b/services/tickets/mailgun/testdata/TestSendMessage_mailgun_request.snap new file mode 100644 index 000000000..3f3ba94d7 --- /dev/null +++ b/services/tickets/mailgun/testdata/TestSendMessage_mailgun_request.snap @@ -0,0 +1,39 @@ +POST /v3/tickets.rapidpro.io/messages HTTP/1.1 +Host: api.mailgun.net +User-Agent: Go-http-client/1.1 +Content-Length: 915 +Authorization: Basic YXBpOjEyMzQ1Njc4OQ== +Content-Type: multipart/form-data; boundary=9688d21d-95aa-4bed-afc7-f31b35731a3d +Accept-Encoding: gzip + +--9688d21d-95aa-4bed-afc7-f31b35731a3d +Content-Disposition: form-data; name="from" + +Bob +--9688d21d-95aa-4bed-afc7-f31b35731a3d +Content-Disposition: form-data; name="to" + +support@acme.com +--9688d21d-95aa-4bed-afc7-f31b35731a3d +Content-Disposition: form-data; name="subject" + +Need help +--9688d21d-95aa-4bed-afc7-f31b35731a3d +Content-Disposition: form-data; name="text" + +Where are my cookies? +--9688d21d-95aa-4bed-afc7-f31b35731a3d +Content-Disposition: form-data; name="attachment"; filename="test.jpg" +Content-Type: application/octet-stream + +IMANIMAGE +--9688d21d-95aa-4bed-afc7-f31b35731a3d +Content-Disposition: form-data; name="attachment"; filename="test.mp4" +Content-Type: application/octet-stream + +IMAVIDEO +--9688d21d-95aa-4bed-afc7-f31b35731a3d +Content-Disposition: form-data; name="h:In-Reply-To" + +12415 +--9688d21d-95aa-4bed-afc7-f31b35731a3d-- diff --git a/services/tickets/mailgun/web.go b/services/tickets/mailgun/web.go index 858fe8aed..e026b8474 100644 --- a/services/tickets/mailgun/web.go +++ b/services/tickets/mailgun/web.go @@ -87,7 +87,7 @@ func handleReceive(ctx context.Context, s *web.Server, r *http.Request, l *model if request.Sender != configuredAddress { body := fmt.Sprintf("The address %s is not allowed to reply to this ticket\n", request.Sender) - mailgun.send(mailgun.noReplyAddress(), request.From, "Ticket reply rejected", body, nil, l.Ticketer(ticketer)) + mailgun.send(mailgun.noReplyAddress(), request.From, "Ticket reply rejected", body, nil, nil, l.Ticketer(ticketer)) return &receiveResponse{Action: "rejected", TicketUUID: ticket.UUID()}, http.StatusOK, nil } @@ -112,7 +112,7 @@ func handleReceive(ctx context.Context, s *web.Server, r *http.Request, l *model return errors.Wrapf(err, "error updating ticket: %s", ticket.UUID()), http.StatusInternalServerError, nil } - msg, err := tickets.SendReply(ctx, s.DB, s.RP, s.Storage, s.Config.S3MediaPrefix, ticket, request.StrippedText, nil) + msg, err := tickets.SendReply(ctx, s.DB, s.RP, s.Storage, ticket, request.StrippedText, nil) if err != nil { return err, http.StatusInternalServerError, nil } diff --git a/services/tickets/utils.go b/services/tickets/utils.go index 2a2f1d97d..3a48d7feb 100644 --- a/services/tickets/utils.go +++ b/services/tickets/utils.go @@ -7,14 +7,14 @@ import ( "time" "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/storage" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/courier" "github.com/nyaruka/mailroom/models" - "github.com/nyaruka/mailroom/utils/storage" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" @@ -76,7 +76,7 @@ func FromTicketerUUID(ctx context.Context, db *sqlx.DB, uuid assets.TicketerUUID } // SendReply sends a message reply from the ticket system user to the contact -func SendReply(ctx context.Context, db *sqlx.DB, rp *redis.Pool, store storage.Storage, mediaPrefix string, ticket *models.Ticket, text string, fileURLs []string) (*models.Msg, error) { +func SendReply(ctx context.Context, db *sqlx.DB, rp *redis.Pool, store storage.Storage, ticket *models.Ticket, text string, fileURLs []string) (*models.Msg, error) { // look up our assets oa, err := models.GetOrgAssets(ctx, db, ticket.OrgID()) if err != nil { @@ -93,7 +93,7 @@ func SendReply(ctx context.Context, db *sqlx.DB, rp *redis.Pool, store storage.S filename := string(uuids.New()) + filepath.Ext(fileURL) - attachments[i], err = oa.Org().StoreAttachment(store, mediaPrefix, filename, fileBody) + attachments[i], err = oa.Org().StoreAttachment(store, filename, fileBody) if err != nil { return nil, errors.Wrapf(err, "error storing attachment %s for ticket reply", fileURL) } diff --git a/services/tickets/utils_test.go b/services/tickets/utils_test.go index 9d2cc8047..db3074d55 100644 --- a/services/tickets/utils_test.go +++ b/services/tickets/utils_test.go @@ -5,10 +5,10 @@ import ( "testing" "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/services/tickets" _ "github.com/nyaruka/mailroom/services/tickets/mailgun" @@ -146,7 +146,7 @@ func TestSendReply(t *testing.T) { ticket, err := models.LookupTicketByUUID(ctx, db, ticketUUID) require.NoError(t, err) - msg, err := tickets.SendReply(ctx, db, rp, testsuite.Storage(), "media", ticket, "I'll get back to you", []string{"http://coolfilesfortickets.com/a.jpg"}) + msg, err := tickets.SendReply(ctx, db, rp, testsuite.Storage(), ticket, "I'll get back to you", []string{"http://coolfilesfortickets.com/a.jpg"}) require.NoError(t, err) assert.Equal(t, "I'll get back to you", msg.Text()) @@ -155,6 +155,6 @@ func TestSendReply(t *testing.T) { assert.FileExists(t, "_test_storage/media/1/1ae9/6956/1ae96956-4b34-433e-8d1a-f05fe6923d6d.jpg") // try with file that can't be fetched - _, err = tickets.SendReply(ctx, db, rp, testsuite.Storage(), "media", ticket, "I'll get back to you", []string{"http://badfiles.com/b.jpg"}) + _, err = tickets.SendReply(ctx, db, rp, testsuite.Storage(), ticket, "I'll get back to you", []string{"http://badfiles.com/b.jpg"}) assert.EqualError(t, err, "error fetching file http://badfiles.com/b.jpg for ticket reply: fetch returned non-200 response") } diff --git a/services/tickets/zendesk/service.go b/services/tickets/zendesk/service.go index 0d2c2a399..ed2a9ea25 100644 --- a/services/tickets/zendesk/service.go +++ b/services/tickets/zendesk/service.go @@ -3,12 +3,15 @@ package zendesk import ( "fmt" "net/http" + "net/url" + "strings" "github.com/nyaruka/gocommon/dates" "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/models" "github.com/pkg/errors" @@ -94,10 +97,15 @@ func (s *service) Open(session flows.Session, subject, body string, logHTTP flow return flows.NewTicket(ticketUUID, s.ticketer.Reference(), subject, body, ""), nil } -func (s *service) Forward(ticket *models.Ticket, msgUUID flows.MsgUUID, text string, logHTTP flows.HTTPLogCallback) error { +func (s *service) Forward(ticket *models.Ticket, msgUUID flows.MsgUUID, text string, attachments []utils.Attachment, logHTTP flows.HTTPLogCallback) error { contactUUID := ticket.Config("contact-uuid") contactDisplay := ticket.Config("contact-display") + fileURLs, err := s.convertAttachments(attachments) + if err != nil { + return errors.Wrap(err, "error converting attachments") + } + msg := &ExternalResource{ ExternalID: string(msgUUID), Message: text, @@ -107,6 +115,7 @@ func (s *service) Forward(ticket *models.Ticket, msgUUID flows.MsgUUID, text str ExternalID: contactUUID, Name: contactDisplay, }, + FileURLs: fileURLs, AllowChannelback: true, } @@ -233,3 +242,29 @@ func (s *service) push(msg *ExternalResource, logHTTP flows.HTTPLogCallback) err } return nil } + +// convert attachments to URLs which Zendesk can POST to. +// +// For example https://mybucket.s3.amazonaws.com/attachments/1/01c1/1aa4/01c11aa4-770a-4783.jpg +// is sent to Zendesk as file/1/01c1/1aa4/01c11aa4-770a-4783.jpg +// which it will request as POST https://textit.com/tickets/types/zendesk/file/1/01c1/1aa4/01c11aa4-770a-4783.jpg +// +func (s *service) convertAttachments(attachments []utils.Attachment) ([]string, error) { + prefix := config.Mailroom.S3MediaPrefix + if !strings.HasPrefix(prefix, "/") { + prefix = "/" + prefix + } + + fileURLs := make([]string, len(attachments)) + for i, a := range attachments { + u, err := url.Parse(a.URL()) + if err != nil { + return nil, err + } + path := strings.TrimPrefix(u.Path, prefix) + path = strings.TrimPrefix(path, "/") + + fileURLs[i] = "file/" + path + } + return fileURLs, nil +} diff --git a/services/tickets/zendesk/service_test.go b/services/tickets/zendesk/service_test.go index f6397ac6d..47e43ce85 100644 --- a/services/tickets/zendesk/service_test.go +++ b/services/tickets/zendesk/service_test.go @@ -7,12 +7,13 @@ import ( "github.com/nyaruka/gocommon/dates" "github.com/nyaruka/gocommon/httpx" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/assets/static/types" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/test" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/services/tickets/zendesk" @@ -104,7 +105,13 @@ func TestOpenAndForward(t *testing.T) { }) logger = &flows.HTTPLogger{} - err = svc.Forward(dbTicket, flows.MsgUUID("ca5607f0-cba8-4c94-9cd5-c4fbc24aa767"), "It's urgent", logger.Log) + err = svc.Forward( + dbTicket, + flows.MsgUUID("ca5607f0-cba8-4c94-9cd5-c4fbc24aa767"), + "It's urgent", + []utils.Attachment{utils.Attachment("image/jpg:http://myfiles.com/media/0123/attachment1.jpg")}, + logger.Log, + ) assert.NoError(t, err) assert.Equal(t, 1, len(logger.Logs)) diff --git a/services/tickets/zendesk/testdata/TestOpenAndForward_forward_message.snap b/services/tickets/zendesk/testdata/TestOpenAndForward_forward_message.snap index e7f1991ba..e8759ae66 100644 --- a/services/tickets/zendesk/testdata/TestOpenAndForward_forward_message.snap +++ b/services/tickets/zendesk/testdata/TestOpenAndForward_forward_message.snap @@ -1,9 +1,9 @@ POST /api/v2/any_channel/push.json HTTP/1.1 Host: nyaruka.zendesk.com User-Agent: Go-http-client/1.1 -Content-Length: 367 +Content-Length: 409 Authorization: Bearer **************** Content-Type: application/json Accept-Encoding: gzip -{"instance_push_id":"1234-abcd","request_id":"sesame:1570461699000000000","external_resources":[{"external_id":"ca5607f0-cba8-4c94-9cd5-c4fbc24aa767","message":"It's urgent","thread_id":"59d74b86-3e2f-4a93-aece-b05d2fdcde0c","created_at":"2019-10-07T15:21:38Z","author":{"external_id":"6393abc0-283d-4c9b-a1b3-641a035c34bf","name":"Cathy"},"allow_channelback":true}]} \ No newline at end of file +{"instance_push_id":"1234-abcd","request_id":"sesame:1570461699000000000","external_resources":[{"external_id":"ca5607f0-cba8-4c94-9cd5-c4fbc24aa767","message":"It's urgent","thread_id":"59d74b86-3e2f-4a93-aece-b05d2fdcde0c","created_at":"2019-10-07T15:21:38Z","author":{"external_id":"6393abc0-283d-4c9b-a1b3-641a035c34bf","name":"Cathy"},"allow_channelback":true,"file_urls":["file/0123/attachment1.jpg"]}]} \ No newline at end of file diff --git a/services/tickets/zendesk/testdata/channelback.json b/services/tickets/zendesk/testdata/channelback.json index 7eb0dc244..95e3c27a4 100644 --- a/services/tickets/zendesk/testdata/channelback.json +++ b/services/tickets/zendesk/testdata/channelback.json @@ -45,5 +45,30 @@ "count": 1 } ] + }, + { + "label": "create message with attachments", + "method": "POST", + "path": "/mr/tickets/types/zendesk/channelback", + "body": "file_urls%5B%5D=https%3A%2F%2Fd3v-nyaruka.zendesk.com%2Fattachments%2Ftoken%2FEWTWEGWE%2F%3Fname%3DIhCY7aKs_400x400.jpg&message=Like%20this&recipient_id=1234&thread_id=c69f103c-db64-4481-815b-1112890419ef&metadata=%7B%22ticketer%22%3A%224ee6d4f3-f92b-439b-9718-8da90c05490c%22%2C%22secret%22%3A%22sesame%22%7D", + "http_mocks": { + "https://d3v-nyaruka.zendesk.com/attachments/token/EWTWEGWE/?name=IhCY7aKs_400x400.jpg": [ + { + "status": 200, + "body": "IMAGE" + } + ] + }, + "status": 200, + "response": { + "external_id": "2", + "allow_channelback": true + }, + "db_assertions": [ + { + "query": "select count(*) from msgs_msg where direction = 'O' and text = 'Like this' and attachments = '{text/plain:https:///_test_storage/media/1/6929/26ea/692926ea-09d6-4942-bd38-d266ec8d3716.jpg}'", + "count": 1 + } + ] } ] \ No newline at end of file diff --git a/services/tickets/zendesk/web.go b/services/tickets/zendesk/web.go index bc7b60b71..cea051b48 100644 --- a/services/tickets/zendesk/web.go +++ b/services/tickets/zendesk/web.go @@ -26,7 +26,7 @@ func init() { web.RegisterJSONRoute(http.MethodPost, base+"/channelback", handleChannelback) web.RegisterJSONRoute(http.MethodPost, base+"/event_callback", web.WithHTTPLogs(handleEventCallback)) - web.RegisterJSONRoute(http.MethodPost, base+"/target/{ticketer:[a-f0-9\\-]+}", web.WithHTTPLogs(handleTicketerTarget)) + web.RegisterJSONRoute(http.MethodPost, base+`/target/{ticketer:[a-f0-9\-]+}`, web.WithHTTPLogs(handleTicketerTarget)) } type integrationMetadata struct { @@ -36,7 +36,7 @@ type integrationMetadata struct { type channelbackRequest struct { Message string `form:"message" validate:"required"` - FileURLs []string `form:"file_urls"` + FileURLs []string `form:"file_urls[]"` ParentID string `form:"parent_id"` ThreadID string `form:"thread_id" validate:"required"` RecipientID string `form:"recipient_id" validate:"required"` @@ -76,7 +76,7 @@ func handleChannelback(ctx context.Context, s *web.Server, r *http.Request) (int return errors.Wrapf(err, "error updating ticket: %s", ticket.UUID()), http.StatusBadRequest, nil } - msg, err := tickets.SendReply(ctx, s.DB, s.RP, s.Storage, s.Config.S3MediaPrefix, ticket, request.Message, request.FileURLs) + msg, err := tickets.SendReply(ctx, s.DB, s.RP, s.Storage, ticket, request.Message, request.FileURLs) if err != nil { return err, http.StatusBadRequest, nil } diff --git a/tasks/base.go b/tasks/base.go new file mode 100644 index 000000000..1ee00c03b --- /dev/null +++ b/tasks/base.go @@ -0,0 +1,57 @@ +package tasks + +import ( + "context" + "encoding/json" + "time" + + "github.com/nyaruka/goflow/utils" + "github.com/nyaruka/mailroom" + "github.com/nyaruka/mailroom/queue" + + "github.com/pkg/errors" +) + +var registeredTypes = map[string](func() Task){} + +// RegisterType registers a new type of task +func RegisterType(name string, initFunc func() Task) { + registeredTypes[name] = initFunc + + mailroom.AddTaskFunction(name, func(ctx context.Context, mr *mailroom.Mailroom, task *queue.Task) error { + // decode our task body + typedTask, err := ReadTask(task.Type, task.Task) + if err != nil { + return errors.Wrapf(err, "error reading task of type %s", task.Type) + } + + ctx, cancel := context.WithTimeout(ctx, typedTask.Timeout()) + defer cancel() + + return typedTask.Perform(ctx, mr) + }) +} + +// Task is the common interface for all task types +type Task interface { + // Timeout is the maximum amount of time the task can run for + Timeout() time.Duration + + // Perform performs the task + Perform(ctx context.Context, mr *mailroom.Mailroom) error +} + +//------------------------------------------------------------------------------------------ +// JSON Encoding / Decoding +//------------------------------------------------------------------------------------------ + +// ReadTask reads an action from the given JSON +func ReadTask(typeName string, data json.RawMessage) (Task, error) { + f := registeredTypes[typeName] + if f == nil { + return nil, errors.Errorf("unknown task type: '%s'", typeName) + } + + task := f() + return task, utils.UnmarshalAndValidate(data, task) +} diff --git a/tasks/base_test.go b/tasks/base_test.go new file mode 100644 index 000000000..f4e2d4018 --- /dev/null +++ b/tasks/base_test.go @@ -0,0 +1,26 @@ +package tasks_test + +import ( + "testing" + + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/tasks" + "github.com/nyaruka/mailroom/tasks/groups" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadTask(t *testing.T) { + task, err := tasks.ReadTask("populate_dynamic_group", []byte(`{ + "org_id": 2, + "group_id": 23, + "query": "gender = F" + }`)) + require.NoError(t, err) + + typedTask := task.(*groups.PopulateDynamicGroupTask) + assert.Equal(t, models.OrgID(2), typedTask.OrgID) + assert.Equal(t, models.GroupID(23), typedTask.GroupID) + assert.Equal(t, "gender = F", typedTask.Query) +} diff --git a/tasks/broadcasts/worker_test.go b/tasks/broadcasts/worker_test.go index 901888296..17a6f72f6 100644 --- a/tasks/broadcasts/worker_test.go +++ b/tasks/broadcasts/worker_test.go @@ -47,7 +47,7 @@ func TestBroadcastEvents(t *testing.T) { // add an extra URN fo cathy db.MustExec( `INSERT INTO contacts_contacturn(org_id, contact_id, scheme, path, identity, priority) - VALUES(1, $1, 'tel', '+12065551212', 'tel:+12065551212', 100)`, models.CathyID) + VALUES(1, $1, 'tel', '+12065551212', 'tel:+12065551212', 1001)`, models.CathyID) // change george's URN to an invalid twitter URN so it can't be sent db.MustExec( @@ -171,7 +171,7 @@ func TestBroadcastTask(t *testing.T) { // add an extra URN fo cathy db.MustExec( `INSERT INTO contacts_contacturn(org_id, contact_id, scheme, path, identity, priority) - VALUES(1, $1, 'tel', '+12065551212', 'tel:+12065551212', 100)`, models.CathyID) + VALUES(1, $1, 'tel', '+12065551212', 'tel:+12065551212', 1001)`, models.CathyID) tcs := []struct { BroadcastID models.BroadcastID diff --git a/tasks/campaigns/cron.go b/tasks/campaigns/cron.go index 2e1ae5407..556c31491 100644 --- a/tasks/campaigns/cron.go +++ b/tasks/campaigns/cron.go @@ -61,7 +61,7 @@ func fireCampaignEvents(ctx context.Context, db *sqlx.DB, rp *redis.Pool, lockNa defer rc.Close() queued := 0 - queueTask := func(task *eventFireTask) error { + queueTask := func(task *FireCampaignEventTask) error { if task.EventID == 0 { return nil } @@ -75,7 +75,7 @@ func fireCampaignEvents(ctx context.Context, db *sqlx.DB, rp *redis.Pool, lockNa task.FireIDs = fireIDs[:batchSize] fireIDs = fireIDs[batchSize:] - err = queue.AddTask(rc, queue.BatchQueue, queue.FireCampaignEvent, int(task.OrgID), task, queue.DefaultPriority) + err = queue.AddTask(rc, queue.BatchQueue, TypeFireCampaignEvent, int(task.OrgID), task, queue.DefaultPriority) if err != nil { return errors.Wrap(err, "error queuing task") } @@ -95,7 +95,7 @@ func fireCampaignEvents(ctx context.Context, db *sqlx.DB, rp *redis.Pool, lockNa } // while we have rows - task := &eventFireTask{} + task := &FireCampaignEventTask{} for rows.Next() { row := &eventFireRow{} err := rows.StructScan(row) @@ -128,7 +128,7 @@ func fireCampaignEvents(ctx context.Context, db *sqlx.DB, rp *redis.Pool, lockNa } // and create a new one based on this row - task = &eventFireTask{ + task = &FireCampaignEventTask{ FireIDs: []int64{row.FireID}, EventID: row.EventID, EventUUID: row.EventUUID, @@ -151,16 +151,6 @@ func fireCampaignEvents(ctx context.Context, db *sqlx.DB, rp *redis.Pool, lockNa return nil } -type eventFireTask struct { - FireIDs []int64 `json:"fire_ids"` - EventID int64 `json:"event_id"` - EventUUID string `json:"event_uuid"` - FlowUUID assets.FlowUUID `json:"flow_uuid"` - CampaignUUID string `json:"campaign_uuid"` - CampaignName string `json:"campaign_name"` - OrgID models.OrgID `json:"org_id"` -} - type eventFireRow struct { FireID int64 `db:"fire_id"` EventID int64 `db:"event_id"` diff --git a/tasks/campaigns/campaigns_test.go b/tasks/campaigns/cron_test.go similarity index 85% rename from tasks/campaigns/campaigns_test.go rename to tasks/campaigns/cron_test.go index af05c9f31..ce4faf3e1 100644 --- a/tasks/campaigns/campaigns_test.go +++ b/tasks/campaigns/cron_test.go @@ -4,23 +4,30 @@ import ( "testing" "time" + "github.com/nyaruka/mailroom" + "github.com/nyaruka/mailroom/config" _ "github.com/nyaruka/mailroom/hooks" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" + "github.com/nyaruka/mailroom/tasks" "github.com/nyaruka/mailroom/testsuite" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestCampaigns(t *testing.T) { testsuite.Reset() ctx := testsuite.CTX() + db := testsuite.DB() rp := testsuite.RP() rc := testsuite.RC() defer rc.Close() + mr := &mailroom.Mailroom{Config: config.Mailroom, DB: db, RP: testsuite.RP(), ElasticClient: nil} + // let's create a campaign event fire for one of our contacts (for now this is totally hacked, they aren't in the group and // their relative to date isn't relative, but this still tests execution) - db := testsuite.DB() db.MustExec(`INSERT INTO campaigns_eventfire(scheduled, contact_id, event_id) VALUES (NOW(), $1, $3), (NOW(), $2, $3);`, models.CathyID, models.GeorgeID, models.RemindersEvent1ID) time.Sleep(10 * time.Millisecond) @@ -33,8 +40,11 @@ func TestCampaigns(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, task) + typedTask, err := tasks.ReadTask(task.Type, task.Task) + require.NoError(t, err) + // work on that task - err = fireEventFires(ctx, db, rp, task) + err = typedTask.Perform(ctx, mr) assert.NoError(t, err) // should now have a flow run for that contact and flow @@ -45,13 +55,15 @@ func TestCampaigns(t *testing.T) { func TestIVRCampaigns(t *testing.T) { testsuite.Reset() ctx := testsuite.CTX() + db := testsuite.DB() rp := testsuite.RP() rc := testsuite.RC() defer rc.Close() + mr := &mailroom.Mailroom{Config: config.Mailroom, DB: db, RP: testsuite.RP(), ElasticClient: nil} + // let's create a campaign event fire for one of our contacts (for now this is totally hacked, they aren't in the group and // their relative to date isn't relative, but this still tests execution) - db := testsuite.DB() db.MustExec(`UPDATE campaigns_campaignevent SET flow_id = $1 WHERE id = $2`, models.IVRFlowID, models.RemindersEvent1ID) db.MustExec(`INSERT INTO campaigns_eventfire(scheduled, contact_id, event_id) VALUES (NOW(), $1, $3), (NOW(), $2, $3);`, models.CathyID, models.GeorgeID, models.RemindersEvent1ID) time.Sleep(10 * time.Millisecond) @@ -65,8 +77,11 @@ func TestIVRCampaigns(t *testing.T) { assert.NoError(t, err) assert.NotNil(t, task) + typedTask, err := tasks.ReadTask(task.Type, task.Task) + require.NoError(t, err) + // work on that task - err = fireEventFires(ctx, db, rp, task) + err = typedTask.Perform(ctx, mr) assert.NoError(t, err) // should now have a flow start created diff --git a/tasks/campaigns/worker.go b/tasks/campaigns/fire_campaign_event.go similarity index 51% rename from tasks/campaigns/worker.go rename to tasks/campaigns/fire_campaign_event.go index 06d5eb029..5fd5339b8 100644 --- a/tasks/campaigns/worker.go +++ b/tasks/campaigns/fire_campaign_event.go @@ -2,62 +2,62 @@ package campaigns import ( "context" - "encoding/json" "fmt" "time" - "github.com/gomodule/redigo/redis" - "github.com/jmoiron/sqlx" + "github.com/nyaruka/goflow/assets" "github.com/nyaruka/goflow/flows/triggers" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/models" - "github.com/nyaruka/mailroom/queue" "github.com/nyaruka/mailroom/runner" + "github.com/nyaruka/mailroom/tasks" "github.com/nyaruka/mailroom/utils/marker" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) +// TypeFireCampaignEvent is the type of the fire event task +const TypeFireCampaignEvent = "fire_campaign_event" + func init() { - mailroom.AddTaskFunction(queue.FireCampaignEvent, HandleCampaignEvent) + tasks.RegisterType(TypeFireCampaignEvent, func() tasks.Task { return &FireCampaignEventTask{} }) } -// HandleCampaignEvent is called by mailroom when a campaign event task is ready to be processed. -func HandleCampaignEvent(ctx context.Context, mr *mailroom.Mailroom, task *queue.Task) error { - ctx, cancel := context.WithTimeout(ctx, time.Minute*5) - defer cancel() +// FireCampaignEventTask is the task to handle firing campaign events +type FireCampaignEventTask struct { + FireIDs []int64 `json:"fire_ids"` + EventID int64 `json:"event_id"` + EventUUID string `json:"event_uuid"` + FlowUUID assets.FlowUUID `json:"flow_uuid"` + CampaignUUID string `json:"campaign_uuid"` + CampaignName string `json:"campaign_name"` + OrgID models.OrgID `json:"org_id"` +} - return fireEventFires(ctx, mr.DB, mr.RP, task) +// Timeout is the maximum amount of time the task can run for +func (t *FireCampaignEventTask) Timeout() time.Duration { + return time.Minute * 5 } -// fireEventFires handles expired campaign events -// For each event: -// - loads the event to fire -// - loads the org asset for that event +// Perform handles firing campaign events +// - loads the org assets for that event // - locks on the contact // - loads the contact for that event // - creates the trigger for that event // - runs the flow that is to be started through our engine // - saves the flow run and session resulting from our run -func fireEventFires(ctx context.Context, db *sqlx.DB, rp *redis.Pool, task *queue.Task) error { - log := logrus.WithField("comp", "campaign_worker").WithField("task", string(task.Task)) - - // decode our task body - if task.Type != queue.FireCampaignEvent { - return errors.Errorf("unknown event type passed to campaign worker: %s", task.Type) - } - eventTask := eventFireTask{} - err := json.Unmarshal(task.Task, &eventTask) - if err != nil { - return errors.Wrapf(err, "error unmarshalling event fire task: %s", string(task.Task)) - } +func (t *FireCampaignEventTask) Perform(ctx context.Context, mr *mailroom.Mailroom) error { + db := mr.DB + rp := mr.RP + log := logrus.WithField("comp", "campaign_worker").WithField("event_id", t.EventID) // grab all the fires for this event - fires, err := models.LoadEventFires(ctx, db, eventTask.FireIDs) + fires, err := models.LoadEventFires(ctx, db, t.FireIDs) if err != nil { // unmark all these fires as fires so they can retry rc := rp.Get() - for _, id := range eventTask.FireIDs { + for _, id := range t.FireIDs { rerr := marker.RemoveTask(rc, campaignsLock, fmt.Sprintf("%d", id)) if rerr != nil { log.WithError(rerr).WithField("fire_id", id).Error("error unmarking campaign fire") @@ -66,7 +66,7 @@ func fireEventFires(ctx context.Context, db *sqlx.DB, rp *redis.Pool, task *queu rc.Close() // if we had an error, return that - return errors.Wrapf(err, "error loading event fire from db: %v", eventTask.FireIDs) + return errors.Wrapf(err, "error loading event fire from db: %v", t.FireIDs) } // no fires returned @@ -80,9 +80,9 @@ func fireEventFires(ctx context.Context, db *sqlx.DB, rp *redis.Pool, task *queu contactMap[fire.ContactID] = fire } - campaign := triggers.NewCampaignReference(triggers.CampaignUUID(eventTask.CampaignUUID), eventTask.CampaignName) + campaign := triggers.NewCampaignReference(triggers.CampaignUUID(t.CampaignUUID), t.CampaignName) - started, err := runner.FireCampaignEvents(ctx, db, rp, eventTask.OrgID, fires, eventTask.FlowUUID, campaign, triggers.CampaignEventUUID(eventTask.EventUUID)) + started, err := runner.FireCampaignEvents(ctx, db, rp, t.OrgID, fires, t.FlowUUID, campaign, triggers.CampaignEventUUID(t.EventUUID)) // remove all the contacts that were started for _, contactID := range started { @@ -99,7 +99,7 @@ func fireEventFires(ctx context.Context, db *sqlx.DB, rp *redis.Pool, task *queu } if err != nil { - return errors.Wrapf(err, "error firing campaign events: %d", eventTask.FireIDs) + return errors.Wrapf(err, "error firing campaign events: %d", t.FireIDs) } return nil diff --git a/tasks/campaigns/schedule_campaign_event.go b/tasks/campaigns/schedule_campaign_event.go new file mode 100644 index 000000000..654dd9a88 --- /dev/null +++ b/tasks/campaigns/schedule_campaign_event.go @@ -0,0 +1,54 @@ +package campaigns + +import ( + "context" + "fmt" + "time" + + "github.com/nyaruka/mailroom" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/tasks" + "github.com/nyaruka/mailroom/utils/locker" + + "github.com/pkg/errors" +) + +// TypeScheduleCampaignEvent is the type of the schedule event task +const TypeScheduleCampaignEvent = "schedule_campaign_event" + +const scheduleLockKey string = "schedule_campaign_event_%d" + +func init() { + tasks.RegisterType(TypeScheduleCampaignEvent, func() tasks.Task { return &ScheduleCampaignEventTask{} }) +} + +// ScheduleCampaignEventTask is our definition of our event recalculation task +type ScheduleCampaignEventTask struct { + OrgID models.OrgID `json:"org_id"` + CampaignEventID models.CampaignEventID `json:"campaign_event_id"` +} + +// Timeout is the maximum amount of time the task can run for +func (t *ScheduleCampaignEventTask) Timeout() time.Duration { + return time.Hour +} + +// Perform creates the actual event fires to schedule the given campaign event +func (t *ScheduleCampaignEventTask) Perform(ctx context.Context, mr *mailroom.Mailroom) error { + db := mr.DB + rp := mr.RP + lockKey := fmt.Sprintf(scheduleLockKey, t.CampaignEventID) + + lock, err := locker.GrabLock(rp, lockKey, time.Hour, time.Minute*5) + if err != nil { + return errors.Wrapf(err, "error grabbing lock to schedule campaign event %d", t.CampaignEventID) + } + defer locker.ReleaseLock(rp, lockKey, lock) + + err = models.ScheduleCampaignEvent(ctx, db, t.OrgID, t.CampaignEventID) + if err != nil { + return errors.Wrapf(err, "error scheduling campaign event %d", t.CampaignEventID) + } + + return nil +} diff --git a/tasks/campaigns/schedule_campaign_event_test.go b/tasks/campaigns/schedule_campaign_event_test.go new file mode 100644 index 000000000..cc3d28f7d --- /dev/null +++ b/tasks/campaigns/schedule_campaign_event_test.go @@ -0,0 +1,130 @@ +package campaigns_test + +import ( + "testing" + "time" + + "github.com/nyaruka/gocommon/uuids" + "github.com/nyaruka/mailroom" + "github.com/nyaruka/mailroom/config" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/tasks/campaigns" + "github.com/nyaruka/mailroom/testsuite" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestScheduleCampaignEvent(t *testing.T) { + testsuite.Reset() + ctx := testsuite.CTX() + db := testsuite.DB() + mr := &mailroom.Mailroom{Config: config.Mailroom, DB: db, RP: testsuite.RP(), ElasticClient: nil} + + models.FlushCache() + + // add bob, george and alexandria to doctors group which campaign is based on + db.MustExec(`INSERT INTO contacts_contactgroup_contacts(contact_id, contactgroup_id) VALUES($1, $2)`, models.BobID, models.DoctorsGroupID) + db.MustExec(`INSERT INTO contacts_contactgroup_contacts(contact_id, contactgroup_id) VALUES($1, $2)`, models.GeorgeID, models.DoctorsGroupID) + db.MustExec(`INSERT INTO contacts_contactgroup_contacts(contact_id, contactgroup_id) VALUES($1, $2)`, models.AlexandriaID, models.DoctorsGroupID) + + // give bob and george values for joined in the future + db.MustExec(`UPDATE contacts_contact SET fields = '{"d83aae24-4bbf-49d0-ab85-6bfd201eac6d": {"datetime": "2030-01-01T00:00:00Z"}}' WHERE id = $1`, models.BobID) + db.MustExec(`UPDATE contacts_contact SET fields = '{"d83aae24-4bbf-49d0-ab85-6bfd201eac6d": {"datetime": "2030-08-18T11:31:30Z"}}' WHERE id = $1`, models.GeorgeID) + + // give alexandria a value in the past + db.MustExec(`UPDATE contacts_contact SET fields = '{"d83aae24-4bbf-49d0-ab85-6bfd201eac6d": {"datetime": "2015-01-01T00:00:00Z"}}' WHERE id = $1`, models.AlexandriaID) + + db.MustExec(`DELETE FROM campaigns_eventfire`) + + // campaign has two events configured on the joined field + // 1. +5 Days (12:00) start favorites flow + // 2. +10 Minutes send message + + // schedule first event... + task := &campaigns.ScheduleCampaignEventTask{OrgID: models.Org1, CampaignEventID: models.RemindersEvent1ID} + err := task.Perform(ctx, mr) + require.NoError(t, err) + + // cathy has no value for joined and alexandia has a value too far in past, but bob and george will have values... + assertContactFires(t, models.RemindersEvent1ID, map[models.ContactID]time.Time{ + models.BobID: time.Date(2030, 1, 5, 20, 0, 0, 0, time.UTC), // 12:00 in PST + models.GeorgeID: time.Date(2030, 8, 23, 19, 0, 0, 0, time.UTC), // 12:00 in PST with DST + }) + + // schedule second event... + task = &campaigns.ScheduleCampaignEventTask{OrgID: models.Org1, CampaignEventID: models.RemindersEvent2ID} + err = task.Perform(ctx, mr) + require.NoError(t, err) + + assertContactFires(t, models.RemindersEvent2ID, map[models.ContactID]time.Time{ + models.BobID: time.Date(2030, 1, 1, 0, 10, 0, 0, time.UTC), + models.GeorgeID: time.Date(2030, 8, 18, 11, 42, 0, 0, time.UTC), + }) + + // fires for first event unaffected + assertContactFires(t, models.RemindersEvent1ID, map[models.ContactID]time.Time{ + models.BobID: time.Date(2030, 1, 5, 20, 0, 0, 0, time.UTC), + models.GeorgeID: time.Date(2030, 8, 23, 19, 0, 0, 0, time.UTC), + }) + + // remove alexandria from campaign group + db.MustExec(`DELETE FROM contacts_contactgroup_contacts WHERE contact_id = $1`, models.AlexandriaID) + + // bump created_on for cathy and alexandria + db.MustExec(`UPDATE contacts_contact SET created_on = '2035-01-01T00:00:00Z' WHERE id = $1 OR id = $2`, models.CathyID, models.AlexandriaID) + + // create new campaign event based on created_on + 5 minutes + event3 := insertCampaignEvent(t, models.DoctorRemindersCampaignID, models.FavoritesFlowID, models.CreatedOnFieldID, 5, "M") + + task = &campaigns.ScheduleCampaignEventTask{OrgID: models.Org1, CampaignEventID: event3} + err = task.Perform(ctx, mr) + require.NoError(t, err) + + // only cathy is in the group and new enough to have a fire + assertContactFires(t, event3, map[models.ContactID]time.Time{ + models.CathyID: time.Date(2035, 1, 1, 0, 5, 0, 0, time.UTC), + }) + + // create new campaign event based on last_seen_on + 1 day + event4 := insertCampaignEvent(t, models.DoctorRemindersCampaignID, models.FavoritesFlowID, models.LastSeenOnFieldID, 1, "D") + + // bump last_seen_on for bob + db.MustExec(`UPDATE contacts_contact SET last_seen_on = '2040-01-01T00:00:00Z' WHERE id = $1`, models.BobID) + + task = &campaigns.ScheduleCampaignEventTask{OrgID: models.Org1, CampaignEventID: event4} + err = task.Perform(ctx, mr) + require.NoError(t, err) + + assertContactFires(t, event4, map[models.ContactID]time.Time{ + models.BobID: time.Date(2040, 1, 2, 0, 0, 0, 0, time.UTC), + }) +} + +func insertCampaignEvent(t *testing.T, campaignID models.CampaignID, flowID models.FlowID, relativeToID models.FieldID, offset int, unit string) models.CampaignEventID { + var eventID models.CampaignEventID + err := testsuite.DB().Get(&eventID, ` + INSERT INTO campaigns_campaignevent(is_active, created_on, modified_on, uuid, "offset", unit, event_type, delivery_hour, campaign_id, created_by_id, modified_by_id, flow_id, relative_to_id, start_mode) + VALUES(TRUE, NOW(), NOW(), $1, $5, $6, 'F', -1, $2, 1, 1, $3, $4, 'I') RETURNING id`, uuids.New(), campaignID, flowID, relativeToID, offset, unit) + require.NoError(t, err) + + return eventID +} + +func assertContactFires(t *testing.T, eventID models.CampaignEventID, expected map[models.ContactID]time.Time) { + type idAndTime struct { + ContactID models.ContactID `db:"contact_id"` + Scheduled time.Time `db:"scheduled"` + } + + actualAsSlice := make([]idAndTime, 0) + err := testsuite.DB().Select(&actualAsSlice, `SELECT contact_id, scheduled FROM campaigns_eventfire WHERE event_id = $1`, eventID) + require.NoError(t, err) + + actual := make(map[models.ContactID]time.Time) + for _, it := range actualAsSlice { + actual[it.ContactID] = it.Scheduled + } + + assert.Equal(t, expected, actual) +} diff --git a/tasks/expirations/cron_test.go b/tasks/expirations/cron_test.go index 0303c363c..f0650857c 100644 --- a/tasks/expirations/cron_test.go +++ b/tasks/expirations/cron_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/gocommon/uuids" _ "github.com/nyaruka/mailroom/hooks" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" diff --git a/tasks/groups/worker.go b/tasks/groups/populate_dynamic_group.go similarity index 62% rename from tasks/groups/worker.go rename to tasks/groups/populate_dynamic_group.go index d18dbaa46..23811eea6 100644 --- a/tasks/groups/worker.go +++ b/tasks/groups/populate_dynamic_group.go @@ -2,46 +2,41 @@ package groups import ( "context" - "encoding/json" "fmt" "time" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/models" - "github.com/nyaruka/mailroom/queue" + "github.com/nyaruka/mailroom/tasks" "github.com/nyaruka/mailroom/utils/locker" + "github.com/pkg/errors" "github.com/sirupsen/logrus" ) +// TypePopulateDynamicGroup is the type of the populate group task +const TypePopulateDynamicGroup = "populate_dynamic_group" + +const populateLockKey string = "pop_dyn_group_%d" + func init() { - mailroom.AddTaskFunction(queue.PopulateDynamicGroup, handlePopulateDynamicGroup) + tasks.RegisterType(TypePopulateDynamicGroup, func() tasks.Task { return &PopulateDynamicGroupTask{} }) } -// PopulateTask is our definition of our group population -type PopulateTask struct { +// PopulateDynamicGroupTask is our task to populate the contacts for a dynamic group +type PopulateDynamicGroupTask struct { OrgID models.OrgID `json:"org_id"` GroupID models.GroupID `json:"group_id"` Query string `json:"query"` } -const populateLockKey string = "pop_dyn_group_%d" - -// handlePopulateDynamicGroup figures out the membership for a dynamic group then repopulates it -func handlePopulateDynamicGroup(ctx context.Context, mr *mailroom.Mailroom, task *queue.Task) error { - ctx, cancel := context.WithTimeout(ctx, time.Hour) - defer cancel() - - // decode our task body - if task.Type != queue.PopulateDynamicGroup { - return errors.Errorf("unknown event type passed to populate dynamic group worker: %s", task.Type) - } - t := &PopulateTask{} - err := json.Unmarshal(task.Task, t) - if err != nil { - return errors.Wrapf(err, "error unmarshalling task: %s", string(task.Task)) - } +// Timeout is the maximum amount of time the task can run for +func (t *PopulateDynamicGroupTask) Timeout() time.Duration { + return time.Hour +} +// Perform figures out the membership for a query based group then repopulates it +func (t *PopulateDynamicGroupTask) Perform(ctx context.Context, mr *mailroom.Mailroom) error { lockKey := fmt.Sprintf(populateLockKey, t.GroupID) lock, err := locker.GrabLock(mr.RP, lockKey, time.Hour, time.Minute*5) if err != nil { diff --git a/tasks/groups/populate_dynamic_group_test.go b/tasks/groups/populate_dynamic_group_test.go new file mode 100644 index 000000000..a2f5c4f85 --- /dev/null +++ b/tasks/groups/populate_dynamic_group_test.go @@ -0,0 +1,78 @@ +package groups_test + +import ( + "fmt" + "testing" + + "github.com/nyaruka/gocommon/uuids" + "github.com/nyaruka/mailroom" + "github.com/nyaruka/mailroom/config" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/tasks/groups" + "github.com/nyaruka/mailroom/testsuite" + + "github.com/olivere/elastic" + "github.com/stretchr/testify/require" +) + +func TestPopulateTask(t *testing.T) { + testsuite.Reset() + ctx := testsuite.CTX() + db := testsuite.DB() + + mes := testsuite.NewMockElasticServer() + defer mes.Close() + + es, err := elastic.NewClient( + elastic.SetURL(mes.URL()), + elastic.SetHealthcheck(false), + elastic.SetSniff(false), + ) + require.NoError(t, err) + + mr := &mailroom.Mailroom{Config: config.Mailroom, DB: db, RP: testsuite.RP(), ElasticClient: es} + + mes.NextResponse = fmt.Sprintf(`{ + "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", + "took": 2, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": null, + "hits": [ + { + "_index": "contacts", + "_type": "_doc", + "_id": "%d", + "_score": null, + "_routing": "1", + "sort": [15124352] + } + ] + } + }`, models.CathyID) + + var groupID models.GroupID + err = db.Get(&groupID, + `INSERT INTO contacts_contactgroup(uuid, org_id, group_type, name, query, status, is_active, created_by_id, created_on, modified_by_id, modified_on) + VALUES($1, $2, 'U', $3, $4, 'R', TRUE, 1, NOW(), 1, NOW()) RETURNING id`, + uuids.New(), models.Org1, "Women", "gender = F", + ) + require.NoError(t, err) + + task := &groups.PopulateDynamicGroupTask{ + OrgID: models.Org1, + GroupID: groupID, + Query: "gender = F", + } + err = task.Perform(ctx, mr) + require.NoError(t, err) + + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM contacts_contactgroup_contacts WHERE contactgroup_id = $1`, []interface{}{groupID}, 1) +} diff --git a/tasks/handler/cron_test.go b/tasks/handler/cron_test.go index 9a7b79e4c..b761ee1ee 100644 --- a/tasks/handler/cron_test.go +++ b/tasks/handler/cron_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/gocommon/uuids" _ "github.com/nyaruka/mailroom/hooks" "github.com/nyaruka/mailroom/models" diff --git a/tasks/handler/handler_test.go b/tasks/handler/handler_test.go index fcd529ad1..d3783622c 100644 --- a/tasks/handler/handler_test.go +++ b/tasks/handler/handler_test.go @@ -7,8 +7,8 @@ import ( "time" "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/flows" - "github.com/nyaruka/goflow/utils/uuids" _ "github.com/nyaruka/mailroom/hooks" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" diff --git a/tasks/handler/worker.go b/tasks/handler/worker.go index d9199010b..9dc2af2e5 100644 --- a/tasks/handler/worker.go +++ b/tasks/handler/worker.go @@ -46,6 +46,21 @@ func AddHandleTask(rc redis.Conn, contactID models.ContactID, task *queue.Task) return addHandleTask(rc, contactID, task, false) } +// addContactTask pushes a single contact task on our queue. Note this does not push the actual content of the task +// only that a task exists for the contact, addHandleTask should be used if the task has already been pushed +// off the contact specific queue. +func addContactTask(rc redis.Conn, orgID models.OrgID, contactID models.ContactID) error { + // create our contact event + contactTask := &HandleEventTask{ContactID: contactID} + + // then add a handle task for that contact on our global handler queue + err := queue.AddTask(rc, queue.HandlerQueue, queue.HandleContactEvent, int(orgID), contactTask, queue.DefaultPriority) + if err != nil { + return errors.Wrapf(err, "error adding handle event task") + } + return nil +} + // addHandleTask adds a single task for the passed in contact. `front` specifies whether the task // should be inserted in front of all other tasks for that contact func addHandleTask(rc redis.Conn, contactID models.ContactID, task *queue.Task, front bool) error { @@ -67,15 +82,7 @@ func addHandleTask(rc redis.Conn, contactID models.ContactID, task *queue.Task, return errors.Wrapf(err, "error adding contact event") } - // create our contact event - contactTask := &HandleEventTask{ContactID: contactID} - - // then add a handle task for that contact - err = queue.AddTask(rc, queue.HandlerQueue, queue.HandleContactEvent, task.OrgID, contactTask, queue.DefaultPriority) - if err != nil { - return errors.Wrapf(err, "error adding handle event task") - } - return nil + return addContactTask(rc, models.OrgID(task.OrgID), contactID) } func handleEvent(ctx context.Context, mr *mailroom.Mailroom, task *queue.Task) error { @@ -96,12 +103,24 @@ func handleContactEvent(ctx context.Context, db *sqlx.DB, rp *redis.Pool, task * // acquire the lock for this contact lockID := models.ContactLock(models.OrgID(task.OrgID), eventTask.ContactID) - lock, err := locker.GrabLock(rp, lockID, time.Minute*5, time.Minute*5) + lock, err := locker.GrabLock(rp, lockID, time.Minute*5, time.Second*10) if err != nil { return errors.Wrapf(err, "error acquiring lock for contact %d", eventTask.ContactID) } + + // we didn't get the lock within our timeout, skip and requeue for later if lock == "" { - return errors.Errorf("unable to acquire lock for contact %d in timeout period, skipping", eventTask.ContactID) + rc := rp.Get() + defer rc.Close() + err = addContactTask(rc, models.OrgID(task.OrgID), eventTask.ContactID) + if err != nil { + return errors.Wrapf(err, "error re-adding contact task after failing to get lock") + } + logrus.WithFields(logrus.Fields{ + "org_id": task.OrgID, + "contact_id": eventTask.ContactID, + }).Info("failed to get lock for contact, requeued and skipping") + return nil } defer locker.ReleaseLock(rp, lockID, lock) diff --git a/tasks/interrupts/worker.go b/tasks/interrupts/interrupt_sessions.go similarity index 62% rename from tasks/interrupts/worker.go rename to tasks/interrupts/interrupt_sessions.go index 5179d3794..1280718e4 100644 --- a/tasks/interrupts/worker.go +++ b/tasks/interrupts/interrupt_sessions.go @@ -2,19 +2,21 @@ package interrupts import ( "context" - "encoding/json" "time" - "github.com/jmoiron/sqlx" - "github.com/lib/pq" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/models" - "github.com/nyaruka/mailroom/queue" + "github.com/nyaruka/mailroom/tasks" + + "github.com/lib/pq" "github.com/pkg/errors" ) +// TypeInterruptSessions is the type of the interrupt session task +const TypeInterruptSessions = "interrupt_sessions" + func init() { - mailroom.AddTaskFunction(queue.InterruptSessions, handleInterruptSessions) + tasks.RegisterType(TypeInterruptSessions, func() tasks.Task { return &InterruptSessionsTask{} }) } // InterruptSessionsTask is our task for interrupting sessions @@ -56,36 +58,24 @@ WHERE fs.current_flow_id = ANY($1); ` -// handleInterruptSessions interrupts all the passed in sessions -func handleInterruptSessions(ctx context.Context, mr *mailroom.Mailroom, task *queue.Task) error { - ctx, cancel := context.WithTimeout(ctx, time.Minute*60) - defer cancel() - - // decode our task body - if task.Type != queue.InterruptSessions { - return errors.Errorf("unknown event type passed to interrupt worker: %s", task.Type) - } - intTask := &InterruptSessionsTask{} - err := json.Unmarshal(task.Task, intTask) - if err != nil { - return errors.Wrapf(err, "error unmarshalling interrupt task: %s", string(task.Task)) - } - - return interruptSessions(ctx, mr.DB, intTask) +// Timeout is the maximum amount of time the task can run for +func (t *InterruptSessionsTask) Timeout() time.Duration { + return time.Hour } -// InterruptSessions interrupts all the passed in sessions -func interruptSessions(ctx context.Context, db *sqlx.DB, task *InterruptSessionsTask) error { +func (t *InterruptSessionsTask) Perform(ctx context.Context, mr *mailroom.Mailroom) error { + db := mr.DB + sessionIDs := make(map[models.SessionID]bool) - for _, sid := range task.SessionIDs { + for _, sid := range t.SessionIDs { sessionIDs[sid] = true } // if we have ivr channel ids, explode those to session ids - if len(task.ChannelIDs) > 0 { - channelSessionIDs := make([]models.SessionID, 0, len(task.ChannelIDs)) + if len(t.ChannelIDs) > 0 { + channelSessionIDs := make([]models.SessionID, 0, len(t.ChannelIDs)) - err := db.SelectContext(ctx, &channelSessionIDs, activeSessionIDsForChannelsSQL, pq.Array(task.ChannelIDs)) + err := db.SelectContext(ctx, &channelSessionIDs, activeSessionIDsForChannelsSQL, pq.Array(t.ChannelIDs)) if err != nil { return errors.Wrapf(err, "error selecting sessions for channels") } @@ -96,10 +86,10 @@ func interruptSessions(ctx context.Context, db *sqlx.DB, task *InterruptSessions } // if we have contact ids, explode those to session ids - if len(task.ContactIDs) > 0 { - contactSessionIDs := make([]models.SessionID, 0, len(task.ContactIDs)) + if len(t.ContactIDs) > 0 { + contactSessionIDs := make([]models.SessionID, 0, len(t.ContactIDs)) - err := db.SelectContext(ctx, &contactSessionIDs, activeSessionIDsForContactsSQL, pq.Array(task.ContactIDs)) + err := db.SelectContext(ctx, &contactSessionIDs, activeSessionIDsForContactsSQL, pq.Array(t.ContactIDs)) if err != nil { return errors.Wrapf(err, "error selecting sessions for contacts") } @@ -110,10 +100,10 @@ func interruptSessions(ctx context.Context, db *sqlx.DB, task *InterruptSessions } // if we have flow ids, explode those to session ids - if len(task.FlowIDs) > 0 { - flowSessionIDs := make([]models.SessionID, 0, len(task.FlowIDs)) + if len(t.FlowIDs) > 0 { + flowSessionIDs := make([]models.SessionID, 0, len(t.FlowIDs)) - err := db.SelectContext(ctx, &flowSessionIDs, activeSessionIDsForFlowsSQL, pq.Array(task.FlowIDs)) + err := db.SelectContext(ctx, &flowSessionIDs, activeSessionIDsForFlowsSQL, pq.Array(t.FlowIDs)) if err != nil { return errors.Wrapf(err, "error selecting sessions for flows") } diff --git a/tasks/interrupts/worker_test.go b/tasks/interrupts/interrupt_sessions_test.go similarity index 94% rename from tasks/interrupts/worker_test.go rename to tasks/interrupts/interrupt_sessions_test.go index dfb35b311..d60f57c30 100644 --- a/tasks/interrupts/worker_test.go +++ b/tasks/interrupts/interrupt_sessions_test.go @@ -3,7 +3,9 @@ package interrupts import ( "testing" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/gocommon/uuids" + "github.com/nyaruka/mailroom" + "github.com/nyaruka/mailroom/config" _ "github.com/nyaruka/mailroom/hooks" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" @@ -16,6 +18,8 @@ func TestInterrupts(t *testing.T) { ctx := testsuite.CTX() db := testsuite.DB() + mr := &mailroom.Mailroom{Config: config.Mailroom, DB: db, RP: testsuite.RP(), ElasticClient: nil} + insertConnection := func(orgID models.OrgID, channelID models.ChannelID, contactID models.ContactID, urnID models.URNID) models.ConnectionID { var connectionID models.ConnectionID err := db.Get(&connectionID, @@ -101,7 +105,7 @@ func TestInterrupts(t *testing.T) { } // execute it - err := interruptSessions(ctx, db, task) + err := task.Perform(ctx, mr) assert.NoError(t, err) // check session statuses are as expected diff --git a/tasks/starts/worker.go b/tasks/starts/worker.go index 58be58d36..caab3bd62 100644 --- a/tasks/starts/worker.go +++ b/tasks/starts/worker.go @@ -5,16 +5,16 @@ import ( "encoding/json" "time" - "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/contactql" - - "github.com/gomodule/redigo/redis" - "github.com/jmoiron/sqlx" - "github.com/lib/pq" + "github.com/nyaruka/goflow/envs" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" "github.com/nyaruka/mailroom/runner" + + "github.com/gomodule/redigo/redis" + "github.com/jmoiron/sqlx" + "github.com/lib/pq" "github.com/olivere/elastic" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -84,11 +84,11 @@ func CreateFlowBatches(ctx context.Context, db *sqlx.DB, rp *redis.Pool, ec *ela // if we are meant to create a new contact, do so if start.CreateContact() { - newID, err := models.CreateContact(ctx, db, oa, urns.NilURN) + contact, _, err := models.CreateContact(ctx, db, oa, models.NilUserID, "", envs.NilLanguage, nil) if err != nil { return errors.Wrapf(err, "error creating new contact") } - contactIDs[newID] = true + contactIDs[contact.ID()] = true } // now add all the ids for our groups diff --git a/tasks/starts/worker_test.go b/tasks/starts/worker_test.go index 552de3f38..82ddcb54d 100644 --- a/tasks/starts/worker_test.go +++ b/tasks/starts/worker_test.go @@ -5,7 +5,7 @@ import ( "fmt" "testing" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/mailroom" "github.com/nyaruka/mailroom/config" _ "github.com/nyaruka/mailroom/hooks" diff --git a/tasks/timeouts/cron_test.go b/tasks/timeouts/cron_test.go index 47480286c..d03175e9d 100644 --- a/tasks/timeouts/cron_test.go +++ b/tasks/timeouts/cron_test.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/gocommon/uuids" _ "github.com/nyaruka/mailroom/hooks" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/queue" diff --git a/testsuite/testsuite.go b/testsuite/testsuite.go index 9ede02a35..10d880c12 100644 --- a/testsuite/testsuite.go +++ b/testsuite/testsuite.go @@ -9,7 +9,7 @@ import ( "strings" "testing" - "github.com/nyaruka/mailroom/utils/storage" + "github.com/nyaruka/gocommon/storage" "github.com/gomodule/redigo/redis" "github.com/jmoiron/sqlx" diff --git a/utils/celery/celery.go b/utils/celery/celery.go deleted file mode 100644 index 0fce69068..000000000 --- a/utils/celery/celery.go +++ /dev/null @@ -1,116 +0,0 @@ -package celery - -import ( - "encoding/base64" - "encoding/json" - "fmt" - - "github.com/nyaruka/goflow/utils/uuids" - - "github.com/gomodule/redigo/redis" -) - -// allows queuing a task to celery (with a redis backend) -// -// format to queue a new task to the queue named "handler" at normal priority is: -// "lpush" "handler" "{"body": "W1tdLCB7fSwgeyJjYWxsYmFja3MiOiBudWxsLCAiZXJyYmFja3MiOiBudWxsLCAiY2hhaW4iOiBudWxsLCAiY2hvcmQiOiBudWxsfV0=", -// "content-encoding": "utf-8", "content-type": "application/json", "headers": {"lang": "py", "task": "handle_event_task", -// "id": "efca7c4e-952e-430f-87f7-c01c4652ed54", "eta": null, "expires": null, "group": null, "retries": 0, "timelimit": [180, 120], -// "root_id": "efca7c4e-952e-430f-87f7-c01c4652ed54", "parent_id": null, "argsrepr": "()", "kwargsrepr": "{}", -// "origin": "gen12382@ip-172-31-43-31"}, "properties": {"correlation_id": "efca7c4e-952e-430f-87f7-c01c4652ed54", -// "reply_to": "59ad710c-7d28-37c2-a730-89048c13f030", "delivery_mode": 2, "delivery_info": {"exchange": "", -// "routing_key": "handler"}, "priority": 0, "body_encoding": "base64", "delivery_tag": "bf838430-d01c-4550-b0a1-a6a309a28017"}}" -// -// multi -// "zadd" "unacked_index" "1526928218.953298" "bf838430-d01c-4550-b0a1-a6a309a28017" -// "hset" "unacked" "bf838430-d01c-4550-b0a1-a6a309a28017" "[{"body": -// "W1tdLCB7fSwgeyJjYWxsYmFja3MiOiBudWxsLCAiZXJyYmFja3MiOiBudWxsLCAiY2hhaW4iOiBudWxsLCAiY2hvcmQiOiBudWxsfV0=", -// "content-encoding": "utf-8", "content-type": "application/json", "headers": {"lang": "py", "task": "handle_event_task", -// "id": "efca7c4e-952e-430f-87f7-c01c4652ed54", "eta": null, "expires": null, "group": null, "retries": 0, "timelimit": -// [180, 120], "root_id": "efca7c4e-952e-430f-87f7-c01c4652ed54", "parent_id": null, "argsrepr": "()", "kwargsrepr": "{}", -// "origin": "gen12382@ip-172-31-43-31"}, "properties": {"correlation_id": "efca7c4e-952e-430f-87f7-c01c4652ed54", -// "reply_to": "59ad710c-7d28-37c2-a730-89048c13f030", "delivery_mode": 2, "delivery_info": {"exchange": "", "routing_key": "handler"}, -// "priority": 0, "body_encoding": "base64", "delivery_tag": "bf838430-d01c-4550-b0a1-a6a309a28017"}}, "", "handler"]" -// exec -// -// - -const defaultBody = `[%s, {}, {"chord": null, "callbacks": null, "errbacks": null, "chain": null}]` - -// QueueTask queues a new empty task with the passed in task name for the passed in queue -func QueueTask(rc redis.Conn, queueName string, taskName string, args interface{}) error { - // convert our args to json - argJSON, err := json.Marshal(args) - if err != nil { - return err - } - - body := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(defaultBody, string(argJSON)))) - taskUUID := string(uuids.New()) - deliveryTag := string(uuids.New()) - - task := Task{ - Body: body, - Headers: map[string]interface{}{ - "root_id": taskUUID, - "id": taskUUID, - "lang": "py", - "kwargsrepr": "{}", - "argsrepr": args, - "task": taskName, - "expires": nil, - "eta": nil, - "group": nil, - "origin": "courier@localhost", - "parent_id": nil, - "retries": 0, - "timelimit": []int{180, 120}, - }, - ContentType: "application/json", - Properties: TaskProperties{ - BodyEncoding: "base64", - CorrelationID: taskUUID, - ReplyTo: string(uuids.New()), - DeliveryMode: 2, - DeliveryTag: deliveryTag, - DeliveryInfo: TaskDeliveryInfo{ - RoutingKey: queueName, - }, - }, - ContentEncoding: "utf-8", - } - - taskJSON, err := json.Marshal(task) - if err != nil { - return err - } - - rc.Send("lpush", queueName, string(taskJSON)) - return nil -} - -// Task is the outer struct for a celery task -type Task struct { - Body string `json:"body"` - Headers map[string]interface{} `json:"headers"` - ContentType string `json:"content-type"` - Properties TaskProperties `json:"properties"` - ContentEncoding string `json:"content-encoding"` -} - -// TaskProperties is the struct for a task's properties -type TaskProperties struct { - BodyEncoding string `json:"body_encoding"` - CorrelationID string `json:"correlation_id"` - ReplyTo string `json:"reply_to"` - DeliveryInfo TaskDeliveryInfo `json:"delivery_info"` - DeliveryMode int `json:"delivery_mode"` - DeliveryTag string `json:"delivery_tag"` - Priority int `json:"priority"` -} - -// TaskDeliveryInfo is the struct for a task's delivery information -type TaskDeliveryInfo struct { - RoutingKey string `json:"routing_key"` - Exchange string `json:"exchange"` -} diff --git a/utils/celery/celery_test.go b/utils/celery/celery_test.go deleted file mode 100644 index 821609bcd..000000000 --- a/utils/celery/celery_test.go +++ /dev/null @@ -1,46 +0,0 @@ -package celery_test - -import ( - "encoding/json" - "testing" - - "github.com/nyaruka/mailroom/testsuite" - "github.com/nyaruka/mailroom/utils/celery" - - "github.com/gomodule/redigo/redis" -) - -func TestQueue(t *testing.T) { - testsuite.ResetRP() - rc := testsuite.RC() - defer rc.Close() - - // queue to our handler queue - rc.Send("multi") - err := celery.QueueTask(rc, "handler", "handle_event_task", []int64{}) - if err != nil { - t.Error(err) - } - _, err = rc.Do("exec") - if err != nil { - t.Error(err) - } - - // check whether things look right - taskJSON, err := redis.String(rc.Do("LPOP", "handler")) - if err != nil { - t.Errorf("should have value in handler queue: %s", err) - } - - // make sure our task is valid json - task := celery.Task{} - err = json.Unmarshal([]byte(taskJSON), &task) - if err != nil { - t.Errorf("should be JSON: %s", err) - } - - // and is against the right queue - if task.Properties.DeliveryInfo.RoutingKey != "handler" { - t.Errorf("task should have handler as routing key") - } -} diff --git a/utils/dbutil/errors.go b/utils/dbutil/errors.go new file mode 100644 index 000000000..fb20bd441 --- /dev/null +++ b/utils/dbutil/errors.go @@ -0,0 +1,11 @@ +package dbutil + +import "github.com/lib/pq" + +// IsUniqueViolation returns true if the given error is a violation of unique constraint +func IsUniqueViolation(err error) bool { + if pqErr, ok := err.(*pq.Error); ok { + return pqErr.Code.Name() == "unique_violation" + } + return false +} diff --git a/utils/dbutil/errors_test.go b/utils/dbutil/errors_test.go new file mode 100644 index 000000000..f06b2ce32 --- /dev/null +++ b/utils/dbutil/errors_test.go @@ -0,0 +1,18 @@ +package dbutil_test + +import ( + "testing" + + "github.com/lib/pq" + "github.com/nyaruka/mailroom/utils/dbutil" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +func TestIsUniqueViolation(t *testing.T) { + var err error = &pq.Error{Code: pq.ErrorCode("23505")} + + assert.True(t, dbutil.IsUniqueViolation(err)) + assert.False(t, dbutil.IsUniqueViolation(errors.New("boom"))) +} diff --git a/utils/dbutil/json.go b/utils/dbutil/json.go new file mode 100644 index 000000000..f862373ce --- /dev/null +++ b/utils/dbutil/json.go @@ -0,0 +1,33 @@ +package dbutil + +import ( + "encoding/json" + + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" + "gopkg.in/go-playground/validator.v9" +) + +var validate = validator.New() + +// ReadJSONRow reads a row which is JSON into a destination struct +func ReadJSONRow(rows *sqlx.Rows, destination interface{}) error { + var jsonBlob string + err := rows.Scan(&jsonBlob) + if err != nil { + return errors.Wrap(err, "error scanning row JSON") + } + + err = json.Unmarshal([]byte(jsonBlob), destination) + if err != nil { + return errors.Wrap(err, "error unmarshalling row JSON") + } + + // validate our final struct + err = validate.Struct(destination) + if err != nil { + return errors.Wrapf(err, "failed validation for JSON: %s", jsonBlob) + } + + return nil +} diff --git a/utils/dbutil/json_test.go b/utils/dbutil/json_test.go new file mode 100644 index 000000000..c578f31dd --- /dev/null +++ b/utils/dbutil/json_test.go @@ -0,0 +1,50 @@ +package dbutil_test + +import ( + "testing" + + "github.com/jmoiron/sqlx" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/utils/dbutil" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReadJSONRow(t *testing.T) { + ctx := testsuite.CTX() + db := testsuite.DB() + + type group struct { + UUID string `json:"uuid"` + Name string `json:"name"` + } + + queryRows := func(sql string, args ...interface{}) *sqlx.Rows { + rows, err := db.QueryxContext(ctx, sql, args...) + require.NoError(t, err) + require.True(t, rows.Next()) + return rows + } + + // if query returns valid JSON which can be unmarshaled into our struct, all good + rows := queryRows(`SELECT ROW_TO_JSON(r) FROM (SELECT g.uuid as uuid, g.name AS name FROM contacts_contactgroup g WHERE id = $1) r`, models.TestersGroupID) + + g := &group{} + err := dbutil.ReadJSONRow(rows, g) + assert.NoError(t, err) + assert.Equal(t, "5e9d8fab-5e7e-4f51-b533-261af5dea70d", g.UUID) + assert.Equal(t, "Testers", g.Name) + + // error if row value is not JSON + rows = queryRows(`SELECT id FROM contacts_contactgroup g WHERE id = $1`, models.TestersGroupID) + err = dbutil.ReadJSONRow(rows, g) + assert.EqualError(t, err, "error unmarshalling row JSON: json: cannot unmarshal number into Go value of type dbutil_test.group") + + // error if rows aren't ready to be scanned - e.g. next hasn't been called + rows, err = db.QueryxContext(ctx, `SELECT ROW_TO_JSON(r) FROM (SELECT g.uuid as uuid, g.name AS name FROM contacts_contactgroup g WHERE id = $1) r`, models.TestersGroupID) + require.NoError(t, err) + err = dbutil.ReadJSONRow(rows, g) + assert.EqualError(t, err, "error scanning row JSON: sql: Scan called without calling Next") +} diff --git a/utils/dbutil/query.go b/utils/dbutil/query.go new file mode 100644 index 000000000..c7b00bbf1 --- /dev/null +++ b/utils/dbutil/query.go @@ -0,0 +1,132 @@ +package dbutil + +import ( + "context" + "strings" + + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" +) + +// Queryer is the DB/TX functionality needed for operations in this package +type Queryer interface { + Rebind(query string) string + QueryxContext(ctx context.Context, query string, args ...interface{}) (*sqlx.Rows, error) +} + +// BulkQuery runs the query as a bulk operation with the given structs +func BulkQuery(ctx context.Context, tx Queryer, query string, structs []interface{}) error { + // no structs, nothing to do + if len(structs) == 0 { + return nil + } + + // rewrite query as a bulk operation + bulkQuery, args, err := BulkSQL(tx, query, structs) + if err != nil { + return err + } + + rows, err := tx.QueryxContext(ctx, bulkQuery, args...) + if err != nil { + return errors.Wrapf(err, "error making bulk query: %.100s", bulkQuery) + } + defer rows.Close() + + // if have a returning clause, read them back and try to map them + if strings.Contains(strings.ToUpper(query), "RETURNING") { + for _, s := range structs { + if !rows.Next() { + return errors.Errorf("did not receive expected number of rows on insert") + } + + err = rows.StructScan(s) + if err != nil { + return errors.Wrap(err, "error scanning for insert id") + } + } + } + + // iterate our remaining rows + for rows.Next() { + } + + // check for any error + if rows.Err() != nil { + return errors.Wrapf(rows.Err(), "error in row cursor") + } + + return nil +} + +// BulkSQL takes a query which uses VALUES with struct bindings and rewrites it as a bulk operation. +// It returns the new SQL query and the args to pass to it. +func BulkSQL(tx Queryer, sql string, structs []interface{}) (string, []interface{}, error) { + if len(structs) == 0 { + return "", nil, errors.New("can't generate bulk sql with zero structs") + } + + // this will be our SQL placeholders for values in our final query, built dynamically + values := strings.Builder{} + values.Grow(7 * len(structs)) + + // this will be each of the arguments to match the positional values above + args := make([]interface{}, 0, len(structs)*5) + + // for each value we build a bound SQL statement, then extract the values clause + for i, value := range structs { + valueSQL, valueArgs, err := sqlx.Named(sql, value) + if err != nil { + return "", nil, errors.Wrapf(err, "error converting bulk insert args") + } + + args = append(args, valueArgs...) + argValues := extractValues(valueSQL) + if argValues == "" { + return "", nil, errors.Errorf("error extracting VALUES from sql: %s", valueSQL) + } + + // append to our global values, adding comma if necessary + values.WriteString(argValues) + if i+1 < len(structs) { + values.WriteString(",") + } + } + + valuesSQL := extractValues(sql) + if valuesSQL == "" { + return "", nil, errors.Errorf("error extracting VALUES from sql: %s", sql) + } + + return tx.Rebind(strings.Replace(sql, valuesSQL, values.String(), -1)), args, nil +} + +// extractValues is just a simple utility method that extracts the portion between `VALUE(` +// and `)` in the passed in string. (leaving VALUE but not the parentheses) +func extractValues(sql string) string { + startValues := strings.Index(sql, "VALUES(") + if startValues <= 0 { + return "" + } + + // find the matching end parentheses, we need to count balances here + openCount := 1 + endValues := -1 + for i, r := range sql[startValues+7:] { + if r == '(' { + openCount++ + } else if r == ')' { + openCount-- + if openCount == 0 { + endValues = i + startValues + 7 + break + } + } + } + + if endValues <= 0 { + return "" + } + + return sql[startValues+6 : endValues+1] +} diff --git a/utils/dbutil/query_test.go b/utils/dbutil/query_test.go new file mode 100644 index 000000000..5c3801335 --- /dev/null +++ b/utils/dbutil/query_test.go @@ -0,0 +1,78 @@ +package dbutil_test + +import ( + "testing" + + "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/utils/dbutil" + + _ "github.com/lib/pq" + "github.com/stretchr/testify/assert" +) + +func TestBulkSQL(t *testing.T) { + db := testsuite.DB() + + type contact struct { + ID int `db:"id"` + Name string `db:"name"` + } + + _, _, err := dbutil.BulkSQL(db, `UPDATE contact_contact SET name = :name WHERE id = :id`, []interface{}{contact{ID: 1, Name: "Bob"}}) + assert.EqualError(t, err, "error extracting VALUES from sql: UPDATE contact_contact SET name = ? WHERE id = ?") + + sql := `INSERT INTO contacts_contact (id, name) VALUES(:id, :name)` + + // try with zero structs + query, args, err := dbutil.BulkSQL(db, sql, []interface{}{}) + assert.EqualError(t, err, "can't generate bulk sql with zero structs") + + // try with one struct + query, args, err = dbutil.BulkSQL(db, sql, []interface{}{contact{ID: 1, Name: "Bob"}}) + assert.NoError(t, err) + assert.Equal(t, `INSERT INTO contacts_contact (id, name) VALUES($1, $2)`, query) + assert.Equal(t, []interface{}{1, "Bob"}, args) + + // try with multiple... + query, args, err = dbutil.BulkSQL(db, sql, []interface{}{contact{ID: 1, Name: "Bob"}, contact{ID: 2, Name: "Cathy"}, contact{ID: 3, Name: "George"}}) + assert.NoError(t, err) + assert.Equal(t, `INSERT INTO contacts_contact (id, name) VALUES($1, $2),($3, $4),($5, $6)`, query) + assert.Equal(t, []interface{}{1, "Bob", 2, "Cathy", 3, "George"}, args) +} + +func TestBulkQuery(t *testing.T) { + ctx := testsuite.CTX() + db := testsuite.DB() + defer testsuite.Reset() + + db.MustExec(`CREATE TABLE foo (id serial NOT NULL PRIMARY KEY, name TEXT, age INT)`) + + type foo struct { + ID int `db:"id"` + Name string `db:"name"` + Age int `db:"age"` + } + + sql := `INSERT INTO foo (name, age) VALUES(:name, :age) RETURNING id` + + // noop with zero structs + err := dbutil.BulkQuery(ctx, db, sql, nil) + assert.NoError(t, err) + + // returned ids are scanned into structs + foo1 := &foo{Name: "Bob", Age: 64} + foo2 := &foo{Name: "Jon", Age: 34} + err = dbutil.BulkQuery(ctx, db, sql, []interface{}{foo1, foo2}) + assert.NoError(t, err) + assert.Equal(t, 1, foo1.ID) + assert.Equal(t, 2, foo2.ID) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM foo WHERE name = 'Bob' AND age = 64`, nil, 1) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM foo WHERE name = 'Jon' AND age = 34`, nil, 1) + + // returning ids is optional + foo3 := &foo{Name: "Jim", Age: 54} + err = dbutil.BulkQuery(ctx, db, `INSERT INTO foo (name, age) VALUES(:name, :age)`, []interface{}{foo3}) + assert.NoError(t, err) + assert.Equal(t, 0, foo3.ID) + testsuite.AssertQueryCount(t, db, `SELECT count(*) FROM foo WHERE name = 'Jim' AND age = 54`, nil, 1) +} diff --git a/utils/gsm7/gsm7.go b/utils/gsm7/gsm7.go deleted file mode 100644 index cd136fdcb..000000000 --- a/utils/gsm7/gsm7.go +++ /dev/null @@ -1,295 +0,0 @@ -package gsm7 - -// base gsm7 characters in our normal table -var baseGSM7 = map[rune]byte{ - '@': 0x00, - '£': 0x01, - '$': 0x02, - '¥': 0x03, - 'è': 0x04, - 'é': 0x05, - 'ù': 0x06, - 'ì': 0x07, - 'ò': 0x08, - 'Ç': 0x09, - '\n': 0x0A, - 'Ø': 0x0B, - 'ø': 0x0C, - '\r': 0x0D, - 'Å': 0x0E, - 'å': 0x0F, - 'Δ': 0x10, - '_': 0x11, - 'Φ': 0x12, - 'Γ': 0x13, - 'Λ': 0x14, - 'Ω': 0x15, - 'Π': 0x16, - 'Ψ': 0x17, - 'Σ': 0x18, - 'Θ': 0x19, - 'Ξ': 0x1A, - // 'ESC': 0x1B, // Escape control - 'Æ': 0x1C, - 'æ': 0x1D, - 'ß': 0x1E, - 'É': 0x1F, - ' ': 0x20, - '!': 0x21, - '"': 0x22, - '#': 0x23, - '¤': 0x24, - '%': 0x25, - '&': 0x26, - '\'': 0x27, - '(': 0x28, - ')': 0x29, - '*': 0x2A, - '+': 0x2B, - ',': 0x2C, - '-': 0x2D, - '.': 0x2E, - '/': 0x2F, - '0': 0x30, - '1': 0x31, - '2': 0x32, - '3': 0x33, - '4': 0x34, - '5': 0x35, - '6': 0x36, - '7': 0x37, - '8': 0x38, - '9': 0x39, - ':': 0x3A, - ';': 0x3B, - '<': 0x3C, - '=': 0x3D, - '>': 0x3E, - '?': 0x3F, - '¡': 0x40, - 'A': 0x41, - 'B': 0x42, - 'C': 0x43, - 'D': 0x44, - 'E': 0x45, - 'F': 0x46, - 'G': 0x47, - 'H': 0x48, - 'I': 0x49, - 'J': 0x4A, - 'K': 0x4B, - 'L': 0x4C, - 'M': 0x4D, - 'N': 0x4E, - 'O': 0x4F, - 'P': 0x50, - 'Q': 0x51, - 'R': 0x52, - 'S': 0x53, - 'T': 0x54, - 'U': 0x55, - 'V': 0x56, - 'W': 0x57, - 'X': 0x58, - 'Y': 0x59, - 'Z': 0x5A, - 'Ä': 0x5B, - 'Ö': 0x5C, - 'Ñ': 0x5D, - 'Ü': 0x5E, - '§': 0x5F, - '¿': 0x60, - 'a': 0x61, - 'b': 0x62, - 'c': 0x63, - 'd': 0x64, - 'e': 0x65, - 'f': 0x66, - 'g': 0x67, - 'h': 0x68, - 'i': 0x69, - 'j': 0x6A, - 'k': 0x6B, - 'l': 0x6C, - 'm': 0x6D, - 'n': 0x6E, - 'o': 0x6F, - 'p': 0x70, - 'q': 0x71, - 'r': 0x72, - 's': 0x73, - 't': 0x74, - 'u': 0x75, - 'v': 0x76, - 'w': 0x77, - 'x': 0x78, - 'y': 0x79, - 'z': 0x7A, - 'ä': 0x7B, - 'ö': 0x7C, - 'ñ': 0x7D, - 'ü': 0x7E, - 'à': 0x7F, -} - -// extended gsm7 characters, these my be preceded by our escape -var extendedGSM7 = map[rune]byte{ - ' ': 0x0A, - '^': 0x14, - '{': 0x28, - '}': 0x29, - '\\': 0x2F, - '[': 0x3C, - '~': 0x3D, - ']': 0x3E, - '|': 0x40, - '€': 0x65, -} - -// Characters we replace in GSM7 with versions that can actually be encoded -var gsm7Replacements = map[rune]rune{ - 'á': 'a', - 'ê': 'e', - 'ã': 'a', - 'â': 'a', - 'ç': 'c', - 'í': 'i', - 'î': 'i', - 'ú': 'u', - 'û': 'u', - 'õ': 'o', - 'ô': 'o', - 'ó': 'o', - - 'Á': 'A', - 'Â': 'A', - 'Ã': 'A', - 'À': 'A', - 'Ç': 'C', - 'È': 'E', - 'Ê': 'E', - 'Í': 'I', - 'Î': 'I', - 'Ì': 'I', - 'Ó': 'O', - 'Ô': 'O', - 'Ò': 'O', - 'Õ': 'O', - 'Ú': 'U', - 'Ù': 'U', - 'Û': 'U', - - // shit Word likes replacing automatically - '’': '\'', - '‘': '\'', - '“': '"', - '”': '"', - '–': '-', - '\xa0': ' ', -} - -// esc is our escape byte for the extended charset -const esc byte = 0x1B - -// unknown is the rune we replace invalid characters with -const unknown byte = '?' - -// max GSM7 value -const max byte = 0x7F - -// our reverse mapping from GSM7 byte to rune -var gsm7ToBase = make(map[byte]rune) -var gsm7ToExtended = make(map[byte]rune) - -// we create our reverse mappings in our init -func init() { - for r, b := range baseGSM7 { - gsm7ToBase[b] = r - } - - for r, b := range extendedGSM7 { - gsm7ToExtended[b] = r - } -} - -// IsValid returns whether the passed in string is made up of entirely GSM7 characters -func IsValid(text string) bool { - for _, r := range text { - _, present := baseGSM7[r] - if !present { - _, present = extendedGSM7[r] - if !present { - return false - } - } - } - return true -} - -// Segments calculates the number of SMS segments it will take to send the passed in text. -// This automatically figures out if the text is GSM7 or UCS2 and then calculates how many segments it -// will break up into. -// -// UCS2 messages can be 70 characters per segnment max, if more, each segment is 67 -// GSM7 messages can be 160 characters per segmend max, if more, each segment is 153 -// -// TODO: likely some optimizations could be made here by comparing ranges instead of map lookups -func Segments(text string) int { - // are we ucs2? - isGSM7 := IsValid(text) - - // first figure out if we are multipart - isMultipart := false - size := 0 - - for _, c := range text { - _, isExtended := extendedGSM7[c] - if isExtended && isGSM7 { - size += 2 - } else { - size += 1 - } - - if !isGSM7 && size > 70 { - isMultipart = true - break - } - - if isGSM7 && size > 160 { - isMultipart = true - break - } - } - - // we aren't multipart, so just return a single segment - if !isMultipart { - return 1 - } - - // our current segment count - segments := 1 - size = 0 - - // calculate our total number of segments, we can't do simple division because multibyte extended chars - // may land on a boundary between messages (`{` as character 153 will not fit and force another segment) - for _, c := range text { - _, isExtended := extendedGSM7[c] - if isExtended && isGSM7 { - size += 2 - } else { - size += 1 - } - - if isGSM7 && size > 153 { - size -= 153 - segments += 1 - } - - if !isGSM7 && size > 67 { - size -= 67 - segments += 1 - } - } - - return segments -} diff --git a/utils/gsm7/gsm7_test.go b/utils/gsm7/gsm7_test.go deleted file mode 100644 index 7aa50ee48..000000000 --- a/utils/gsm7/gsm7_test.go +++ /dev/null @@ -1,40 +0,0 @@ -package gsm7_test - -import ( - "testing" - - "github.com/nyaruka/mailroom/utils/gsm7" - - "github.com/stretchr/testify/assert" -) - -func TestSegments(t *testing.T) { - // utility pads - tenChars := "0123456789" - unicodeTenChars := "☺123456789" - extendedTenChars := "[123456789" - fiftyChars := tenChars + tenChars + tenChars + tenChars + tenChars - hundredChars := fiftyChars + fiftyChars - unicode := "☺" - - tcs := []struct { - Text string - Segments int - }{ - {"", 1}, - {"hello", 1}, - {"“word”", 1}, - {hundredChars + fiftyChars + tenChars, 1}, - {hundredChars + fiftyChars + tenChars + "Z", 2}, - {hundredChars + fiftyChars + extendedTenChars, 2}, - {hundredChars + hundredChars + hundredChars + "123456", 2}, - {hundredChars + hundredChars + hundredChars + "1234567", 3}, - {fiftyChars + "zZ" + unicode, 1}, - {fiftyChars + tenChars + unicodeTenChars, 1}, - {fiftyChars + tenChars + unicodeTenChars + "z", 2}, - } - - for _, tc := range tcs { - assert.Equal(t, tc.Segments, gsm7.Segments(tc.Text), "unexpected num of segments for: %s", tc.Text) - } -} diff --git a/utils/storage/base.go b/utils/storage/base.go deleted file mode 100644 index c074046ec..000000000 --- a/utils/storage/base.go +++ /dev/null @@ -1,20 +0,0 @@ -package storage - -import ( - "github.com/nyaruka/mailroom/config" -) - -// Storage is the interface that provides storage of atttachments etc -type Storage interface { - Name() string - Test() error - Put(path string, contentType string, contents []byte) (string, error) -} - -// New creates a new storage service -func New(cfg *config.Config) (Storage, error) { - if cfg.AWSAccessKeyID != "" && cfg.AWSSecretAccessKey != "" { - return NewS3(cfg) - } - return NewFS("_storage"), nil -} diff --git a/utils/storage/fs.go b/utils/storage/fs.go deleted file mode 100644 index d6f492bda..000000000 --- a/utils/storage/fs.go +++ /dev/null @@ -1,47 +0,0 @@ -package storage - -import ( - "io/ioutil" - "os" - "path/filepath" -) - -type fsStorage struct { - directory string - perms os.FileMode -} - -// NewFS creates a new file system storage service suitable for use in tests -func NewFS(directory string) Storage { - return &fsStorage{directory: directory, perms: 0766} -} - -func (s *fsStorage) Name() string { - return "file system" -} - -func (s *fsStorage) Test() error { - path, err := s.Put("test.txt", "text/plain", []byte(`test`)) - if err != nil { - return err - } - - os.Remove(path) - return nil -} - -func (s *fsStorage) Put(path string, contentType string, contents []byte) (string, error) { - fullPath := filepath.Join(s.directory, path) - - err := os.MkdirAll(filepath.Dir(fullPath), s.perms) - if err != nil { - return "", err - } - - err = ioutil.WriteFile(fullPath, contents, s.perms) - if err != nil { - return "", err - } - - return fullPath, nil -} diff --git a/utils/storage/fs_test.go b/utils/storage/fs_test.go deleted file mode 100644 index af761f681..000000000 --- a/utils/storage/fs_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package storage_test - -import ( - "io/ioutil" - "os" - "testing" - - "github.com/nyaruka/mailroom/utils/storage" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestFS(t *testing.T) { - s := storage.NewFS("_testing") - assert.NoError(t, s.Test()) - - // break our ability to write to that directory - require.NoError(t, os.Chmod("_testing", 0555)) - - assert.EqualError(t, s.Test(), "open _testing/test.txt: permission denied") - - require.NoError(t, os.Chmod("_testing", 0777)) - - url, err := s.Put("/foo/bar.txt", "text/plain", []byte(`hello world`)) - assert.NoError(t, err) - assert.Equal(t, "_testing/foo/bar.txt", url) - - data, err := ioutil.ReadFile(url) - assert.NoError(t, err) - assert.Equal(t, []byte(`hello world`), data) - - require.NoError(t, os.RemoveAll("_testing")) -} diff --git a/utils/storage/s3.go b/utils/storage/s3.go deleted file mode 100644 index 9930aeeb7..000000000 --- a/utils/storage/s3.go +++ /dev/null @@ -1,73 +0,0 @@ -package storage - -import ( - "bytes" - "fmt" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/s3" - "github.com/aws/aws-sdk-go/service/s3/s3iface" - "github.com/nyaruka/mailroom/config" - "github.com/sirupsen/logrus" -) - -var s3BucketURL = "https://%s.s3.amazonaws.com%s" - -type s3Storage struct { - client s3iface.S3API - bucket string -} - -// NewS3 creates a new S3 storage service -func NewS3(cfg *config.Config) (Storage, error) { - s3Session, err := session.NewSession(&aws.Config{ - Credentials: credentials.NewStaticCredentials(cfg.AWSAccessKeyID, cfg.AWSSecretAccessKey, ""), - Endpoint: aws.String(cfg.S3Endpoint), - Region: aws.String(cfg.S3Region), - DisableSSL: aws.Bool(cfg.S3DisableSSL), - S3ForcePathStyle: aws.Bool(cfg.S3ForcePathStyle), - }) - if err != nil { - return nil, err - } - - return &s3Storage{ - client: s3.New(s3Session), - bucket: cfg.S3MediaBucket, - }, nil -} - -func (s *s3Storage) Name() string { - return "S3" -} - -// Test tests whether our S3 client is properly configured -func (s *s3Storage) Test() error { - params := &s3.HeadBucketInput{ - Bucket: aws.String(s.bucket), - } - _, err := s.client.HeadBucket(params) - return err -} - -// Put writes the passed in file to the bucket with the passed in content type -func (s *s3Storage) Put(path string, contentType string, contents []byte) (string, error) { - params := &s3.PutObjectInput{ - Bucket: aws.String(s.bucket), - Body: bytes.NewReader(contents), - Key: aws.String(path), - ContentType: aws.String(contentType), - ACL: aws.String(s3.BucketCannedACLPublicRead), - } - - logrus.WithField("path", path).Info("** uploading s3 file") - - _, err := s.client.PutObject(params) - if err != nil { - return "", err - } - - return fmt.Sprintf(s3BucketURL, s.bucket, path), nil -} diff --git a/web/contact/contact.go b/web/contact/contact.go index 5b3ee550b..f06fefa87 100644 --- a/web/contact/contact.go +++ b/web/contact/contact.go @@ -5,8 +5,6 @@ import ( "encoding/json" "net/http" - "github.com/nyaruka/goflow/assets" - "github.com/nyaruka/goflow/contactql" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils" "github.com/nyaruka/mailroom/goflow" @@ -17,208 +15,60 @@ import ( ) func init() { - web.RegisterJSONRoute(http.MethodPost, "/mr/contact/search", web.RequireAuthToken(handleSearch)) - web.RegisterJSONRoute(http.MethodPost, "/mr/contact/parse_query", web.RequireAuthToken(handleParseQuery)) + web.RegisterJSONRoute(http.MethodPost, "/mr/contact/create", web.RequireAuthToken(handleCreate)) web.RegisterJSONRoute(http.MethodPost, "/mr/contact/modify", web.RequireAuthToken(handleModify)) } -// Searches the contacts for an org +// Request to create a new contact. // // { // "org_id": 1, -// "group_uuid": "985a83fe-2e9f-478d-a3ec-fa602d5e7ddd", -// "query": "age > 10", -// "sort": "-age" +// "user_id": 1, +// "contact": { +// "name": "Joe Blow", +// "language": "eng", +// "urns": ["tel:+250788123123"], +// "fields": {"age": "39"}, +// "groups": ["b0b778db-6657-430b-9272-989ad43a10db"] +// } // } // -type searchRequest struct { - OrgID models.OrgID `json:"org_id" validate:"required"` - GroupUUID assets.GroupUUID `json:"group_uuid" validate:"required"` - Query string `json:"query"` - PageSize int `json:"page_size"` - Offset int `json:"offset"` - Sort string `json:"sort"` -} - -// Response for a contact search -// -// { -// "query": "age > 10", -// "contact_ids": [5,10,15], -// "total": 3, -// "offset": 0, -// "metadata": { -// "fields": [ -// {"key": "age", "name": "Age"} -// ], -// "allow_as_group": true -// } -// } -type searchResponse struct { - Query string `json:"query"` - ContactIDs []models.ContactID `json:"contact_ids"` - Total int64 `json:"total"` - Offset int `json:"offset"` - Sort string `json:"sort"` - Metadata *contactql.Inspection `json:"metadata,omitempty"` - - // deprecated - Fields []string `json:"fields"` - AllowAsGroup bool `json:"allow_as_group"` +type createRequest struct { + OrgID models.OrgID `json:"org_id" validate:"required"` + UserID models.UserID `json:"user_id"` + Contact *Spec `json:"contact" validate:"required"` } -// handles a contact search request -func handleSearch(ctx context.Context, s *web.Server, r *http.Request) (interface{}, int, error) { - request := &searchRequest{ - Offset: 0, - PageSize: 50, - Sort: "-id", - } +// handles a request to create the given contacts +func handleCreate(ctx context.Context, s *web.Server, r *http.Request) (interface{}, int, error) { + request := &createRequest{} if err := utils.UnmarshalAndValidateWithLimit(r.Body, request, web.MaxRequestBytes); err != nil { return errors.Wrapf(err, "request failed validation"), http.StatusBadRequest, nil } - // grab our org assets - oa, err := models.GetOrgAssetsWithRefresh(s.CTX, s.DB, request.OrgID, models.RefreshFields|models.RefreshGroups) + // grab our org + oa, err := models.GetOrgAssets(s.CTX, s.DB, request.OrgID) if err != nil { return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load org assets") } - // Perform our search - parsed, hits, total, err := models.ContactIDsForQueryPage(ctx, s.ElasticClient, oa, - request.GroupUUID, request.Query, request.Sort, request.Offset, request.PageSize) - - if err != nil { - isQueryError, qerr := contactql.IsQueryError(err) - if isQueryError { - return qerr, http.StatusBadRequest, nil - } - return nil, http.StatusInternalServerError, err - } - - // normalize and inspect the query - normalized := "" - var metadata *contactql.Inspection - allowAsGroup := false - fields := make([]string, 0) - - if parsed != nil { - normalized = parsed.String() - metadata = contactql.Inspect(parsed) - fields = append(fields, metadata.Attributes...) - for _, f := range metadata.Fields { - fields = append(fields, f.Key) - } - allowAsGroup = metadata.AllowAsGroup - } - - // build our response - response := &searchResponse{ - Query: normalized, - ContactIDs: hits, - Total: total, - Offset: request.Offset, - Sort: request.Sort, - Metadata: metadata, - Fields: fields, - AllowAsGroup: allowAsGroup, - } - - return response, http.StatusOK, nil -} - -// Request to parse the passed in query -// -// { -// "org_id": 1, -// "query": "age > 10", -// "group_uuid": "123123-123-123-" -// } -// -type parseRequest struct { - OrgID models.OrgID `json:"org_id" validate:"required"` - Query string `json:"query" validate:"required"` - GroupUUID assets.GroupUUID `json:"group_uuid"` -} - -// Response for a parse query request -// -// { -// "query": "age > 10", -// "elastic_query": { .. }, -// "metadata": { -// "fields": [ -// {"key": "age", "name": "Age"} -// ], -// "allow_as_group": true -// } -// } -type parseResponse struct { - Query string `json:"query"` - ElasticQuery interface{} `json:"elastic_query"` - Metadata *contactql.Inspection `json:"metadata,omitempty"` - - // deprecated - Fields []string `json:"fields"` - AllowAsGroup bool `json:"allow_as_group"` -} - -// handles a query parsing request -func handleParseQuery(ctx context.Context, s *web.Server, r *http.Request) (interface{}, int, error) { - request := &parseRequest{} - if err := utils.UnmarshalAndValidateWithLimit(r.Body, request, web.MaxRequestBytes); err != nil { - return errors.Wrapf(err, "request failed validation"), http.StatusBadRequest, nil - } - - // grab our org assets - oa, err := models.GetOrgAssetsWithRefresh(s.CTX, s.DB, request.OrgID, models.RefreshFields|models.RefreshGroups) + c, err := request.Contact.Validate(oa.Env(), oa.SessionAssets()) if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load org assets") + return err, http.StatusBadRequest, nil } - env := oa.Env() - parsed, err := contactql.ParseQuery(env, request.Query, oa.SessionAssets()) - + _, contact, err := models.CreateContact(ctx, s.DB, oa, request.UserID, c.Name, c.Language, c.URNs) if err != nil { - isQueryError, qerr := contactql.IsQueryError(err) - if isQueryError { - return qerr, http.StatusBadRequest, nil - } - return nil, http.StatusInternalServerError, err + return err, http.StatusBadRequest, nil } - // normalize and inspect the query - normalized := "" - var metadata *contactql.Inspection - allowAsGroup := false - fields := make([]string, 0) - - if parsed != nil { - normalized = parsed.String() - metadata = contactql.Inspect(parsed) - fields = append(fields, metadata.Attributes...) - for _, f := range metadata.Fields { - fields = append(fields, f.Key) - } - allowAsGroup = metadata.AllowAsGroup - } - - eq := models.BuildElasticQuery(oa, request.GroupUUID, parsed) - eqj, err := eq.Source() + modifiersByContact := map[*flows.Contact][]flows.Modifier{contact: c.Mods} + _, err = ModifyContacts(ctx, s.DB, s.RP, oa, modifiersByContact) if err != nil { - return nil, http.StatusInternalServerError, err - } - - // build our response - response := &parseResponse{ - Query: normalized, - ElasticQuery: eqj, - Metadata: metadata, - Fields: fields, - AllowAsGroup: allowAsGroup, + return nil, http.StatusInternalServerError, errors.Wrap(err, "error modifying new contact") } - return response, http.StatusOK, nil + return map[string]interface{}{"contact": contact}, http.StatusOK, nil } // Request that a set of contacts is modified. @@ -293,40 +143,32 @@ func handleModify(ctx context.Context, s *web.Server, r *http.Request) (interfac // load our contacts contacts, err := models.LoadContacts(ctx, s.DB, oa, request.ContactIDs) if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load contact") + return nil, http.StatusBadRequest, errors.Wrapf(err, "unable to load contact") } - results := make(map[models.ContactID]modifyResult) - - // create an environment instance with location support - env := flows.NewEnvironment(oa.Env(), oa.SessionAssets().Locations()) - - // gather up events for our contacts - contactEvents := make(map[*flows.Contact][]flows.Event, len(contacts)) - + // convert to map of flow contacts to modifiers + modifiersByContact := make(map[*flows.Contact][]flows.Modifier, len(contacts)) for _, contact := range contacts { flowContact, err := contact.FlowContact(oa) if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "error creating flow contact for contact: %d", contact.ID()) - } - - result := modifyResult{ - Contact: flowContact, - Events: make([]flows.Event, 0, len(mods)), - } - - // apply our modifiers - for _, mod := range mods { - mod.Apply(env, oa.SessionAssets(), flowContact, func(e flows.Event) { result.Events = append(result.Events, e) }) + return nil, http.StatusBadRequest, errors.Wrapf(err, "error creating flow contact for contact: %d", contact.ID()) } - results[contact.ID()] = result - contactEvents[flowContact] = result.Events + modifiersByContact[flowContact] = mods } - err = models.HandleAndCommitEvents(ctx, s.DB, s.RP, oa, contactEvents) + eventsByContact, err := ModifyContacts(ctx, s.DB, s.RP, oa, modifiersByContact) if err != nil { - return nil, http.StatusInternalServerError, err + return nil, http.StatusBadRequest, err + } + + // create our results + results := make(map[flows.ContactID]modifyResult, len(contacts)) + for flowContact := range modifiersByContact { + results[flowContact.ID()] = modifyResult{ + Contact: flowContact, + Events: eventsByContact[flowContact], + } } return results, http.StatusOK, nil diff --git a/web/contact/contact_test.go b/web/contact/contact_test.go index 5665a1027..97f3361a8 100644 --- a/web/contact/contact_test.go +++ b/web/contact/contact_test.go @@ -1,175 +1,26 @@ package contact import ( - "bytes" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "net/http" - "sync" "testing" "time" - "github.com/nyaruka/goflow/utils/uuids" - "github.com/nyaruka/mailroom/config" + "github.com/nyaruka/gocommon/uuids" _ "github.com/nyaruka/mailroom/hooks" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/testsuite" "github.com/nyaruka/mailroom/web" - - "github.com/olivere/elastic" - "github.com/stretchr/testify/assert" ) -func TestSearch(t *testing.T) { +func TestCreateContacts(t *testing.T) { testsuite.Reset() - ctx := testsuite.CTX() db := testsuite.DB() - rp := testsuite.RP() - wg := &sync.WaitGroup{} - - es := testsuite.NewMockElasticServer() - defer es.Close() - - client, err := elastic.NewClient( - elastic.SetURL(es.URL()), - elastic.SetHealthcheck(false), - elastic.SetSniff(false), - ) - assert.NoError(t, err) - - server := web.NewServer(ctx, config.Mailroom, db, rp, nil, client, wg) - server.Start() - - // give our server time to start - time.Sleep(time.Second) - - defer server.Stop() - - singleESResponse := fmt.Sprintf(`{ - "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", - "took": 2, - "timed_out": false, - "_shards": { - "total": 1, - "successful": 1, - "skipped": 0, - "failed": 0 - }, - "hits": { - "total": 1, - "max_score": null, - "hits": [ - { - "_index": "contacts", - "_type": "_doc", - "_id": "%d", - "_score": null, - "_routing": "1", - "sort": [ - 15124352 - ] - } - ] - } - }`, models.CathyID) - - tcs := []struct { - URL string - Method string - Body string - Status int - Error string - Hits []models.ContactID - Query string - Fields []string - ESResponse string - }{ - {"/mr/contact/search", "GET", "", 405, "illegal method: GET", nil, "", nil, ""}, - { - "/mr/contact/search", "POST", - fmt.Sprintf(`{"org_id": 1, "query": "birthday = tomorrow", "group_uuid": "%s"}`, models.AllContactsGroupUUID), - 400, "can't resolve 'birthday' to attribute, scheme or field", - nil, "", nil, "", - }, - { - "/mr/contact/search", "POST", - fmt.Sprintf(`{"org_id": 1, "query": "age > tomorrow", "group_uuid": "%s"}`, models.AllContactsGroupUUID), - 400, "can't convert 'tomorrow' to a number", - nil, "", nil, "", - }, - { - "/mr/contact/search", "POST", - fmt.Sprintf(`{"org_id": 1, "query": "Cathy", "group_uuid": "%s"}`, models.AllContactsGroupUUID), - 200, - "", - []models.ContactID{models.CathyID}, - `name ~ "Cathy"`, - []string{"name"}, - singleESResponse, - }, - { - "/mr/contact/search", "POST", - fmt.Sprintf(`{"org_id": 1, "query": "AGE = 10 and gender = M", "group_uuid": "%s"}`, models.AllContactsGroupUUID), - 200, - "", - []models.ContactID{models.CathyID}, - `age = 10 AND gender = "M"`, - []string{"age", "gender"}, - singleESResponse, - }, - { - "/mr/contact/search", "POST", - fmt.Sprintf(`{"org_id": 1, "query": "", "group_uuid": "%s"}`, models.AllContactsGroupUUID), - 200, - "", - []models.ContactID{models.CathyID}, - ``, - []string{}, - singleESResponse, - }, - } - for i, tc := range tcs { - var body io.Reader - es.NextResponse = tc.ESResponse - - if tc.Body != "" { - body = bytes.NewReader([]byte(tc.Body)) - } - - req, err := http.NewRequest(tc.Method, "http://localhost:8090"+tc.URL, body) - assert.NoError(t, err, "%d: error creating request", i) - - resp, err := http.DefaultClient.Do(req) - assert.NoError(t, err, "%d: error making request", i) - - assert.Equal(t, tc.Status, resp.StatusCode, "%d: unexpected status", i) - - content, err := ioutil.ReadAll(resp.Body) - assert.NoError(t, err, "%d: error reading body", i) + // detach Cathy's tel URN + db.MustExec(`UPDATE contacts_contacturn SET contact_id = NULL WHERE contact_id = $1`, models.CathyID) - // on 200 responses parse them - if resp.StatusCode == 200 { - r := &searchResponse{} - err = json.Unmarshal(content, r) - assert.NoError(t, err) - assert.Equal(t, tc.Hits, r.ContactIDs) - assert.Equal(t, tc.Query, r.Query) - assert.Equal(t, tc.Fields, r.Fields) - } else { - r := &web.ErrorResponse{} - err = json.Unmarshal(content, r) - assert.NoError(t, err) - assert.Equal(t, tc.Error, r.Error) - } - } -} + db.MustExec(`ALTER SEQUENCE contacts_contact_id_seq RESTART WITH 30000`) -func TestParse(t *testing.T) { - testsuite.Reset() - web.RunWebTests(t, "testdata/parse_query.json") + web.RunWebTests(t, "testdata/create.json") } func TestModifyContacts(t *testing.T) { diff --git a/web/contact/search.go b/web/contact/search.go new file mode 100644 index 000000000..50289dfef --- /dev/null +++ b/web/contact/search.go @@ -0,0 +1,219 @@ +package contact + +import ( + "context" + "net/http" + + "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/goflow/contactql" + "github.com/nyaruka/goflow/utils" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/web" + + "github.com/pkg/errors" +) + +func init() { + web.RegisterJSONRoute(http.MethodPost, "/mr/contact/search", web.RequireAuthToken(handleSearch)) + web.RegisterJSONRoute(http.MethodPost, "/mr/contact/parse_query", web.RequireAuthToken(handleParseQuery)) +} + +// Searches the contacts for an org +// +// { +// "org_id": 1, +// "group_uuid": "985a83fe-2e9f-478d-a3ec-fa602d5e7ddd", +// "query": "age > 10", +// "sort": "-age" +// } +// +type searchRequest struct { + OrgID models.OrgID `json:"org_id" validate:"required"` + GroupUUID assets.GroupUUID `json:"group_uuid" validate:"required"` + ExcludeIDs []models.ContactID `json:"exclude_ids"` + Query string `json:"query"` + PageSize int `json:"page_size"` + Offset int `json:"offset"` + Sort string `json:"sort"` +} + +// Response for a contact search +// +// { +// "query": "age > 10", +// "contact_ids": [5,10,15], +// "total": 3, +// "offset": 0, +// "metadata": { +// "fields": [ +// {"key": "age", "name": "Age"} +// ], +// "allow_as_group": true +// } +// } +type searchResponse struct { + Query string `json:"query"` + ContactIDs []models.ContactID `json:"contact_ids"` + Total int64 `json:"total"` + Offset int `json:"offset"` + Sort string `json:"sort"` + Metadata *contactql.Inspection `json:"metadata,omitempty"` + + // deprecated + Fields []string `json:"fields"` + AllowAsGroup bool `json:"allow_as_group"` +} + +// handles a contact search request +func handleSearch(ctx context.Context, s *web.Server, r *http.Request) (interface{}, int, error) { + request := &searchRequest{ + Offset: 0, + PageSize: 50, + Sort: "-id", + } + if err := utils.UnmarshalAndValidateWithLimit(r.Body, request, web.MaxRequestBytes); err != nil { + return errors.Wrapf(err, "request failed validation"), http.StatusBadRequest, nil + } + + // grab our org assets + oa, err := models.GetOrgAssetsWithRefresh(s.CTX, s.DB, request.OrgID, models.RefreshFields|models.RefreshGroups) + if err != nil { + return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load org assets") + } + + // perform our search + parsed, hits, total, err := models.ContactIDsForQueryPage(ctx, s.ElasticClient, oa, + request.GroupUUID, request.ExcludeIDs, request.Query, request.Sort, request.Offset, request.PageSize) + + if err != nil { + isQueryError, qerr := contactql.IsQueryError(err) + if isQueryError { + return qerr, http.StatusBadRequest, nil + } + return nil, http.StatusInternalServerError, err + } + + // normalize and inspect the query + normalized := "" + var metadata *contactql.Inspection + allowAsGroup := false + fields := make([]string, 0) + + if parsed != nil { + normalized = parsed.String() + metadata = contactql.Inspect(parsed) + fields = append(fields, metadata.Attributes...) + for _, f := range metadata.Fields { + fields = append(fields, f.Key) + } + allowAsGroup = metadata.AllowAsGroup + } + + // build our response + response := &searchResponse{ + Query: normalized, + ContactIDs: hits, + Total: total, + Offset: request.Offset, + Sort: request.Sort, + Metadata: metadata, + Fields: fields, + AllowAsGroup: allowAsGroup, + } + + return response, http.StatusOK, nil +} + +// Request to parse the passed in query +// +// { +// "org_id": 1, +// "query": "age > 10", +// "group_uuid": "123123-123-123-" +// } +// +type parseRequest struct { + OrgID models.OrgID `json:"org_id" validate:"required"` + Query string `json:"query" validate:"required"` + GroupUUID assets.GroupUUID `json:"group_uuid"` +} + +// Response for a parse query request +// +// { +// "query": "age > 10", +// "elastic_query": { .. }, +// "metadata": { +// "fields": [ +// {"key": "age", "name": "Age"} +// ], +// "allow_as_group": true +// } +// } +type parseResponse struct { + Query string `json:"query"` + ElasticQuery interface{} `json:"elastic_query"` + Metadata *contactql.Inspection `json:"metadata,omitempty"` + + // deprecated + Fields []string `json:"fields"` + AllowAsGroup bool `json:"allow_as_group"` +} + +// handles a query parsing request +func handleParseQuery(ctx context.Context, s *web.Server, r *http.Request) (interface{}, int, error) { + request := &parseRequest{} + if err := utils.UnmarshalAndValidateWithLimit(r.Body, request, web.MaxRequestBytes); err != nil { + return errors.Wrapf(err, "request failed validation"), http.StatusBadRequest, nil + } + + // grab our org assets + oa, err := models.GetOrgAssetsWithRefresh(s.CTX, s.DB, request.OrgID, models.RefreshFields|models.RefreshGroups) + if err != nil { + return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to load org assets") + } + + env := oa.Env() + parsed, err := contactql.ParseQuery(env, request.Query, oa.SessionAssets()) + + if err != nil { + isQueryError, qerr := contactql.IsQueryError(err) + if isQueryError { + return qerr, http.StatusBadRequest, nil + } + return nil, http.StatusInternalServerError, err + } + + // normalize and inspect the query + normalized := "" + var metadata *contactql.Inspection + allowAsGroup := false + fields := make([]string, 0) + + if parsed != nil { + normalized = parsed.String() + metadata = contactql.Inspect(parsed) + fields = append(fields, metadata.Attributes...) + for _, f := range metadata.Fields { + fields = append(fields, f.Key) + } + allowAsGroup = metadata.AllowAsGroup + } + + eq := models.BuildElasticQuery(oa, request.GroupUUID, models.NilContactStatus, nil, parsed) + eqj, err := eq.Source() + if err != nil { + return nil, http.StatusInternalServerError, err + } + + // build our response + response := &parseResponse{ + Query: normalized, + ElasticQuery: eqj, + Metadata: metadata, + Fields: fields, + AllowAsGroup: allowAsGroup, + } + + return response, http.StatusOK, nil +} diff --git a/web/contact/search_test.go b/web/contact/search_test.go new file mode 100644 index 000000000..2631c5aa1 --- /dev/null +++ b/web/contact/search_test.go @@ -0,0 +1,244 @@ +package contact + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "sync" + "testing" + "time" + + "github.com/nyaruka/goflow/test" + "github.com/nyaruka/mailroom/config" + _ "github.com/nyaruka/mailroom/hooks" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/web" + + "github.com/olivere/elastic" + "github.com/stretchr/testify/assert" +) + +func TestSearch(t *testing.T) { + testsuite.Reset() + ctx := testsuite.CTX() + db := testsuite.DB() + rp := testsuite.RP() + wg := &sync.WaitGroup{} + + es := testsuite.NewMockElasticServer() + defer es.Close() + + client, err := elastic.NewClient( + elastic.SetURL(es.URL()), + elastic.SetHealthcheck(false), + elastic.SetSniff(false), + ) + assert.NoError(t, err) + + server := web.NewServer(ctx, config.Mailroom, db, rp, nil, client, wg) + server.Start() + + // give our server time to start + time.Sleep(time.Second) + + defer server.Stop() + + singleESResponse := fmt.Sprintf(`{ + "_scroll_id": "DXF1ZXJ5QW5kRmV0Y2gBAAAAAAAbgc0WS1hqbHlfb01SM2lLTWJRMnVOSVZDdw==", + "took": 2, + "timed_out": false, + "_shards": { + "total": 1, + "successful": 1, + "skipped": 0, + "failed": 0 + }, + "hits": { + "total": 1, + "max_score": null, + "hits": [ + { + "_index": "contacts", + "_type": "_doc", + "_id": "%d", + "_score": null, + "_routing": "1", + "sort": [ + 15124352 + ] + } + ] + } + }`, models.CathyID) + + tcs := []struct { + URL string + Method string + Body string + ESResponse string + ExpectedStatus int + ExpectedError string + ExpectedHits []models.ContactID + ExpectedQuery string + ExpectedFields []string + ExpectedESRequest string + }{ + { + Method: "GET", + URL: "/mr/contact/search", + ExpectedStatus: 405, + ExpectedError: "illegal method: GET", + }, + { + Method: "POST", + URL: "/mr/contact/search", + Body: fmt.Sprintf(`{"org_id": 1, "query": "birthday = tomorrow", "group_uuid": "%s"}`, models.AllContactsGroupUUID), + ExpectedStatus: 400, + ExpectedError: "can't resolve 'birthday' to attribute, scheme or field", + }, + { + Method: "POST", + URL: "/mr/contact/search", + Body: fmt.Sprintf(`{"org_id": 1, "query": "age > tomorrow", "group_uuid": "%s"}`, models.AllContactsGroupUUID), + ExpectedStatus: 400, + ExpectedError: "can't convert 'tomorrow' to a number", + }, + { + Method: "POST", + URL: "/mr/contact/search", + Body: fmt.Sprintf(`{"org_id": 1, "query": "Cathy", "group_uuid": "%s"}`, models.AllContactsGroupUUID), + ESResponse: singleESResponse, + ExpectedStatus: 200, + ExpectedHits: []models.ContactID{models.CathyID}, + ExpectedQuery: `name ~ "Cathy"`, + ExpectedFields: []string{"name"}, + }, + { + Method: "POST", + URL: "/mr/contact/search", + Body: fmt.Sprintf(`{"org_id": 1, "query": "Cathy", "group_uuid": "%s", "exclude_ids": [%d, %d]}`, models.AllContactsGroupUUID, models.BobID, models.GeorgeID), + ESResponse: singleESResponse, + ExpectedStatus: 200, + ExpectedHits: []models.ContactID{models.CathyID}, + ExpectedQuery: `name ~ "Cathy"`, + ExpectedFields: []string{"name"}, + ExpectedESRequest: `{ + "_source": false, + "from": 0, + "query": { + "bool": { + "must": [ + { + "term": { + "org_id": 1 + } + }, + { + "term": { + "is_active": true + } + }, + { + "term": { + "groups": "d1ee73f0-bdb5-47ce-99dd-0c95d4ebf008" + } + }, + { + "match": { + "name": { + "query": "cathy" + } + } + } + ], + "must_not": { + "ids": { + "type": "_doc", + "values": [ + "10001", "10002" + ] + } + } + } + }, + "size": 50, + "sort": [ + { + "id": { + "order": "desc" + } + } + ] + }`, + }, + { + Method: "POST", + URL: "/mr/contact/search", + Body: fmt.Sprintf(`{"org_id": 1, "query": "AGE = 10 and gender = M", "group_uuid": "%s"}`, models.AllContactsGroupUUID), + ESResponse: singleESResponse, + ExpectedStatus: 200, + ExpectedHits: []models.ContactID{models.CathyID}, + ExpectedQuery: `age = 10 AND gender = "M"`, + ExpectedFields: []string{"age", "gender"}, + }, + { + Method: "POST", + URL: "/mr/contact/search", + Body: fmt.Sprintf(`{"org_id": 1, "query": "", "group_uuid": "%s"}`, models.AllContactsGroupUUID), + ESResponse: singleESResponse, + ExpectedStatus: 200, + ExpectedHits: []models.ContactID{models.CathyID}, + ExpectedQuery: ``, + ExpectedFields: []string{}, + }, + } + + for i, tc := range tcs { + var body io.Reader + es.NextResponse = tc.ESResponse + + if tc.Body != "" { + body = bytes.NewReader([]byte(tc.Body)) + } + + req, err := http.NewRequest(tc.Method, "http://localhost:8090"+tc.URL, body) + assert.NoError(t, err, "%d: error creating request", i) + + resp, err := http.DefaultClient.Do(req) + assert.NoError(t, err, "%d: error making request", i) + + assert.Equal(t, tc.ExpectedStatus, resp.StatusCode, "%d: unexpected status", i) + + content, err := ioutil.ReadAll(resp.Body) + assert.NoError(t, err, "%d: error reading body", i) + + // on 200 responses parse them + if resp.StatusCode == 200 { + r := &searchResponse{} + err = json.Unmarshal(content, r) + assert.NoError(t, err) + assert.Equal(t, tc.ExpectedHits, r.ContactIDs) + assert.Equal(t, tc.ExpectedQuery, r.Query) + assert.Equal(t, tc.ExpectedFields, r.Fields) + + if tc.ExpectedESRequest != "" { + test.AssertEqualJSON(t, []byte(tc.ExpectedESRequest), []byte(es.LastBody), "elastic request mismatch") + } + } else { + r := &web.ErrorResponse{} + err = json.Unmarshal(content, r) + assert.NoError(t, err) + assert.Equal(t, tc.ExpectedError, r.Error) + } + } +} + +func TestParse(t *testing.T) { + testsuite.Reset() + + web.RunWebTests(t, "testdata/parse_query.json") +} diff --git a/web/contact/testdata/create.json b/web/contact/testdata/create.json new file mode 100644 index 000000000..3ba819678 --- /dev/null +++ b/web/contact/testdata/create.json @@ -0,0 +1,165 @@ +[ + { + "label": "error if contact not provided", + "method": "POST", + "path": "/mr/contact/create", + "body": { + "org_id": 1, + "user_id": 1 + }, + "status": 400, + "response": { + "error": "request failed validation: field 'contact' is required" + }, + "db_assertions": [ + { + "query": "SELECT count(*) FROM contacts_contact WHERE created_by_id != 2", + "count": 0 + } + ] + }, + { + "label": "create empty contact", + "method": "POST", + "path": "/mr/contact/create", + "body": { + "org_id": 1, + "user_id": 1, + "contact": {} + }, + "status": 200, + "response": { + "contact": { + "uuid": "d2f852ec-7b4e-457f-ae7f-f8b243c49ff5", + "id": 30000, + "status": "active", + "timezone": "America/Los_Angeles", + "created_on": "2018-07-06T12:30:00.123457Z" + } + }, + "db_assertions": [ + { + "query": "SELECT count(*) FROM contacts_contact WHERE name IS NULL AND language IS NULL AND created_by_id != 2", + "count": 1 + } + ] + }, + { + "label": "create contact with all properties", + "method": "POST", + "path": "/mr/contact/create", + "body": { + "org_id": 1, + "user_id": 1, + "contact": { + "name": "José", + "language": "spa", + "urns": [ + "tel:+16055700001" + ], + "fields": { + "gender": "M", + "age": "39" + }, + "groups": [ + "c153e265-f7c9-4539-9dbc-9b358714b638" + ] + } + }, + "status": 200, + "response": { + "contact": { + "uuid": "692926ea-09d6-4942-bd38-d266ec8d3716", + "id": 30001, + "name": "José", + "language": "spa", + "status": "active", + "timezone": "America/Los_Angeles", + "created_on": "2018-07-06T12:30:00.123457Z", + "urns": [ + "tel:+16055700001?id=20121&priority=1000" + ], + "groups": [ + { + "uuid": "c153e265-f7c9-4539-9dbc-9b358714b638", + "name": "Doctors" + } + ], + "fields": { + "age": { + "text": "39", + "number": 39 + }, + "gender": { + "text": "M" + } + } + } + } + }, + { + "label": "error if try to create contact with invalid language", + "method": "POST", + "path": "/mr/contact/create", + "body": { + "org_id": 1, + "user_id": 1, + "contact": { + "name": "María", + "language": "xyz" + } + }, + "status": 400, + "response": { + "error": "invalid language: unrecognized language code: xyz" + } + }, + { + "label": "error if try to create contact with taken URN", + "method": "POST", + "path": "/mr/contact/create", + "body": { + "org_id": 1, + "user_id": 1, + "contact": { + "name": "María", + "urns": [ + "tel:+16055700001" + ] + } + }, + "status": 400, + "response": { + "error": "URNs in use by other contacts" + } + }, + { + "label": "though ok to take an orphaned URN", + "method": "POST", + "path": "/mr/contact/create", + "body": { + "org_id": 1, + "user_id": 1, + "contact": { + "name": "María", + "urns": [ + "tel:+16055741111" + ] + } + }, + "status": 200, + "response": { + "contact": { + "uuid": "c34b6c7d-fa06-4563-92a3-d648ab64bccb", + "id": 30003, + "name": "María", + "status": "active", + "timezone": "America/Los_Angeles", + "created_on": "2018-07-06T12:30:00.123457Z", + "urns": [ + "tel:+16055741111?id=10000&priority=1000" + ] + } + } + } +] \ No newline at end of file diff --git a/web/contact/testdata/modify.json b/web/contact/testdata/modify.json index 92aaa552f..e79db1e7e 100644 --- a/web/contact/testdata/modify.json +++ b/web/contact/testdata/modify.json @@ -155,7 +155,7 @@ }, "db_assertions": [ { - "query": "SELECT count(*) FROM contacts_contact WHERE id = 10000 AND name = ''", + "query": "SELECT count(*) FROM contacts_contact WHERE id = 10000 AND name IS NULL", "count": 1 } ] diff --git a/web/contact/utils.go b/web/contact/utils.go new file mode 100644 index 000000000..14a76f70d --- /dev/null +++ b/web/contact/utils.go @@ -0,0 +1,106 @@ +package contact + +import ( + "context" + + "github.com/nyaruka/gocommon/urns" + "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/goflow/envs" + "github.com/nyaruka/goflow/flows" + "github.com/nyaruka/goflow/flows/modifiers" + "github.com/nyaruka/mailroom/models" + + "github.com/gomodule/redigo/redis" + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" +) + +// Creation a validated contact creation task +type Creation struct { + Name string + Language envs.Language + URNs []urns.URN + Mods []flows.Modifier +} + +// Spec describes a contact to be created +type Spec struct { + Name string `json:"name"` + Language string `json:"language"` + URNs []urns.URN `json:"urns"` + Fields map[string]string `json:"fields"` + Groups []assets.GroupUUID `json:"groups"` +} + +// Validate validates that the spec is valid for the given assets +func (s *Spec) Validate(env envs.Environment, sa flows.SessionAssets) (*Creation, error) { + country := string(env.DefaultCountry()) + var err error + validated := &Creation{Name: s.Name} + + if s.Language != "" { + validated.Language, err = envs.ParseLanguage(s.Language) + if err != nil { + return nil, errors.Wrap(err, "invalid language") + } + } + + validated.URNs = make([]urns.URN, len(s.URNs)) + for i, urn := range s.URNs { + validated.URNs[i] = urn.Normalize(country) + } + + validated.Mods = make([]flows.Modifier, 0, len(s.Fields)) + + for key, value := range s.Fields { + field := sa.Fields().Get(key) + if field == nil { + return nil, errors.Errorf("unknown contact field '%s'", key) + } + if value != "" { + validated.Mods = append(validated.Mods, modifiers.NewField(field, value)) + } + } + + if len(s.Groups) > 0 { + groups := make([]*flows.Group, len(s.Groups)) + for i, uuid := range s.Groups { + group := sa.Groups().Get(uuid) + if group == nil { + return nil, errors.Errorf("unknown contact group '%s'", uuid) + } + if group.UsesQuery() { + return nil, errors.Errorf("can't add contact to query based group '%s'", uuid) + } + groups[i] = group + } + + validated.Mods = append(validated.Mods, modifiers.NewGroups(groups, modifiers.GroupsAdd)) + } + + return validated, nil +} + +// ModifyContacts modifies contacts by applying modifiers and handling the resultant events +func ModifyContacts(ctx context.Context, db *sqlx.DB, rp *redis.Pool, oa *models.OrgAssets, modifiersByContact map[*flows.Contact][]flows.Modifier) (map[*flows.Contact][]flows.Event, error) { + // create an environment instance with location support + env := flows.NewEnvironment(oa.Env(), oa.SessionAssets().Locations()) + + eventsByContact := make(map[*flows.Contact][]flows.Event, len(modifiersByContact)) + + // apply the modifiers to get the events for each contact + for contact, mods := range modifiersByContact { + events := make([]flows.Event, 0) + for _, mod := range mods { + mod.Apply(env, oa.SessionAssets(), contact, func(e flows.Event) { events = append(events, e) }) + } + eventsByContact[contact] = events + } + + err := models.HandleAndCommitEvents(ctx, db, rp, oa, eventsByContact) + if err != nil { + return nil, errors.Wrap(err, "error commiting events") + } + + return eventsByContact, nil +} diff --git a/web/contact/utils_test.go b/web/contact/utils_test.go new file mode 100644 index 000000000..3cb478d70 --- /dev/null +++ b/web/contact/utils_test.go @@ -0,0 +1,55 @@ +package contact_test + +import ( + "testing" + + "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/goflow/envs" + "github.com/nyaruka/mailroom/models" + "github.com/nyaruka/mailroom/testsuite" + "github.com/nyaruka/mailroom/web/contact" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateSpec(t *testing.T) { + testsuite.Reset() + db := testsuite.DB() + ctx := testsuite.CTX() + + oa, err := models.GetOrgAssets(ctx, db, models.Org1) + require.NoError(t, err) + + sa := oa.SessionAssets() + env := envs.NewBuilder().Build() + + // empty spec is valid + s := &contact.Spec{} + c, err := s.Validate(env, sa) + assert.NoError(t, err) + assert.Equal(t, "", c.Name) + assert.Equal(t, envs.NilLanguage, c.Language) + assert.Equal(t, 0, len(c.URNs)) + assert.Equal(t, 0, len(c.Mods)) + + // try to set invalid language + s = &contact.Spec{Language: "xyzd"} + _, err = s.Validate(env, sa) + assert.EqualError(t, err, "invalid language: iso-639-3 codes must be 3 characters, got: xyzd") + + // try to set non-existent contact field + s = &contact.Spec{Fields: map[string]string{"goats": "7"}} + _, err = s.Validate(env, sa) + assert.EqualError(t, err, "unknown contact field 'goats'") + + // try to add to non-existent group + s = &contact.Spec{Groups: []assets.GroupUUID{"52f6c50e-f9a8-4f24-bb80-5c9f144ed27f"}} + _, err = s.Validate(env, sa) + assert.EqualError(t, err, "unknown contact group '52f6c50e-f9a8-4f24-bb80-5c9f144ed27f'") + + // try to add to dynamic group + s = &contact.Spec{Groups: []assets.GroupUUID{"52f6c50e-f9a8-4f24-bb80-5c9f144ed27f"}} + _, err = s.Validate(env, sa) + assert.EqualError(t, err, "unknown contact group '52f6c50e-f9a8-4f24-bb80-5c9f144ed27f'") +} diff --git a/web/flow/flow.go b/web/flow/flow.go index 0693cf0b6..89bc74631 100644 --- a/web/flow/flow.go +++ b/web/flow/flow.go @@ -5,10 +5,10 @@ import ( "encoding/json" "net/http" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/utils" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/goflow" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/web" diff --git a/web/ivr/ivr.go b/web/ivr/ivr.go index fa0fe0612..900a248c5 100644 --- a/web/ivr/ivr.go +++ b/web/ivr/ivr.go @@ -1,18 +1,15 @@ package ivr import ( - "bytes" "context" "database/sql" "encoding/json" "fmt" "net/http" - "net/http/httputil" "net/url" "time" - "github.com/go-chi/chi" - "github.com/go-chi/chi/middleware" + "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/assets" "github.com/nyaruka/mailroom/config" @@ -20,142 +17,130 @@ import ( "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/tasks/handler" "github.com/nyaruka/mailroom/web" + + "github.com/go-chi/chi" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) func init() { - web.RegisterRoute(http.MethodPost, "/mr/ivr/c/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}/handle", handleFlow) - web.RegisterRoute(http.MethodPost, "/mr/ivr/c/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}/status", handleStatus) - web.RegisterRoute(http.MethodPost, "/mr/ivr/c/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}/incoming", handleIncomingCall) + web.RegisterRoute(http.MethodPost, "/mr/ivr/c/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}/handle", newIVRHandler(handleFlow)) + web.RegisterRoute(http.MethodPost, "/mr/ivr/c/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}/status", newIVRHandler(handleStatus)) + web.RegisterRoute(http.MethodPost, "/mr/ivr/c/{uuid:[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}}/incoming", newIVRHandler(handleIncomingCall)) } -// TODO: creation of requests is awkward, would be nice to figure out how to unify how all that works - -func handleIncomingCall(ctx context.Context, s *web.Server, r *http.Request, rawW http.ResponseWriter) error { - start := time.Now() +type ivrHandlerFn func(ctx context.Context, s *web.Server, r *http.Request, w http.ResponseWriter) (*models.Channel, *models.ChannelConnection, error) + +func newIVRHandler(handler ivrHandlerFn) web.Handler { + return func(ctx context.Context, s *web.Server, r *http.Request, w http.ResponseWriter) error { + recorder := httpx.NewRecorder(r, w) + ww := recorder.ResponseWriter + + channel, connection, rerr := handler(ctx, s, r, ww) + + if channel != nil { + trace, err := recorder.End() + if err != nil { + logrus.WithError(err).WithField("http_request", r).Error("error recording IVR request") + } + + desc := "IVR event handled" + isError := false + if trace.Response == nil || trace.Response.StatusCode != http.StatusOK { + desc = "IVR Error" + isError = true + } + + log := models.NewChannelLog(trace, isError, desc, channel, connection) + err = models.InsertChannelLogs(ctx, s.DB, []*models.ChannelLog{log}) + if err != nil { + logrus.WithError(err).WithField("http_request", r).Error("error writing ivr channel log") + } + } - // dump our request - requestTrace, err := httputil.DumpRequest(r, true) - if err != nil { - return errors.Wrapf(err, "error creating request trace") + return rerr } +} - // wrap our writer - responseTrace := &bytes.Buffer{} - w := middleware.NewWrapResponseWriter(rawW, r.ProtoMajor) - w.Tee(responseTrace) - +func handleIncomingCall(ctx context.Context, s *web.Server, r *http.Request, w http.ResponseWriter) (*models.Channel, *models.ChannelConnection, error) { channelUUID := assets.ChannelUUID(chi.URLParam(r, "uuid")) // load the org id for this UUID (we could load the entire channel here but we want to take the same paths through everything else) orgID, err := models.OrgIDForChannelUUID(ctx, s.DB, channelUUID) if err != nil { - return writeClientError(w, err) + return nil, nil, writeClientError(w, err) } // load our org assets oa, err := models.GetOrgAssets(ctx, s.DB, orgID) if err != nil { - return writeClientError(w, errors.Wrapf(err, "error loading org assets")) + return nil, nil, writeClientError(w, errors.Wrapf(err, "error loading org assets")) } // and our channel channel := oa.ChannelByUUID(channelUUID) if channel == nil { - return writeClientError(w, errors.Wrapf(err, "no active channel with uuid: %s", channelUUID)) + return nil, nil, writeClientError(w, errors.Wrapf(err, "no active channel with uuid: %s", channelUUID)) } - var conn *models.ChannelConnection - - // create a channel log for this request and connection - defer func() { - desc := "IVR event handled" - isError := false - if w.Status() != http.StatusOK { - desc = "IVR Error" - isError = true - } - - path := r.URL.RequestURI() - proxyPath := r.Header.Get("X-Forwarded-Path") - if proxyPath != "" { - path = proxyPath - } - - url := fmt.Sprintf("https://%s%s", r.Host, path) - _, err := models.InsertChannelLog( - ctx, s.DB, desc, isError, - r.Method, url, requestTrace, w.Status(), responseTrace.Bytes(), - start, time.Since(start), - channel, conn, - ) - if err != nil { - logrus.WithError(err).WithField("http_request", r).Error("error writing ivr channel log") - } - }() - // get the right kind of client client, err := ivr.GetClient(channel) if client == nil { - return writeClientError(w, errors.Wrapf(err, "unable to load client for channel: %s", channelUUID)) + return channel, nil, writeClientError(w, errors.Wrapf(err, "unable to load client for channel: %s", channelUUID)) } // validate this request's signature err = client.ValidateRequestSignature(r) if err != nil { - return client.WriteErrorResponse(w, errors.Wrapf(err, "request failed signature validation")) + return channel, nil, client.WriteErrorResponse(w, errors.Wrapf(err, "request failed signature validation")) } // lookup the URN of the caller urn, err := client.URNForRequest(r) if err != nil { - return client.WriteErrorResponse(w, errors.Wrapf(err, "unable to find URN in request")) + return channel, nil, client.WriteErrorResponse(w, errors.Wrapf(err, "unable to find URN in request")) } - // get the contact id for this URN - ids, err := models.ContactIDsFromURNs(ctx, s.DB, oa, []urns.URN{urn}) + // get the contact for this URN + contact, _, err := models.GetOrCreateContact(ctx, s.DB, oa, urn) if err != nil { - return client.WriteErrorResponse(w, errors.Wrapf(err, "unable to load contact by urn")) - } - contactID, found := ids[urn] - if !found { - return client.WriteErrorResponse(w, errors.Errorf("no contact for urn: %s", urn)) + return channel, nil, client.WriteErrorResponse(w, errors.Wrapf(err, "unable to get contact by urn")) } urn, err = models.URNForURN(ctx, s.DB, oa, urn) if err != nil { - return client.WriteErrorResponse(w, errors.Wrapf(err, "unable to load urn")) + return channel, nil, client.WriteErrorResponse(w, errors.Wrapf(err, "unable to load urn")) } // urn ID urnID := models.GetURNID(urn) if urnID == models.NilURNID { - return client.WriteErrorResponse(w, errors.Wrapf(err, "unable to get id for URN")) + return channel, nil, client.WriteErrorResponse(w, errors.Wrapf(err, "unable to get id for URN")) } // we first create an incoming call channel event and see if that matches - event := models.NewChannelEvent(models.MOCallEventType, oa.OrgID(), channel.ID(), contactID, urnID, nil, false) + event := models.NewChannelEvent(models.MOCallEventType, oa.OrgID(), channel.ID(), contact.ID(), urnID, nil, false) externalID, err := client.CallIDForRequest(r) if err != nil { - return client.WriteErrorResponse(w, errors.Wrapf(err, "unable to get external id from request")) + return channel, nil, client.WriteErrorResponse(w, errors.Wrapf(err, "unable to get external id from request")) } // create our connection - conn, err = models.InsertIVRConnection( - ctx, s.DB, oa.OrgID(), channel.ID(), models.NilStartID, contactID, urnID, + conn, err := models.InsertIVRConnection( + ctx, s.DB, oa.OrgID(), channel.ID(), models.NilStartID, contact.ID(), urnID, models.ConnectionDirectionIn, models.ConnectionStatusInProgress, externalID, ) if err != nil { - return client.WriteErrorResponse(w, errors.Wrapf(err, "error creating ivr connection")) + return channel, nil, client.WriteErrorResponse(w, errors.Wrapf(err, "error creating ivr connection")) } // try to handle this event session, err := handler.HandleChannelEvent(ctx, s.DB, s.RP, models.MOCallEventType, event, conn) if err != nil { logrus.WithError(err).WithField("http_request", r).Error("error handling incoming call") - return client.WriteErrorResponse(w, errors.Wrapf(err, "error handling incoming call")) + + return channel, conn, client.WriteErrorResponse(w, errors.Wrapf(err, "error handling incoming call")) } // we got a session back so we have an active call trigger @@ -166,29 +151,29 @@ func handleIncomingCall(ctx context.Context, s *web.Server, r *http.Request, raw // have our client output our session status err = client.WriteSessionResponse(session, urn, resumeURL, r, w) if err != nil { - return errors.Wrapf(err, "error writing ivr response for start") + return channel, conn, errors.Wrapf(err, "error writing ivr response for start") } - return nil + return channel, conn, nil } // no session means no trigger, create a missed call event instead // we first create an incoming call channel event and see if that matches - event = models.NewChannelEvent(models.MOMissEventType, oa.OrgID(), channel.ID(), contactID, urnID, nil, false) + event = models.NewChannelEvent(models.MOMissEventType, oa.OrgID(), channel.ID(), contact.ID(), urnID, nil, false) err = event.Insert(ctx, s.DB) if err != nil { - return client.WriteErrorResponse(w, errors.Wrapf(err, "error inserting channel event")) + return channel, conn, client.WriteErrorResponse(w, errors.Wrapf(err, "error inserting channel event")) } // try to handle it, this time looking for a missed call event session, err = handler.HandleChannelEvent(ctx, s.DB, s.RP, models.MOMissEventType, event, nil) if err != nil { logrus.WithError(err).WithField("http_request", r).Error("error handling missed call") - return client.WriteErrorResponse(w, errors.Wrapf(err, "error handling missed call")) + return channel, conn, client.WriteErrorResponse(w, errors.Wrapf(err, "error handling missed call")) } // write our empty response - return client.WriteEmptyResponse(w, "missed call handled") + return channel, conn, client.WriteEmptyResponse(w, "missed call handled") } const ( @@ -231,101 +216,61 @@ func buildResumeURL(channel *models.Channel, conn *models.ChannelConnection, urn } // handleFlow handles all incoming IVR requests related to a flow (status is handled elsewhere) -func handleFlow(ctx context.Context, s *web.Server, r *http.Request, rawW http.ResponseWriter) error { +func handleFlow(ctx context.Context, s *web.Server, r *http.Request, w http.ResponseWriter) (*models.Channel, *models.ChannelConnection, error) { ctx, cancel := context.WithTimeout(ctx, time.Second*55) defer cancel() - // dump our request - requestTrace, err := httputil.DumpRequest(r, true) - if err != nil { - return errors.Wrapf(err, "error creating request trace") - } - - // wrap our writer - responseTrace := &bytes.Buffer{} - w := middleware.NewWrapResponseWriter(rawW, r.ProtoMajor) - w.Tee(responseTrace) - - start := time.Now() - request := &IVRRequest{} if err := web.DecodeAndValidateForm(request, r); err != nil { - return errors.Wrapf(err, "request failed validation") + return nil, nil, errors.Wrapf(err, "request failed validation") } // load our connection conn, err := models.SelectChannelConnection(ctx, s.DB, request.ConnectionID) if err != nil { - return errors.Wrapf(err, "unable to load channel connection with id: %d", request.ConnectionID) + return nil, nil, errors.Wrapf(err, "unable to load channel connection with id: %d", request.ConnectionID) } // load our org assets oa, err := models.GetOrgAssets(ctx, s.DB, conn.OrgID()) if err != nil { - return writeClientError(w, errors.Wrapf(err, "error loading org assets")) + return nil, nil, writeClientError(w, errors.Wrapf(err, "error loading org assets")) } // and our channel channel := oa.ChannelByID(conn.ChannelID()) if channel == nil { - return writeClientError(w, errors.Errorf("no active channel with id: %d", conn.ChannelID())) + return nil, nil, writeClientError(w, errors.Errorf("no active channel with id: %d", conn.ChannelID())) } - // create a channel log for this request and connection - defer func() { - desc := "IVR event handled" - isError := false - if w.Status() != http.StatusOK { - desc = "IVR Error" - isError = true - } - - path := r.URL.RequestURI() - proxyPath := r.Header.Get("X-Forwarded-Path") - if proxyPath != "" { - path = proxyPath - } - - url := fmt.Sprintf("https://%s%s", r.Host, path) - _, err := models.InsertChannelLog( - ctx, s.DB, desc, isError, - r.Method, url, requestTrace, w.Status(), responseTrace.Bytes(), - start, time.Since(start), - channel, conn, - ) - if err != nil { - logrus.WithError(err).WithField("http_request", r).Error("error writing ivr channel log") - } - }() - // get the right kind of client client, err := ivr.GetClient(channel) if client == nil { - return writeClientError(w, errors.Wrapf(err, "unable to load client for channel: %d", conn.ChannelID())) + return channel, conn, writeClientError(w, errors.Wrapf(err, "unable to load client for channel: %d", conn.ChannelID())) } // validate this request's signature if relevant err = client.ValidateRequestSignature(r) if err != nil { - return writeClientError(w, errors.Wrapf(err, "request failed signature validation")) + return channel, conn, writeClientError(w, errors.Wrapf(err, "request failed signature validation")) } // load our contact contacts, err := models.LoadContacts(ctx, s.DB, oa, []models.ContactID{conn.ContactID()}) if err != nil { - return client.WriteErrorResponse(w, errors.Wrapf(err, "no such contact")) + return channel, conn, client.WriteErrorResponse(w, errors.Wrapf(err, "no such contact")) } if len(contacts) == 0 { - return client.WriteErrorResponse(w, errors.Errorf("no contact with id: %d", conn.ContactID())) + return channel, conn, client.WriteErrorResponse(w, errors.Errorf("no contact with id: %d", conn.ContactID())) } if contacts[0].Status() != models.ContactStatusActive { - return client.WriteErrorResponse(w, errors.Errorf("no contact with id: %d", conn.ContactID())) + return channel, conn, client.WriteErrorResponse(w, errors.Errorf("no contact with id: %d", conn.ContactID())) } // load the URN for this connection urn, err := models.URNForID(ctx, s.DB, oa, conn.ContactURNID()) if err != nil { - return client.WriteErrorResponse(w, errors.Errorf("unable to find connection urn: %d", conn.ContactURNID())) + return channel, conn, client.WriteErrorResponse(w, errors.Errorf("unable to find connection urn: %d", conn.ContactURNID())) } // make sure our URN is indeed present on our contact, no funny business @@ -336,7 +281,7 @@ func handleFlow(ctx context.Context, s *web.Server, r *http.Request, rawW http.R } } if !found { - return client.WriteErrorResponse(w, errors.Errorf("unable to find URN: %s on contact: %d", urn, conn.ContactID())) + return channel, conn, client.WriteErrorResponse(w, errors.Errorf("unable to find URN: %s on contact: %d", urn, conn.ContactID())) } resumeURL := buildResumeURL(channel, conn, urn) @@ -370,111 +315,71 @@ func handleFlow(ctx context.Context, s *web.Server, r *http.Request, rawW http.R // had an error? mark our connection as errored and log it if err != nil { logrus.WithError(err).WithField("http_request", r).Error("error while handling IVR") - return ivr.WriteErrorResponse(ctx, s.DB, client, conn, w, err) + return channel, conn, ivr.WriteErrorResponse(ctx, s.DB, client, conn, w, err) } - return nil + return channel, conn, nil } // handleStatus handles all incoming IVR events / status updates -func handleStatus(ctx context.Context, s *web.Server, r *http.Request, rawW http.ResponseWriter) error { +func handleStatus(ctx context.Context, s *web.Server, r *http.Request, w http.ResponseWriter) (*models.Channel, *models.ChannelConnection, error) { ctx, cancel := context.WithTimeout(ctx, time.Second*55) defer cancel() - // dump our request - requestTrace, err := httputil.DumpRequest(r, true) - if err != nil { - return errors.Wrapf(err, "error creating request trace") - } - - // wrap our writer - responseTrace := &bytes.Buffer{} - w := middleware.NewWrapResponseWriter(rawW, r.ProtoMajor) - w.Tee(responseTrace) - - start := time.Now() - channelUUID := assets.ChannelUUID(chi.URLParam(r, "uuid")) // load the org id for this UUID (we could load the entire channel here but we want to take the same paths through everything else) orgID, err := models.OrgIDForChannelUUID(ctx, s.DB, channelUUID) if err != nil { - return writeClientError(w, err) + return nil, nil, writeClientError(w, err) } // load our org assets oa, err := models.GetOrgAssets(ctx, s.DB, orgID) if err != nil { - return writeClientError(w, errors.Wrapf(err, "error loading org assets")) + return nil, nil, writeClientError(w, errors.Wrapf(err, "error loading org assets")) } // and our channel channel := oa.ChannelByUUID(channelUUID) if channel == nil { - return writeClientError(w, errors.Wrapf(err, "no active channel with uuid: %s", channelUUID)) + return nil, nil, writeClientError(w, errors.Wrapf(err, "no active channel with uuid: %s", channelUUID)) } // get the right kind of client client, err := ivr.GetClient(channel) if client == nil { - return writeClientError(w, errors.Wrapf(err, "unable to load client for channel: %s", channelUUID)) + return channel, nil, writeClientError(w, errors.Wrapf(err, "unable to load client for channel: %s", channelUUID)) } // validate this request's signature if relevant err = client.ValidateRequestSignature(r) if err != nil { - return writeClientError(w, errors.Wrapf(err, "request failed signature validation")) + return channel, nil, writeClientError(w, errors.Wrapf(err, "request failed signature validation")) } // get our external id externalID, err := client.CallIDForRequest(r) if err != nil { - return client.WriteErrorResponse(w, errors.Wrapf(err, "unable to get call id for request")) + return channel, nil, client.WriteErrorResponse(w, errors.Wrapf(err, "unable to get call id for request")) } // load our connection conn, err := models.SelectChannelConnectionByExternalID(ctx, s.DB, channel.ID(), models.ConnectionTypeIVR, externalID) if errors.Cause(err) == sql.ErrNoRows { - return client.WriteEmptyResponse(w, "unknown connection, ignoring") + return channel, nil, client.WriteEmptyResponse(w, "unknown connection, ignoring") } if err != nil { - return client.WriteErrorResponse(w, errors.Wrapf(err, "unable to load channel connection with id: %s", externalID)) + return channel, nil, client.WriteErrorResponse(w, errors.Wrapf(err, "unable to load channel connection with id: %s", externalID)) } - // create a channel log for this request and connection - defer func() { - desc := "IVR event handled" - isError := false - if w.Status() != http.StatusOK { - desc = "IVR Error" - isError = true - } - - path := r.URL.RequestURI() - proxyPath := r.Header.Get("X-Forwarded-Path") - if proxyPath != "" { - path = proxyPath - } - - url := fmt.Sprintf("https://%s%s", r.Host, path) - _, err := models.InsertChannelLog( - ctx, s.DB, desc, isError, - r.Method, url, requestTrace, w.Status(), responseTrace.Bytes(), - start, time.Since(start), - channel, conn, - ) - if err != nil { - logrus.WithError(err).WithField("http_request", r).Error("error writing ivr channel log") - } - }() - err = ivr.HandleIVRStatus(ctx, s.DB, s.RP, oa, client, conn, r, w) // had an error? mark our connection as errored and log it if err != nil { logrus.WithError(err).WithField("http_request", r).Error("error while handling status") - return ivr.WriteErrorResponse(ctx, s.DB, client, conn, w, err) + return channel, conn, ivr.WriteErrorResponse(ctx, s.DB, client, conn, w, err) } - return nil + return channel, conn, nil } diff --git a/web/org/metrics.go b/web/org/metrics.go index 73c4e89f9..aabde1766 100644 --- a/web/org/metrics.go +++ b/web/org/metrics.go @@ -6,8 +6,8 @@ import ( "github.com/go-chi/chi" "github.com/golang/protobuf/proto" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/assets" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/models" "github.com/nyaruka/mailroom/web" "github.com/pkg/errors" diff --git a/web/org/metrics_test.go b/web/org/metrics_test.go index c1141e32a..6f6868ec1 100644 --- a/web/org/metrics_test.go +++ b/web/org/metrics_test.go @@ -36,44 +36,45 @@ func TestMetrics(t *testing.T) { URL string Username string Password string + Response string Contains []string }{ { URL: fmt.Sprintf("http://localhost:8090/mr/org/%s/metrics", models.Org1UUID), Username: "", Password: "", - Contains: []string{`{"error": "invalid authentication"}`}, + Response: `{"error": "invalid authentication"}`, }, { URL: fmt.Sprintf("http://localhost:8090/mr/org/%s/metrics", models.Org1UUID), Username: "metrics", Password: "invalid", - Contains: []string{`{"error": "invalid authentication"}`}, + Response: `{"error": "invalid authentication"}`, }, { URL: fmt.Sprintf("http://localhost:8090/mr/org/%s/metrics", models.Org1UUID), Username: "invalid", Password: promToken, - Contains: []string{`{"error": "invalid authentication"}`}, + Response: `{"error": "invalid authentication"}`, }, { URL: fmt.Sprintf("http://localhost:8090/mr/org/%s/metrics", models.Org2UUID), Username: "metrics", Password: promToken, - Contains: []string{`{"error": "invalid authentication"}`}, + Response: `{"error": "invalid authentication"}`, }, { URL: fmt.Sprintf("http://localhost:8090/mr/org/%s/metrics", models.Org1UUID), Username: "metrics", Password: adminToken, - Contains: []string{`{"error": "invalid authentication"}`}, + Response: `{"error": "invalid authentication"}`, }, { URL: fmt.Sprintf("http://localhost:8090/mr/org/%s/metrics", models.Org1UUID), Username: "metrics", Password: promToken, Contains: []string{ - `rapidpro_group_contact_count{group_name="All Contacts",group_uuid="d1ee73f0-bdb5-47ce-99dd-0c95d4ebf008",group_type="system",org="UNICEF"} 124`, + `rapidpro_group_contact_count{group_name="Active",group_uuid="d1ee73f0-bdb5-47ce-99dd-0c95d4ebf008",group_type="system",org="UNICEF"} 124`, `rapidpro_group_contact_count{group_name="Doctors",group_uuid="c153e265-f7c9-4539-9dbc-9b358714b638",group_type="user",org="UNICEF"} 121`, `rapidpro_channel_msg_count{channel_name="Nexmo",channel_uuid="19012bfd-3ce3-4cae-9bb9-76cf92c73d49",channel_type="NX",msg_direction="out",msg_type="message",org="UNICEF"} 0`, }, @@ -88,6 +89,9 @@ func TestMetrics(t *testing.T) { body, _ := ioutil.ReadAll(resp.Body) + if tc.Response != "" { + assert.Equal(t, string(body), tc.Response, "%d: response mismatch", i) + } for _, contains := range tc.Contains { assert.Contains(t, string(body), contains, "%d does not contain: %s", i, contains) } diff --git a/web/po/testdata/favorites.po b/web/po/testdata/favorites.po index 40f1ba995..a9ffb4c2f 100644 --- a/web/po/testdata/favorites.po +++ b/web/po/testdata/favorites.po @@ -10,88 +10,88 @@ msgstr "" "Language-3: \n" "Source-Flows: 9de3663f-c5c5-4c92-9f45-ecbc09abcc85\n" -#: Favorites/e87aeeab-8ede-4173-bc76-8f5583ea7207/name:0 +#: Favorites/491f3ed1-9154-4acb-8fdd-0a37567e0574/name:0 msgid "All Responses" msgstr "" -#: Favorites/34a421ac-34cb-49d8-a2a5-534f52c60851/arguments:0 -#: Favorites/c102acfc-8cc5-41fa-89ed-41cbfa362ba6/name:0 +#: Favorites/34a421ac-34cb-49d8-a2a5-534f52c60851/name:0 +#: Favorites/baf07ebb-8a2a-4e63-aa08-d19aa408cd45/arguments:0 msgid "Blue" msgstr "" -#: Favorites/3b400f91-db69-42b9-9fe2-24ad556b067a/arguments:0 -#: Favorites/8d2e259c-bc3c-464f-8c15-985bc736e212/name:0 +#: Favorites/3b400f91-db69-42b9-9fe2-24ad556b067a/name:0 +#: Favorites/6e367c0c-65ab-479a-82e3-c597d8e35eef/arguments:0 msgid "Cyan" msgstr "" -#: Favorites/3e2dcf45-ffc0-4197-b5ab-25ed974ea612/text:0 +#: Favorites/7624633a-01a9-48f0-abca-957e7290df0a/text:0 msgid "Good choice, I like @results.color.category_localized too! What is your favorite beer?" msgstr "" -#: Favorites/58284598-805a-4740-8966-dcb09e3b670a/name:0 -#: Favorites/b0c29972-6fd4-485e-83c2-057a3f7a04da/arguments:0 +#: Favorites/b0c29972-6fd4-485e-83c2-057a3f7a04da/name:0 +#: Favorites/c102acfc-8cc5-41fa-89ed-41cbfa362ba6/arguments:0 msgid "Green" msgstr "" -#: Favorites/943f85bb-50bc-40c3-8d6f-57dbe34c87f7/text:0 +#: Favorites/9631dddf-0dd7-4310-b263-5f7cad4795e0/text:0 msgid "I don't know that color. Try again." msgstr "" -#: Favorites/4cadf512-1299-468f-85e4-26af9edec193/text:0 +#: Favorites/aac779a9-e2a6-4a11-9efa-9670e081a33a/text:0 msgid "I don't know that one, try again please." msgstr "" -#: Favorites/52d7a9ab-52b7-4e82-ba7f-672fb8d6ec91/text:0 +#: Favorites/ada3d96a-a1a2-41eb-aac7-febdb98a9b4c/text:0 msgid "Mmmmm... delicious @results.beer.category_localized. If only they made @(lower(results.color)) @results.beer.category_localized! Lastly, what is your name?" msgstr "" -#: Favorites/87b850ff-ddc5-4add-8a4f-c395c3a9ac38/name:0 -#: Favorites/a813de57-c92a-4128-804d-56e80b332142/arguments:0 +#: Favorites/a813de57-c92a-4128-804d-56e80b332142/name:0 +#: Favorites/b9d718d3-b5e0-4d26-998e-2da31b24f2f9/arguments:0 msgid "Mutzig" msgstr "" -#: Favorites/baf07ebb-8a2a-4e63-aa08-d19aa408cd45/arguments:0 +#: Favorites/8d2e259c-bc3c-464f-8c15-985bc736e212/arguments:0 msgid "Navy" msgstr "" -#: Favorites/6e367c0c-65ab-479a-82e3-c597d8e35eef/name:0 +#: Favorites/3e2dcf45-ffc0-4197-b5ab-25ed974ea612/name:0 msgid "No Response" msgstr "" -#: Favorites/c169352e-1944-4451-8d32-eb39c41cb3ae/name:0 -#: Favorites/e0ec2076-2746-43b4-a410-c3af47d6a121/name:0 +#: Favorites/5563a722-9680-419c-a792-b1fa9df92e06/name:0 +#: Favorites/87b850ff-ddc5-4add-8a4f-c395c3a9ac38/name:0 msgid "Other" msgstr "" -#: Favorites/a03dceb1-7ac1-491d-93ef-23d3e099633b/arguments:0 -#: Favorites/b9d718d3-b5e0-4d26-998e-2da31b24f2f9/name:0 +#: Favorites/a03dceb1-7ac1-491d-93ef-23d3e099633b/name:0 +#: Favorites/f1ca9ac8-d0aa-4758-a969-195be7330267/arguments:0 msgid "Primus" msgstr "" -#: Favorites/3ffb6f24-2ed8-4fd5-bcc0-b2e2668672a8/arguments:0 -#: Favorites/5563a722-9680-419c-a792-b1fa9df92e06/name:0 +#: Favorites/3ffb6f24-2ed8-4fd5-bcc0-b2e2668672a8/name:0 +#: Favorites/58284598-805a-4740-8966-dcb09e3b670a/arguments:0 msgid "Red" msgstr "" -#: Favorites/2ba89eb6-6981-4c0d-a19d-3cf1fde52a43/arguments:0 -#: Favorites/dbc3b9d2-e6ce-4ebe-9552-8ddce482c1d1/name:0 +#: Favorites/2ba89eb6-6981-4c0d-a19d-3cf1fde52a43/name:0 +#: Favorites/52d7a9ab-52b7-4e82-ba7f-672fb8d6ec91/arguments:0 msgid "Skol" msgstr "" -#: Favorites/e92b12c5-1817-468e-aa2f-8791fb6247e9/text:0 +#: Favorites/cb6fc9b4-d6e9-4ed3-8a11-3f4d19654a48/text:0 msgid "Sorry you can't participate right now, I'll try again later." msgstr "" -#: Favorites/491f3ed1-9154-4acb-8fdd-0a37567e0574/text:0 +#: Favorites/a602e75e-0814-4034-bb95-770906ddfe34/text:0 msgid "Thanks @results.name, we are all done!" msgstr "" -#: Favorites/58119801-ed31-4538-888d-23779a01707f/arguments:0 -#: Favorites/f1ca9ac8-d0aa-4758-a969-195be7330267/name:0 +#: Favorites/58119801-ed31-4538-888d-23779a01707f/name:0 +#: Favorites/dbc3b9d2-e6ce-4ebe-9552-8ddce482c1d1/arguments:0 msgid "Turbo King" msgstr "" -#: Favorites/8c2504ef-0acc-405f-9efe-d5fc2c434a93/text:0 +#: Favorites/f4495f19-37ee-4e51-a7d5-d99ef6be147a/text:0 msgid "What is your favorite color?" msgstr "" diff --git a/web/po/testdata/import.json b/web/po/testdata/import.json index 81ec0d3fa..f7bcadd96 100644 --- a/web/po/testdata/import.json +++ b/web/po/testdata/import.json @@ -40,12 +40,12 @@ "localization": { "spa": { "34a421ac-34cb-49d8-a2a5-534f52c60851": { - "arguments": [ + "name": [ "Azul" ] }, - "c102acfc-8cc5-41fa-89ed-41cbfa362ba6": { - "name": [ + "baf07ebb-8a2a-4e63-aa08-d19aa408cd45": { + "arguments": [ "Azul" ] } @@ -53,186 +53,186 @@ }, "nodes": [ { - "uuid": "b4664fbd-3495-4fc6-aa8b-b397857dcd68", + "uuid": "10c9c241-777f-4010-a841-6e87abed8520", "actions": [ { "type": "send_msg", - "uuid": "8c2504ef-0acc-405f-9efe-d5fc2c434a93", + "uuid": "f4495f19-37ee-4e51-a7d5-d99ef6be147a", "text": "What is your favorite color?" } ], "exits": [ { - "uuid": "f4495f19-37ee-4e51-a7d5-d99ef6be147a", - "destination_uuid": "10c9c241-777f-4010-a841-6e87abed8520" + "uuid": "943f85bb-50bc-40c3-8d6f-57dbe34c87f7", + "destination_uuid": "5253c207-46e8-42a9-998e-a3e54e0e0542" } ] }, { - "uuid": "1b828e78-e478-4357-9472-47a30ec1f60b", + "uuid": "8c2504ef-0acc-405f-9efe-d5fc2c434a93", "actions": [ { "type": "send_msg", - "uuid": "943f85bb-50bc-40c3-8d6f-57dbe34c87f7", + "uuid": "9631dddf-0dd7-4310-b263-5f7cad4795e0", "text": "I don't know that color. Try again." } ], "exits": [ { - "uuid": "9631dddf-0dd7-4310-b263-5f7cad4795e0", - "destination_uuid": "10c9c241-777f-4010-a841-6e87abed8520" + "uuid": "66c38ec3-0acd-4bf7-a5d5-278af1bee492", + "destination_uuid": "5253c207-46e8-42a9-998e-a3e54e0e0542" } ] }, { - "uuid": "10c9c241-777f-4010-a841-6e87abed8520", + "uuid": "5253c207-46e8-42a9-998e-a3e54e0e0542", "router": { "type": "switch", "wait": { "type": "msg", "timeout": { "seconds": 300, - "category_uuid": "6e367c0c-65ab-479a-82e3-c597d8e35eef" + "category_uuid": "3e2dcf45-ffc0-4197-b5ab-25ed974ea612" } }, "result_name": "Color", "categories": [ { - "uuid": "5563a722-9680-419c-a792-b1fa9df92e06", + "uuid": "3ffb6f24-2ed8-4fd5-bcc0-b2e2668672a8", "name": "Red", - "exit_uuid": "66c38ec3-0acd-4bf7-a5d5-278af1bee492" + "exit_uuid": "eb048bdf-17ee-4334-a52b-5e82a20189ac" }, { - "uuid": "58284598-805a-4740-8966-dcb09e3b670a", + "uuid": "b0c29972-6fd4-485e-83c2-057a3f7a04da", "name": "Green", - "exit_uuid": "eb048bdf-17ee-4334-a52b-5e82a20189ac" + "exit_uuid": "1349bebf-4653-407a-ad25-9fa60e7d7464" }, { - "uuid": "c102acfc-8cc5-41fa-89ed-41cbfa362ba6", + "uuid": "34a421ac-34cb-49d8-a2a5-534f52c60851", "name": "Blue", - "exit_uuid": "1349bebf-4653-407a-ad25-9fa60e7d7464" + "exit_uuid": "37491e99-f4d3-40ae-9ed1-bff62b0e2529" }, { - "uuid": "8d2e259c-bc3c-464f-8c15-985bc736e212", + "uuid": "3b400f91-db69-42b9-9fe2-24ad556b067a", "name": "Cyan", - "exit_uuid": "37491e99-f4d3-40ae-9ed1-bff62b0e2529" + "exit_uuid": "456e75bd-32cc-40c1-a5ef-ffef2e57642c" }, { - "uuid": "c169352e-1944-4451-8d32-eb39c41cb3ae", + "uuid": "5563a722-9680-419c-a792-b1fa9df92e06", "name": "Other", - "exit_uuid": "456e75bd-32cc-40c1-a5ef-ffef2e57642c" + "exit_uuid": "405cf157-1e43-46d8-a0d1-49adcb539267" }, { - "uuid": "6e367c0c-65ab-479a-82e3-c597d8e35eef", + "uuid": "3e2dcf45-ffc0-4197-b5ab-25ed974ea612", "name": "No Response", - "exit_uuid": "405cf157-1e43-46d8-a0d1-49adcb539267" + "exit_uuid": "c169352e-1944-4451-8d32-eb39c41cb3ae" } ], "operand": "@input", "cases": [ { - "uuid": "3ffb6f24-2ed8-4fd5-bcc0-b2e2668672a8", + "uuid": "58284598-805a-4740-8966-dcb09e3b670a", "type": "has_any_word", "arguments": [ "Red" ], - "category_uuid": "5563a722-9680-419c-a792-b1fa9df92e06" + "category_uuid": "3ffb6f24-2ed8-4fd5-bcc0-b2e2668672a8" }, { - "uuid": "b0c29972-6fd4-485e-83c2-057a3f7a04da", + "uuid": "c102acfc-8cc5-41fa-89ed-41cbfa362ba6", "type": "has_any_word", "arguments": [ "Green" ], - "category_uuid": "58284598-805a-4740-8966-dcb09e3b670a" + "category_uuid": "b0c29972-6fd4-485e-83c2-057a3f7a04da" }, { - "uuid": "34a421ac-34cb-49d8-a2a5-534f52c60851", + "uuid": "baf07ebb-8a2a-4e63-aa08-d19aa408cd45", "type": "has_any_word", "arguments": [ "Blue" ], - "category_uuid": "c102acfc-8cc5-41fa-89ed-41cbfa362ba6" + "category_uuid": "34a421ac-34cb-49d8-a2a5-534f52c60851" }, { - "uuid": "baf07ebb-8a2a-4e63-aa08-d19aa408cd45", + "uuid": "8d2e259c-bc3c-464f-8c15-985bc736e212", "type": "has_any_word", "arguments": [ "Navy" ], - "category_uuid": "c102acfc-8cc5-41fa-89ed-41cbfa362ba6" + "category_uuid": "34a421ac-34cb-49d8-a2a5-534f52c60851" }, { - "uuid": "3b400f91-db69-42b9-9fe2-24ad556b067a", + "uuid": "6e367c0c-65ab-479a-82e3-c597d8e35eef", "type": "has_any_word", "arguments": [ "Cyan" ], - "category_uuid": "8d2e259c-bc3c-464f-8c15-985bc736e212" + "category_uuid": "3b400f91-db69-42b9-9fe2-24ad556b067a" } ], - "default_category_uuid": "c169352e-1944-4451-8d32-eb39c41cb3ae" + "default_category_uuid": "5563a722-9680-419c-a792-b1fa9df92e06" }, "exits": [ - { - "uuid": "66c38ec3-0acd-4bf7-a5d5-278af1bee492", - "destination_uuid": "5253c207-46e8-42a9-998e-a3e54e0e0542" - }, { "uuid": "eb048bdf-17ee-4334-a52b-5e82a20189ac", - "destination_uuid": "5253c207-46e8-42a9-998e-a3e54e0e0542" + "destination_uuid": "333fa9a0-85a3-47c5-817e-153a1a124991" }, { "uuid": "1349bebf-4653-407a-ad25-9fa60e7d7464", - "destination_uuid": "5253c207-46e8-42a9-998e-a3e54e0e0542" + "destination_uuid": "333fa9a0-85a3-47c5-817e-153a1a124991" }, { - "uuid": "37491e99-f4d3-40ae-9ed1-bff62b0e2529" + "uuid": "37491e99-f4d3-40ae-9ed1-bff62b0e2529", + "destination_uuid": "333fa9a0-85a3-47c5-817e-153a1a124991" }, { - "uuid": "456e75bd-32cc-40c1-a5ef-ffef2e57642c", - "destination_uuid": "1b828e78-e478-4357-9472-47a30ec1f60b" + "uuid": "456e75bd-32cc-40c1-a5ef-ffef2e57642c" }, { "uuid": "405cf157-1e43-46d8-a0d1-49adcb539267", - "destination_uuid": "b0ae4ad9-5def-4778-8b0a-818d0f4bd3cf" + "destination_uuid": "8c2504ef-0acc-405f-9efe-d5fc2c434a93" + }, + { + "uuid": "c169352e-1944-4451-8d32-eb39c41cb3ae", + "destination_uuid": "1b828e78-e478-4357-9472-47a30ec1f60b" } ] }, { - "uuid": "5253c207-46e8-42a9-998e-a3e54e0e0542", + "uuid": "333fa9a0-85a3-47c5-817e-153a1a124991", "actions": [ { "type": "send_msg", - "uuid": "3e2dcf45-ffc0-4197-b5ab-25ed974ea612", + "uuid": "7624633a-01a9-48f0-abca-957e7290df0a", "text": "Good choice, I like @results.color.category_localized too! What is your favorite beer?" } ], "exits": [ { - "uuid": "7624633a-01a9-48f0-abca-957e7290df0a", - "destination_uuid": "48f2ecb3-8e8e-4f7b-9510-1ee08bd6a434" + "uuid": "4cadf512-1299-468f-85e4-26af9edec193", + "destination_uuid": "a84399b1-0e7b-42ee-8759-473137b510db" } ] }, { - "uuid": "48fd5325-d660-4404-bdf3-05ad1b024cc0", + "uuid": "48f2ecb3-8e8e-4f7b-9510-1ee08bd6a434", "actions": [ { "type": "send_msg", - "uuid": "4cadf512-1299-468f-85e4-26af9edec193", + "uuid": "aac779a9-e2a6-4a11-9efa-9670e081a33a", "text": "I don't know that one, try again please." } ], "exits": [ { - "uuid": "aac779a9-e2a6-4a11-9efa-9670e081a33a", - "destination_uuid": "48f2ecb3-8e8e-4f7b-9510-1ee08bd6a434" + "uuid": "0f0e66a8-9062-444f-b636-3d5374466e31", + "destination_uuid": "a84399b1-0e7b-42ee-8759-473137b510db" } ] }, { - "uuid": "48f2ecb3-8e8e-4f7b-9510-1ee08bd6a434", + "uuid": "a84399b1-0e7b-42ee-8759-473137b510db", "router": { "type": "switch", "wait": { @@ -241,109 +241,109 @@ "result_name": "Beer", "categories": [ { - "uuid": "87b850ff-ddc5-4add-8a4f-c395c3a9ac38", + "uuid": "a813de57-c92a-4128-804d-56e80b332142", "name": "Mutzig", - "exit_uuid": "0f0e66a8-9062-444f-b636-3d5374466e31" + "exit_uuid": "0891f63c-9e82-42bb-a815-8b44aff33046" }, { - "uuid": "b9d718d3-b5e0-4d26-998e-2da31b24f2f9", + "uuid": "a03dceb1-7ac1-491d-93ef-23d3e099633b", "name": "Primus", - "exit_uuid": "0891f63c-9e82-42bb-a815-8b44aff33046" + "exit_uuid": "b341b58e-58fe-41bf-b26e-6274765ccc0e" }, { - "uuid": "f1ca9ac8-d0aa-4758-a969-195be7330267", + "uuid": "58119801-ed31-4538-888d-23779a01707f", "name": "Turbo King", - "exit_uuid": "b341b58e-58fe-41bf-b26e-6274765ccc0e" + "exit_uuid": "e4697b6f-12a9-47ae-a927-96d95d9f8f77" }, { - "uuid": "dbc3b9d2-e6ce-4ebe-9552-8ddce482c1d1", + "uuid": "2ba89eb6-6981-4c0d-a19d-3cf1fde52a43", "name": "Skol", - "exit_uuid": "e4697b6f-12a9-47ae-a927-96d95d9f8f77" + "exit_uuid": "d03c8f97-9f3b-4a6a-8ba9-bdc82a6f09b8" }, { - "uuid": "e0ec2076-2746-43b4-a410-c3af47d6a121", + "uuid": "87b850ff-ddc5-4add-8a4f-c395c3a9ac38", "name": "Other", - "exit_uuid": "d03c8f97-9f3b-4a6a-8ba9-bdc82a6f09b8" + "exit_uuid": "e0ec2076-2746-43b4-a410-c3af47d6a121" } ], "operand": "@input", "cases": [ { - "uuid": "a813de57-c92a-4128-804d-56e80b332142", + "uuid": "b9d718d3-b5e0-4d26-998e-2da31b24f2f9", "type": "has_any_word", "arguments": [ "Mutzig" ], - "category_uuid": "87b850ff-ddc5-4add-8a4f-c395c3a9ac38" + "category_uuid": "a813de57-c92a-4128-804d-56e80b332142" }, { - "uuid": "a03dceb1-7ac1-491d-93ef-23d3e099633b", + "uuid": "f1ca9ac8-d0aa-4758-a969-195be7330267", "type": "has_any_word", "arguments": [ "Primus" ], - "category_uuid": "b9d718d3-b5e0-4d26-998e-2da31b24f2f9" + "category_uuid": "a03dceb1-7ac1-491d-93ef-23d3e099633b" }, { - "uuid": "58119801-ed31-4538-888d-23779a01707f", + "uuid": "dbc3b9d2-e6ce-4ebe-9552-8ddce482c1d1", "type": "has_any_word", "arguments": [ "Turbo King" ], - "category_uuid": "f1ca9ac8-d0aa-4758-a969-195be7330267" + "category_uuid": "58119801-ed31-4538-888d-23779a01707f" }, { - "uuid": "2ba89eb6-6981-4c0d-a19d-3cf1fde52a43", + "uuid": "52d7a9ab-52b7-4e82-ba7f-672fb8d6ec91", "type": "has_any_word", "arguments": [ "Skol" ], - "category_uuid": "dbc3b9d2-e6ce-4ebe-9552-8ddce482c1d1" + "category_uuid": "2ba89eb6-6981-4c0d-a19d-3cf1fde52a43" } ], - "default_category_uuid": "e0ec2076-2746-43b4-a410-c3af47d6a121" + "default_category_uuid": "87b850ff-ddc5-4add-8a4f-c395c3a9ac38" }, "exits": [ - { - "uuid": "0f0e66a8-9062-444f-b636-3d5374466e31", - "destination_uuid": "333fa9a0-85a3-47c5-817e-153a1a124991" - }, { "uuid": "0891f63c-9e82-42bb-a815-8b44aff33046", - "destination_uuid": "333fa9a0-85a3-47c5-817e-153a1a124991" + "destination_uuid": "48fd5325-d660-4404-bdf3-05ad1b024cc0" }, { "uuid": "b341b58e-58fe-41bf-b26e-6274765ccc0e", - "destination_uuid": "333fa9a0-85a3-47c5-817e-153a1a124991" + "destination_uuid": "48fd5325-d660-4404-bdf3-05ad1b024cc0" }, { "uuid": "e4697b6f-12a9-47ae-a927-96d95d9f8f77", - "destination_uuid": "333fa9a0-85a3-47c5-817e-153a1a124991" + "destination_uuid": "48fd5325-d660-4404-bdf3-05ad1b024cc0" }, { "uuid": "d03c8f97-9f3b-4a6a-8ba9-bdc82a6f09b8", "destination_uuid": "48fd5325-d660-4404-bdf3-05ad1b024cc0" + }, + { + "uuid": "e0ec2076-2746-43b4-a410-c3af47d6a121", + "destination_uuid": "48f2ecb3-8e8e-4f7b-9510-1ee08bd6a434" } ] }, { - "uuid": "333fa9a0-85a3-47c5-817e-153a1a124991", + "uuid": "48fd5325-d660-4404-bdf3-05ad1b024cc0", "actions": [ { "type": "send_msg", - "uuid": "52d7a9ab-52b7-4e82-ba7f-672fb8d6ec91", + "uuid": "ada3d96a-a1a2-41eb-aac7-febdb98a9b4c", "text": "Mmmmm... delicious @results.beer.category_localized. If only they made @(lower(results.color)) @results.beer.category_localized! Lastly, what is your name?" } ], "exits": [ { - "uuid": "ada3d96a-a1a2-41eb-aac7-febdb98a9b4c", - "destination_uuid": "a84399b1-0e7b-42ee-8759-473137b510db" + "uuid": "fc551cb4-e797-4076-b40a-433c44ad492b", + "destination_uuid": "b0ae4ad9-5def-4778-8b0a-818d0f4bd3cf" } ] }, { - "uuid": "a84399b1-0e7b-42ee-8759-473137b510db", + "uuid": "b0ae4ad9-5def-4778-8b0a-818d0f4bd3cf", "router": { "type": "switch", "wait": { @@ -352,49 +352,49 @@ "result_name": "Name", "categories": [ { - "uuid": "e87aeeab-8ede-4173-bc76-8f5583ea7207", + "uuid": "491f3ed1-9154-4acb-8fdd-0a37567e0574", "name": "All Responses", - "exit_uuid": "fc551cb4-e797-4076-b40a-433c44ad492b" + "exit_uuid": "e87aeeab-8ede-4173-bc76-8f5583ea7207" } ], "operand": "@input", "cases": [], - "default_category_uuid": "e87aeeab-8ede-4173-bc76-8f5583ea7207" + "default_category_uuid": "491f3ed1-9154-4acb-8fdd-0a37567e0574" }, "exits": [ { - "uuid": "fc551cb4-e797-4076-b40a-433c44ad492b", - "destination_uuid": "5456940a-d3f7-481a-bffe-debdb02c2108" + "uuid": "e87aeeab-8ede-4173-bc76-8f5583ea7207", + "destination_uuid": "b4664fbd-3495-4fc6-aa8b-b397857dcd68" } ] }, { - "uuid": "5456940a-d3f7-481a-bffe-debdb02c2108", + "uuid": "b4664fbd-3495-4fc6-aa8b-b397857dcd68", "actions": [ { "type": "send_msg", - "uuid": "491f3ed1-9154-4acb-8fdd-0a37567e0574", + "uuid": "a602e75e-0814-4034-bb95-770906ddfe34", "text": "Thanks @results.name, we are all done!" } ], "exits": [ { - "uuid": "a602e75e-0814-4034-bb95-770906ddfe34" + "uuid": "e92b12c5-1817-468e-aa2f-8791fb6247e9" } ] }, { - "uuid": "b0ae4ad9-5def-4778-8b0a-818d0f4bd3cf", + "uuid": "1b828e78-e478-4357-9472-47a30ec1f60b", "actions": [ { "type": "send_msg", - "uuid": "e92b12c5-1817-468e-aa2f-8791fb6247e9", + "uuid": "cb6fc9b4-d6e9-4ed3-8a11-3f4d19654a48", "text": "Sorry you can't participate right now, I'll try again later." } ], "exits": [ { - "uuid": "cb6fc9b4-d6e9-4ed3-8a11-3f4d19654a48" + "uuid": "1470d5e6-08dd-479b-a207-9b2b27b924d3" } ] } @@ -402,73 +402,73 @@ "_ui": { "nodes": { "10c9c241-777f-4010-a841-6e87abed8520": { - "type": "wait_for_response", + "type": "execute_actions", "position": { - "top": 129, - "left": 98 + "top": 0, + "left": 100 } }, "1b828e78-e478-4357-9472-47a30ec1f60b": { "type": "execute_actions", "position": { - "top": 8, - "left": 456 + "top": 1278, + "left": 752 } }, "333fa9a0-85a3-47c5-817e-153a1a124991": { "type": "execute_actions", "position": { - "top": 535, - "left": 191 + "top": 237, + "left": 131 } }, "48f2ecb3-8e8e-4f7b-9510-1ee08bd6a434": { - "type": "wait_for_response", + "type": "execute_actions", "position": { - "top": 387, - "left": 112 + "top": 265, + "left": 512 } }, "48fd5325-d660-4404-bdf3-05ad1b024cc0": { "type": "execute_actions", "position": { - "top": 265, - "left": 512 + "top": 535, + "left": 191 } }, "5253c207-46e8-42a9-998e-a3e54e0e0542": { - "type": "execute_actions", + "type": "wait_for_response", "position": { - "top": 237, - "left": 131 + "top": 129, + "left": 98 } }, - "5456940a-d3f7-481a-bffe-debdb02c2108": { + "8c2504ef-0acc-405f-9efe-d5fc2c434a93": { "type": "execute_actions", "position": { - "top": 805, - "left": 191 + "top": 8, + "left": 456 } }, "a84399b1-0e7b-42ee-8759-473137b510db": { "type": "wait_for_response", "position": { - "top": 702, - "left": 191 + "top": 387, + "left": 112 } }, "b0ae4ad9-5def-4778-8b0a-818d0f4bd3cf": { - "type": "execute_actions", + "type": "wait_for_response", "position": { - "top": 1278, - "left": 752 + "top": 702, + "left": 191 } }, "b4664fbd-3495-4fc6-aa8b-b397857dcd68": { "type": "execute_actions", "position": { - "top": 0, - "left": 100 + "top": 805, + "left": 191 } } }, diff --git a/web/po/testdata/multiple_flows.es.po b/web/po/testdata/multiple_flows.es.po index 0c6e6159a..802df8126 100644 --- a/web/po/testdata/multiple_flows.es.po +++ b/web/po/testdata/multiple_flows.es.po @@ -10,110 +10,110 @@ msgstr "" "Language-3: spa\n" "Source-Flows: 9de3663f-c5c5-4c92-9f45-ecbc09abcc85; 5890fe3a-f204-4661-b74d-025be4ee019c\n" -#: Pick+a+Number/f3087862-dca9-4eaf-8cea-13f85cb52353/arguments:0 +#: Pick+a+Number/f90c9734-3e58-4c07-96cc-315266c8ecfd/arguments:0 msgid "1" msgstr "" -#: Pick+a+Number/0d15ae52-5ad9-4d64-9c64-e27545d48a19/name:0 +#: Pick+a+Number/f3087862-dca9-4eaf-8cea-13f85cb52353/name:0 msgid "1-10" msgstr "" -#: Pick+a+Number/f3087862-dca9-4eaf-8cea-13f85cb52353/arguments:1 +#: Pick+a+Number/f90c9734-3e58-4c07-96cc-315266c8ecfd/arguments:1 msgid "10" msgstr "" -#: Favorites/e87aeeab-8ede-4173-bc76-8f5583ea7207/name:0 -#: Pick+a+Number/225915f1-fb26-48a5-b457-d2ea4300b575/name:0 +#: Favorites/491f3ed1-9154-4acb-8fdd-0a37567e0574/name:0 +#: Pick+a+Number/5bf24536-9ae1-466a-9b76-5c82626d3153/name:0 msgid "All Responses" msgstr "" -#: Favorites/34a421ac-34cb-49d8-a2a5-534f52c60851/arguments:0 -#: Favorites/c102acfc-8cc5-41fa-89ed-41cbfa362ba6/name:0 +#: Favorites/34a421ac-34cb-49d8-a2a5-534f52c60851/name:0 +#: Favorites/baf07ebb-8a2a-4e63-aa08-d19aa408cd45/arguments:0 msgid "Blue" msgstr "" -#: Favorites/3b400f91-db69-42b9-9fe2-24ad556b067a/arguments:0 -#: Favorites/8d2e259c-bc3c-464f-8c15-985bc736e212/name:0 +#: Favorites/3b400f91-db69-42b9-9fe2-24ad556b067a/name:0 +#: Favorites/6e367c0c-65ab-479a-82e3-c597d8e35eef/arguments:0 msgid "Cyan" msgstr "" -#: Favorites/3e2dcf45-ffc0-4197-b5ab-25ed974ea612/text:0 +#: Favorites/7624633a-01a9-48f0-abca-957e7290df0a/text:0 msgid "Good choice, I like @results.color.category_localized too! What is your favorite beer?" msgstr "" -#: Favorites/58284598-805a-4740-8966-dcb09e3b670a/name:0 -#: Favorites/b0c29972-6fd4-485e-83c2-057a3f7a04da/arguments:0 +#: Favorites/b0c29972-6fd4-485e-83c2-057a3f7a04da/name:0 +#: Favorites/c102acfc-8cc5-41fa-89ed-41cbfa362ba6/arguments:0 msgid "Green" msgstr "" -#: Favorites/943f85bb-50bc-40c3-8d6f-57dbe34c87f7/text:0 +#: Favorites/9631dddf-0dd7-4310-b263-5f7cad4795e0/text:0 msgid "I don't know that color. Try again." msgstr "" -#: Favorites/4cadf512-1299-468f-85e4-26af9edec193/text:0 +#: Favorites/aac779a9-e2a6-4a11-9efa-9670e081a33a/text:0 msgid "I don't know that one, try again please." msgstr "" -#: Favorites/52d7a9ab-52b7-4e82-ba7f-672fb8d6ec91/text:0 +#: Favorites/ada3d96a-a1a2-41eb-aac7-febdb98a9b4c/text:0 msgid "Mmmmm... delicious @results.beer.category_localized. If only they made @(lower(results.color)) @results.beer.category_localized! Lastly, what is your name?" msgstr "" -#: Favorites/87b850ff-ddc5-4add-8a4f-c395c3a9ac38/name:0 -#: Favorites/a813de57-c92a-4128-804d-56e80b332142/arguments:0 +#: Favorites/a813de57-c92a-4128-804d-56e80b332142/name:0 +#: Favorites/b9d718d3-b5e0-4d26-998e-2da31b24f2f9/arguments:0 msgid "Mutzig" msgstr "" -#: Favorites/baf07ebb-8a2a-4e63-aa08-d19aa408cd45/arguments:0 +#: Favorites/8d2e259c-bc3c-464f-8c15-985bc736e212/arguments:0 msgid "Navy" msgstr "" -#: Favorites/6e367c0c-65ab-479a-82e3-c597d8e35eef/name:0 +#: Favorites/3e2dcf45-ffc0-4197-b5ab-25ed974ea612/name:0 msgid "No Response" msgstr "" -#: Favorites/c169352e-1944-4451-8d32-eb39c41cb3ae/name:0 -#: Favorites/e0ec2076-2746-43b4-a410-c3af47d6a121/name:0 -#: Pick+a+Number/34ef666f-24a0-41cc-b364-e8b08a6e89ff/name:0 +#: Favorites/5563a722-9680-419c-a792-b1fa9df92e06/name:0 +#: Favorites/87b850ff-ddc5-4add-8a4f-c395c3a9ac38/name:0 +#: Pick+a+Number/0d15ae52-5ad9-4d64-9c64-e27545d48a19/name:0 msgid "Other" msgstr "" -#: Pick+a+Number/a39e1d4b-ceda-45e5-b889-11ddcbc77fde/text:0 +#: Pick+a+Number/bb3276d9-543b-427b-9d00-a926dabc8e24/text:0 msgid "Pick a number between 1-10." msgstr "" -#: Favorites/a03dceb1-7ac1-491d-93ef-23d3e099633b/arguments:0 -#: Favorites/b9d718d3-b5e0-4d26-998e-2da31b24f2f9/name:0 +#: Favorites/a03dceb1-7ac1-491d-93ef-23d3e099633b/name:0 +#: Favorites/f1ca9ac8-d0aa-4758-a969-195be7330267/arguments:0 msgid "Primus" msgstr "" -#: Favorites/3ffb6f24-2ed8-4fd5-bcc0-b2e2668672a8/arguments:0 -#: Favorites/5563a722-9680-419c-a792-b1fa9df92e06/name:0 +#: Favorites/3ffb6f24-2ed8-4fd5-bcc0-b2e2668672a8/name:0 +#: Favorites/58284598-805a-4740-8966-dcb09e3b670a/arguments:0 msgid "Red" msgstr "" -#: Favorites/2ba89eb6-6981-4c0d-a19d-3cf1fde52a43/arguments:0 -#: Favorites/dbc3b9d2-e6ce-4ebe-9552-8ddce482c1d1/name:0 +#: Favorites/2ba89eb6-6981-4c0d-a19d-3cf1fde52a43/name:0 +#: Favorites/52d7a9ab-52b7-4e82-ba7f-672fb8d6ec91/arguments:0 msgid "Skol" msgstr "" -#: Favorites/e92b12c5-1817-468e-aa2f-8791fb6247e9/text:0 +#: Favorites/cb6fc9b4-d6e9-4ed3-8a11-3f4d19654a48/text:0 msgid "Sorry you can't participate right now, I'll try again later." msgstr "" -#: Favorites/491f3ed1-9154-4acb-8fdd-0a37567e0574/text:0 +#: Favorites/a602e75e-0814-4034-bb95-770906ddfe34/text:0 msgid "Thanks @results.name, we are all done!" msgstr "" -#: Favorites/58119801-ed31-4538-888d-23779a01707f/arguments:0 -#: Favorites/f1ca9ac8-d0aa-4758-a969-195be7330267/name:0 +#: Favorites/58119801-ed31-4538-888d-23779a01707f/name:0 +#: Favorites/dbc3b9d2-e6ce-4ebe-9552-8ddce482c1d1/arguments:0 msgid "Turbo King" msgstr "" -#: Favorites/8c2504ef-0acc-405f-9efe-d5fc2c434a93/text:0 +#: Favorites/f4495f19-37ee-4e51-a7d5-d99ef6be147a/text:0 msgid "What is your favorite color?" msgstr "" -#: Pick+a+Number/f90c9734-3e58-4c07-96cc-315266c8ecfd/text:0 +#: Pick+a+Number/b634f07f-7b2d-47bd-8795-051e56cf2609/text:0 msgid "You picked @results.number!" msgstr "" diff --git a/web/po/testdata/multiple_flows_noargs.es.po b/web/po/testdata/multiple_flows_noargs.es.po index c54321a08..81d0a3ea4 100644 --- a/web/po/testdata/multiple_flows_noargs.es.po +++ b/web/po/testdata/multiple_flows_noargs.es.po @@ -10,90 +10,90 @@ msgstr "" "Language-3: spa\n" "Source-Flows: 9de3663f-c5c5-4c92-9f45-ecbc09abcc85; 5890fe3a-f204-4661-b74d-025be4ee019c\n" -#: Pick+a+Number/0d15ae52-5ad9-4d64-9c64-e27545d48a19/name:0 +#: Pick+a+Number/f3087862-dca9-4eaf-8cea-13f85cb52353/name:0 msgid "1-10" msgstr "" -#: Favorites/e87aeeab-8ede-4173-bc76-8f5583ea7207/name:0 -#: Pick+a+Number/225915f1-fb26-48a5-b457-d2ea4300b575/name:0 +#: Favorites/491f3ed1-9154-4acb-8fdd-0a37567e0574/name:0 +#: Pick+a+Number/5bf24536-9ae1-466a-9b76-5c82626d3153/name:0 msgid "All Responses" msgstr "" -#: Favorites/c102acfc-8cc5-41fa-89ed-41cbfa362ba6/name:0 +#: Favorites/34a421ac-34cb-49d8-a2a5-534f52c60851/name:0 msgid "Blue" msgstr "" -#: Favorites/8d2e259c-bc3c-464f-8c15-985bc736e212/name:0 +#: Favorites/3b400f91-db69-42b9-9fe2-24ad556b067a/name:0 msgid "Cyan" msgstr "" -#: Favorites/3e2dcf45-ffc0-4197-b5ab-25ed974ea612/text:0 +#: Favorites/7624633a-01a9-48f0-abca-957e7290df0a/text:0 msgid "Good choice, I like @results.color.category_localized too! What is your favorite beer?" msgstr "" -#: Favorites/58284598-805a-4740-8966-dcb09e3b670a/name:0 +#: Favorites/b0c29972-6fd4-485e-83c2-057a3f7a04da/name:0 msgid "Green" msgstr "" -#: Favorites/943f85bb-50bc-40c3-8d6f-57dbe34c87f7/text:0 +#: Favorites/9631dddf-0dd7-4310-b263-5f7cad4795e0/text:0 msgid "I don't know that color. Try again." msgstr "" -#: Favorites/4cadf512-1299-468f-85e4-26af9edec193/text:0 +#: Favorites/aac779a9-e2a6-4a11-9efa-9670e081a33a/text:0 msgid "I don't know that one, try again please." msgstr "" -#: Favorites/52d7a9ab-52b7-4e82-ba7f-672fb8d6ec91/text:0 +#: Favorites/ada3d96a-a1a2-41eb-aac7-febdb98a9b4c/text:0 msgid "Mmmmm... delicious @results.beer.category_localized. If only they made @(lower(results.color)) @results.beer.category_localized! Lastly, what is your name?" msgstr "" -#: Favorites/87b850ff-ddc5-4add-8a4f-c395c3a9ac38/name:0 +#: Favorites/a813de57-c92a-4128-804d-56e80b332142/name:0 msgid "Mutzig" msgstr "" -#: Favorites/6e367c0c-65ab-479a-82e3-c597d8e35eef/name:0 +#: Favorites/3e2dcf45-ffc0-4197-b5ab-25ed974ea612/name:0 msgid "No Response" msgstr "" -#: Favorites/c169352e-1944-4451-8d32-eb39c41cb3ae/name:0 -#: Favorites/e0ec2076-2746-43b4-a410-c3af47d6a121/name:0 -#: Pick+a+Number/34ef666f-24a0-41cc-b364-e8b08a6e89ff/name:0 +#: Favorites/5563a722-9680-419c-a792-b1fa9df92e06/name:0 +#: Favorites/87b850ff-ddc5-4add-8a4f-c395c3a9ac38/name:0 +#: Pick+a+Number/0d15ae52-5ad9-4d64-9c64-e27545d48a19/name:0 msgid "Other" msgstr "" -#: Pick+a+Number/a39e1d4b-ceda-45e5-b889-11ddcbc77fde/text:0 +#: Pick+a+Number/bb3276d9-543b-427b-9d00-a926dabc8e24/text:0 msgid "Pick a number between 1-10." msgstr "" -#: Favorites/b9d718d3-b5e0-4d26-998e-2da31b24f2f9/name:0 +#: Favorites/a03dceb1-7ac1-491d-93ef-23d3e099633b/name:0 msgid "Primus" msgstr "" -#: Favorites/5563a722-9680-419c-a792-b1fa9df92e06/name:0 +#: Favorites/3ffb6f24-2ed8-4fd5-bcc0-b2e2668672a8/name:0 msgid "Red" msgstr "" -#: Favorites/dbc3b9d2-e6ce-4ebe-9552-8ddce482c1d1/name:0 +#: Favorites/2ba89eb6-6981-4c0d-a19d-3cf1fde52a43/name:0 msgid "Skol" msgstr "" -#: Favorites/e92b12c5-1817-468e-aa2f-8791fb6247e9/text:0 +#: Favorites/cb6fc9b4-d6e9-4ed3-8a11-3f4d19654a48/text:0 msgid "Sorry you can't participate right now, I'll try again later." msgstr "" -#: Favorites/491f3ed1-9154-4acb-8fdd-0a37567e0574/text:0 +#: Favorites/a602e75e-0814-4034-bb95-770906ddfe34/text:0 msgid "Thanks @results.name, we are all done!" msgstr "" -#: Favorites/f1ca9ac8-d0aa-4758-a969-195be7330267/name:0 +#: Favorites/58119801-ed31-4538-888d-23779a01707f/name:0 msgid "Turbo King" msgstr "" -#: Favorites/8c2504ef-0acc-405f-9efe-d5fc2c434a93/text:0 +#: Favorites/f4495f19-37ee-4e51-a7d5-d99ef6be147a/text:0 msgid "What is your favorite color?" msgstr "" -#: Pick+a+Number/f90c9734-3e58-4c07-96cc-315266c8ecfd/text:0 +#: Pick+a+Number/b634f07f-7b2d-47bd-8795-051e56cf2609/text:0 msgid "You picked @results.number!" msgstr "" diff --git a/web/server.go b/web/server.go index 2a3116f7a..21db50dfd 100644 --- a/web/server.go +++ b/web/server.go @@ -1,6 +1,7 @@ package web import ( + "compress/flate" "context" "encoding/json" "fmt" @@ -9,8 +10,8 @@ import ( "time" "github.com/nyaruka/gocommon/jsonx" + "github.com/nyaruka/gocommon/storage" "github.com/nyaruka/mailroom/config" - "github.com/nyaruka/mailroom/utils/storage" "github.com/go-chi/chi" "github.com/go-chi/chi/middleware" @@ -75,7 +76,7 @@ func NewServer(ctx context.Context, config *config.Config, db *sqlx.DB, rp *redi router := chi.NewRouter() // set up our middlewares - router.Use(middleware.DefaultCompress) + router.Use(middleware.Compress(flate.DefaultCompression)) router.Use(middleware.RequestID) router.Use(middleware.RealIP) router.Use(panicRecovery) diff --git a/web/surveyor/surveyor.go b/web/surveyor/surveyor.go index 5758376e7..0ed1a323e 100644 --- a/web/surveyor/surveyor.go +++ b/web/surveyor/surveyor.go @@ -5,8 +5,8 @@ import ( "encoding/json" "net/http" - "github.com/nyaruka/gocommon/urns" "github.com/nyaruka/goflow/assets" + "github.com/nyaruka/goflow/envs" "github.com/nyaruka/goflow/flows" "github.com/nyaruka/goflow/flows/engine" "github.com/nyaruka/goflow/flows/events" @@ -88,31 +88,22 @@ func handleSubmit(ctx context.Context, s *web.Server, r *http.Request) (interfac return nil, http.StatusBadRequest, err } - // create / assign our contact - urn := urns.NilURN - if len(fs.Contact().URNs()) > 0 { - urn = fs.Contact().URNs()[0].URN() - } + // get the current version of this contact from the database + var flowContact *flows.Contact - // create / fetch our contact based on the highest priority URN - contactID, err := models.CreateContact(ctx, s.DB, oa, urn) - if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to look up contact") - } - - // load that contact to get the current groups and UUID - contacts, err := models.LoadContacts(ctx, s.DB, oa, []models.ContactID{contactID}) - if err == nil && len(contacts) == 0 { - err = errors.Errorf("no contacts loaded") - } - if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "error loading contact") - } + if len(fs.Contact().URNs()) > 0 { + // create / fetch our contact based on the highest priority URN + urn := fs.Contact().URNs()[0].URN() - // load our flow contact - flowContact, err := contacts[0].FlowContact(oa) - if err != nil { - return nil, http.StatusInternalServerError, errors.Wrapf(err, "error loading flow contact") + _, flowContact, err = models.GetOrCreateContact(ctx, s.DB, oa, urn) + if err != nil { + return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to look up contact") + } + } else { + _, flowContact, err = models.CreateContact(ctx, s.DB, oa, models.NilUserID, "", envs.NilLanguage, nil) + if err != nil { + return nil, http.StatusInternalServerError, errors.Wrapf(err, "unable to create contact") + } } modifierEvents := make([]flows.Event, 0, len(mods)) diff --git a/web/testing.go b/web/testing.go index 12e4b22bf..fc1ed4f59 100644 --- a/web/testing.go +++ b/web/testing.go @@ -16,8 +16,8 @@ import ( "github.com/nyaruka/gocommon/dates" "github.com/nyaruka/gocommon/httpx" "github.com/nyaruka/gocommon/jsonx" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/goflow/test" - "github.com/nyaruka/goflow/utils/uuids" "github.com/nyaruka/mailroom/config" "github.com/nyaruka/mailroom/testsuite" @@ -36,7 +36,9 @@ func RunWebTests(t *testing.T, truthFile string) { defer dates.SetNowSource(dates.DefaultNowSource) - server := NewServer(context.Background(), config.Mailroom, db, rp, nil, nil, wg) + defer testsuite.ResetStorage() + + server := NewServer(context.Background(), config.Mailroom, db, rp, testsuite.Storage(), nil, wg) server.Start() defer server.Stop() diff --git a/web/ticket/ticket_test.go b/web/ticket/ticket_test.go index 527cbb469..26890cb7d 100644 --- a/web/ticket/ticket_test.go +++ b/web/ticket/ticket_test.go @@ -3,7 +3,7 @@ package ticket import ( "testing" - "github.com/nyaruka/goflow/utils/uuids" + "github.com/nyaruka/gocommon/uuids" "github.com/nyaruka/mailroom/models" _ "github.com/nyaruka/mailroom/services/tickets/mailgun" _ "github.com/nyaruka/mailroom/services/tickets/zendesk"