diff --git a/.github/workflows/build-rp-indexer-push-tag-india-ire.yaml b/.github/workflows/build-rp-indexer-push-tag-india-ire.yaml index ef24d9b..5fc13dd 100644 --- a/.github/workflows/build-rp-indexer-push-tag-india-ire.yaml +++ b/.github/workflows/build-rp-indexer-push-tag-india-ire.yaml @@ -16,7 +16,6 @@ jobs: if grep -qs -e '^.*.*-develop' <<< "${TAG}" ; then echo "Found environment: DEVELOP - ${TAG}" echo "ENVIRONMENT=develop" | tee -a "${GITHUB_ENV}" - exit 1 # stop action elif grep -qs -e '^.*.*-staging' <<< "${TAG}" ; then echo "Found environment: STAGING - ${TAG}" echo "ENVIRONMENT=staging" | tee -a "${GITHUB_ENV}" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84fd438..dac1a9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ name: CI on: [push, pull_request] env: - go-version: "1.19.x" + go-version: "1.21.x" jobs: test: name: Test diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a09aa7..cf588be 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +v9.0.0 (2024-01-05) +------------------------- + * Update dependencies + +v8.3.1 (2023-11-02) +------------------------- + * Update docker dev image to go 1.21 + * Start replacing logrus with slog + * Bump golang.org/x/net from 0.14.0 to 0.17.0 + * Replace deprecated ioutil calls + * Pass context to fetching of contacts + +v8.3.0 (2023-08-10) +------------------------- + * Update to go 1.20 + v8.2.0 (2023-07-31) ------------------------- * Fix docker file diff --git a/Dockerfile b/Dockerfile index 3171e5a..69c6d24 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.20 +FROM golang:1.21 WORKDIR /usr/src/app @@ -9,4 +9,4 @@ RUN go mod download && go mod verify COPY . . RUN go build -v -o /usr/local/bin/app github.com/nyaruka/rp-indexer/v8/cmd/rp-indexer -CMD ["app"] \ No newline at end of file +CMD ["app"] diff --git a/WENI-CHANGELOG.md b/WENI-CHANGELOG.md index 3ba07f6..d3a0208 100644 --- a/WENI-CHANGELOG.md +++ b/WENI-CHANGELOG.md @@ -1,3 +1,8 @@ +1.2.1-indexer-9.0.0 +---------- + * Update to version v9.0.0 + * Fix make json request + 1.2.0-indexer-8.2.0 ---------- * Update go version and add retry modifications \ No newline at end of file diff --git a/cmd/rp-indexer/main.go b/cmd/rp-indexer/main.go index 9cda279..a8d5edb 100644 --- a/cmd/rp-indexer/main.go +++ b/cmd/rp-indexer/main.go @@ -2,6 +2,9 @@ package main import ( "database/sql" + "log" + + "log/slog" "os" "os/signal" "syscall" @@ -13,7 +16,8 @@ import ( "github.com/nyaruka/ezconf" indexer "github.com/nyaruka/rp-indexer/v8" "github.com/nyaruka/rp-indexer/v8/indexers" - log "github.com/sirupsen/logrus" + "github.com/nyaruka/rp-indexer/v8/utils" + "github.com/sirupsen/logrus" ) var ( @@ -27,27 +31,33 @@ func main() { loader := ezconf.NewLoader(cfg, "indexer", "Indexes RapidPro contacts to ElasticSearch", []string{"indexer.toml"}) loader.MustLoad() - level, err := log.ParseLevel(cfg.LogLevel) + level, err := logrus.ParseLevel(cfg.LogLevel) if err != nil { - log.Fatalf("Invalid log level '%s'", level) + logrus.Fatalf("Invalid log level '%s'", level) } - log.SetLevel(level) - log.SetOutput(os.Stdout) - log.SetFormatter(&log.TextFormatter{}) - log.WithField("version", version).WithField("released", date).Info("starting indexer") + logrus.SetLevel(level) + logrus.SetOutput(os.Stdout) + logrus.SetFormatter(&logrus.TextFormatter{}) + logrus.WithField("version", version).WithField("released", date).Info("starting indexer") + + // configure golang std structured logging to route to logrus + slog.SetDefault(slog.New(utils.NewLogrusHandler(logrus.StandardLogger()))) + + logger := slog.With("comp", "main") + logger.Info("starting indexer", "version", version, "released", date) // if we have a DSN entry, try to initialize it if cfg.SentryDSN != "" { - hook, err := logrus_sentry.NewSentryHook(cfg.SentryDSN, []log.Level{log.PanicLevel, log.FatalLevel, log.ErrorLevel}) + hook, err := logrus_sentry.NewSentryHook(cfg.SentryDSN, []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel}) hook.Timeout = 0 hook.StacktraceConfiguration.Enable = true hook.StacktraceConfiguration.Skip = 4 hook.StacktraceConfiguration.Context = 5 if err != nil { - log.Fatalf("invalid sentry DSN: '%s': %s", cfg.SentryDSN, err) + logger.Error("invalid sentry DSN: '%s': %s", cfg.SentryDSN, err) } - log.StandardLogger().Hooks.Add(hook) + logrus.StandardLogger().Hooks.Add(hook) err = sentry.Init(sentry.ClientOptions{ Dsn: cfg.SentryDSN, @@ -60,7 +70,7 @@ func main() { db, err := sql.Open("postgres", cfg.DB) if err != nil { - log.Fatalf("unable to connect to database") + logger.Error("unable to connect to database") } idxrs := []indexers.Indexer{ @@ -72,7 +82,7 @@ func main() { // the rebuild argument can be become the name of the index to rebuild, e.g. --rebuild=contacts idxr := idxrs[0] if _, err := idxr.Index(db, true, cfg.Cleanup); err != nil { - log.WithField("indexer", idxr.Name()).WithError(err).Fatal("error during rebuilding") + logger.Error("error during rebuilding", "error", err, "indexer", idxr.Name()) } } else { d := indexer.NewDaemon(cfg, db, idxrs, time.Duration(cfg.Poll)*time.Second) @@ -91,7 +101,7 @@ func handleSignals(d *indexer.Daemon) { sig := <-sigs switch sig { case syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT: - log.WithField("signal", sig).Info("received exit signal, exiting") + slog.Info("received exit signal, exiting", "signal", sig) d.Stop() return } diff --git a/daemon.go b/daemon.go index 1ede36f..c3e7ee3 100644 --- a/daemon.go +++ b/daemon.go @@ -2,12 +2,12 @@ package indexer import ( "database/sql" + "log/slog" "sync" "time" "github.com/nyaruka/gocommon/analytics" "github.com/nyaruka/rp-indexer/v8/indexers" - "github.com/sirupsen/logrus" ) type Daemon struct { @@ -53,7 +53,7 @@ func (d *Daemon) Start() { func (d *Daemon) startIndexer(indexer indexers.Indexer) { d.wg.Add(1) // add ourselves to the wait group - log := logrus.WithField("indexer", indexer.Name()) + log := slog.With("indexer", indexer.Name()) go func() { defer func() { @@ -68,7 +68,7 @@ func (d *Daemon) startIndexer(indexer indexers.Indexer) { case <-time.After(d.poll): _, err := indexer.Index(d.db, d.cfg.Rebuild, d.cfg.Cleanup) if err != nil { - log.WithError(err).Error("error during indexing") + log.Error("error during indexing", "error", err) } } } @@ -80,7 +80,7 @@ func (d *Daemon) startStatsReporter(interval time.Duration) { go func() { defer func() { - logrus.Info("analytics exiting") + slog.Info("analytics exiting") d.wg.Done() }() @@ -117,11 +117,11 @@ func (d *Daemon) reportStats() { d.prevStats[ix] = stats } - log := logrus.NewEntry(logrus.StandardLogger()) + log := slog.New(slog.Default().Handler()) for k, v := range metrics { analytics.Gauge("indexer."+k, v) - log = log.WithField(k, v) + log = log.With(k, v) } log.Info("stats reported") @@ -129,7 +129,7 @@ func (d *Daemon) reportStats() { // Stop stops this daemon func (d *Daemon) Stop() { - logrus.Info("daemon stopping") + slog.Info("daemon stopping") analytics.Stop() close(d.quit) diff --git a/docker/Dockerfile b/docker/Dockerfile index 34bebc8..c0c6de6 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,20 +1,19 @@ -FROM golang:1.19-alpine3.18 +FROM golang:1.23-bookworm AS builder -WORKDIR /app +WORKDIR /src + +COPY go.mod go.sum ./ +RUN go mod download -x -RUN apk update \ - && apk add --virtual build-deps gcc git \ - && rm -rf /var/cache/apk/* +COPY . ./ -RUN addgroup -S golang \ - && adduser -S -G golang golang +RUN GOOS=linux GOARCH=amd64 go build -o /bin/rp-indexer ./cmd/rp-indexer/*.go -COPY . . +FROM gcr.io/distroless/base-debian12 -RUN go install -v ./cmd/... -RUN chown -R golang /app +WORKDIR /app -USER golang +COPY --from=builder bin/rp-indexer ./ EXPOSE 8080 -ENTRYPOINT ["rp-indexer"] +ENTRYPOINT ["./rp-indexer"] diff --git a/go.mod b/go.mod index 9e028dc..5717feb 100644 --- a/go.mod +++ b/go.mod @@ -1,16 +1,16 @@ module github.com/nyaruka/rp-indexer/v8 -go 1.19 +go 1.21 require ( github.com/evalphobia/logrus_sentry v0.8.2 - github.com/lib/pq v1.10.7 + github.com/lib/pq v1.10.9 github.com/nyaruka/ezconf v0.2.1 - github.com/nyaruka/gocommon v1.31.0 + github.com/nyaruka/gocommon v1.42.7 github.com/olivere/elastic/v7 v7.0.32 github.com/pkg/errors v0.9.1 - github.com/sirupsen/logrus v1.9.0 - github.com/stretchr/testify v1.8.2 + github.com/sirupsen/logrus v1.9.3 + github.com/stretchr/testify v1.8.4 ) require golang.org/x/text v0.14.0 // indirect @@ -19,7 +19,7 @@ require ( github.com/certifi/gocertifi v0.0.0-20210507211836-431795d63e8d // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/structs v1.1.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.1 // indirect + github.com/gabriel-vasile/mimetype v1.4.3 // indirect github.com/getsentry/raven-go v0.2.0 // indirect github.com/getsentry/sentry-go v0.29.0 github.com/go-chi/chi v4.1.2+incompatible // indirect @@ -28,10 +28,14 @@ require ( github.com/mailru/easyjson v0.7.7 // indirect github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.1 // indirect - github.com/nyaruka/librato v1.0.0 // indirect + github.com/nyaruka/librato v1.1.1 // indirect + github.com/nyaruka/null/v2 v2.0.3 // indirect + github.com/nyaruka/phonenumbers v1.3.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect + golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.18.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4101d57..858eb00 100644 --- a/go.sum +++ b/go.sum @@ -9,8 +9,9 @@ github.com/fatih/structs v1.0.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= -github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q= -github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/getsentry/raven-go v0.2.0 h1:no+xWJRb5ZI7eE8TWgIq1jLulQiIoLG0IfYxv5JYMGs= github.com/getsentry/raven-go v0.2.0/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= github.com/getsentry/sentry-go v0.29.0 h1:YtWluuCFg9OfcqnaujpY918N/AhCCwarIDWOYSBAjCA= @@ -18,15 +19,17 @@ github.com/getsentry/sentry-go v0.29.0/go.mod h1:jhPesDAL0Q0W2+2YEuVOvdWmVtdsr1+ github.com/go-chi/chi v4.1.2+incompatible h1:fGFk2Gmi/YKXk0OmGfBh0WgmN3XB8lVnEyNz34tQRec= github.com/go-chi/chi v4.1.2+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxmMeXJVKy9tTv1XzQ= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= -github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= @@ -35,50 +38,43 @@ github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/nyaruka/ezconf v0.2.1 h1:TDXWoqjqYya1uhou1mAJZg7rgFYL98EB0Tb3+BWtUh0= github.com/nyaruka/ezconf v0.2.1/go.mod h1:ey182kYkw2MIi4XiWe1FR/mzI33WCmTWuceDYYxgnQw= -github.com/nyaruka/gocommon v1.31.0 h1:eVRxmyTZxRQ4mBs3JoYaPe33LlNuQD63pxq2M+eHQA8= -github.com/nyaruka/gocommon v1.31.0/go.mod h1:PApT/06fP5Tzs4/kbkJ+rVoyOc9Lbqm1lR0ow8Vqzp0= -github.com/nyaruka/librato v1.0.0 h1:Vznj9WCeC1yZXbBYyYp40KnbmXLbEkjKmHesV/v2SR0= -github.com/nyaruka/librato v1.0.0/go.mod h1:pkRNLFhFurOz0QqBz6/DuTFhHHxAubWxs4Jx+J7yUgg= +github.com/nyaruka/gocommon v1.42.7 h1:4U7Ta1LIHVc/uv8sfqmmV5oRiFU8TcJM9a7QjxVoaeA= +github.com/nyaruka/gocommon v1.42.7/go.mod h1:DMj0TJPT2zi6eoXrBSsJTGBxSAUkpBk+UzcMyAbq5DA= +github.com/nyaruka/librato v1.1.1 h1:0nTYtJLl3Sn7lX3CuHsLf+nXy1k/tGV0OjVxLy3Et4s= +github.com/nyaruka/librato v1.1.1/go.mod h1:fme1Fu1PT2qvkaBZyw8WW+SrnFe2qeeCWpvqmAaKAKE= +github.com/nyaruka/null/v2 v2.0.3 h1:rdmMRQyVzrOF3Jff/gpU/7BDR9mQX0lcLl4yImsA3kw= +github.com/nyaruka/null/v2 v2.0.3/go.mod h1:OCVeCkCXwrg5/qE6RU0c1oUVZBy+ZDrT+xYg1XSaIWA= +github.com/nyaruka/phonenumbers v1.3.0 h1:IFyyJfF2Elg8xGKFghWrRXzb6qAHk+Q3uPqmIgS20JQ= +github.com/nyaruka/phonenumbers v1.3.0/go.mod h1:4jyKp/BFUokLbCHyoZag+T3S1KezFVoEKtgnbpzItC4= github.com/olivere/elastic/v7 v7.0.32 h1:R7CXvbu8Eq+WlsLgxmKVKPox0oOwAE/2T9Si5BnvK6E= github.com/olivere/elastic/v7 v7.0.32/go.mod h1:c7PVmLe3Fxq77PIfY/bZmxY/TAamBhCzZ8xDOE09a9k= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= +golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/indexers/base.go b/indexers/base.go index 5c85a68..f2f4d0f 100644 --- a/indexers/base.go +++ b/indexers/base.go @@ -4,6 +4,7 @@ import ( "database/sql" "encoding/json" "fmt" + "log/slog" "net/http" "sort" "strings" @@ -11,7 +12,6 @@ import ( "github.com/nyaruka/gocommon/jsonx" "github.com/nyaruka/rp-indexer/v8/utils" - "github.com/sirupsen/logrus" ) // indexes a document @@ -76,8 +76,8 @@ func (i *baseIndexer) Stats() Stats { return i.stats } -func (i *baseIndexer) log() *logrus.Entry { - return logrus.WithField("indexer", i.name) +func (i *baseIndexer) log() *slog.Logger { + return slog.With("indexer", i.name) } // records a complete index and updates statistics @@ -86,7 +86,7 @@ func (i *baseIndexer) recordComplete(indexed, deleted int, elapsed time.Duration i.stats.Deleted += int64(deleted) i.stats.Elapsed += elapsed - i.log().WithField("indexed", indexed).WithField("deleted", deleted).WithField("elapsed", elapsed).Info("completed indexing") + i.log().Info("completed indexing", "indexed", indexed, "deleted", deleted, "elapsed", elapsed) } // our response for figuring out the physical index for an alias @@ -111,7 +111,7 @@ func (i *baseIndexer) FindIndexes() []string { // reverse sort order should put our newest index first sort.Sort(sort.Reverse(sort.StringSlice(indexes))) - i.log().WithField("indexes", indexes).Debug("found physical indexes") + i.log().Debug("found physical indexes", "indexes", indexes) return indexes } @@ -153,7 +153,7 @@ func (i *baseIndexer) createNewIndex(def *IndexDefinition) (string, error) { } // all went well, return our physical index name - i.log().WithField("index", index).Info("created new index") + i.log().Info("created new index", "index", index) return index, nil } @@ -191,7 +191,7 @@ func (i *baseIndexer) updateAlias(newIndex string) error { remove.Remove.Index = idx commands = append(commands, remove) - logrus.WithField("indexer", i.name).WithField("index", idx).Debug("removing old alias") + slog.Debug("removing old alias", "indexer", i.name, "index", idx) } // add our new index @@ -204,7 +204,7 @@ func (i *baseIndexer) updateAlias(newIndex string) error { _, err := utils.MakeJSONRequest(http.MethodPost, fmt.Sprintf("%s/_aliases", i.elasticURL), aliasJSON, nil) - i.log().WithField("index", newIndex).Info("updated alias") + i.log().Info("updated alias", "index", newIndex) return err } @@ -236,7 +236,7 @@ func (i *baseIndexer) cleanupIndexes() error { // for each active index, if it starts with our alias but is before our current index, remove it for key := range healthResponse.Indices { if strings.HasPrefix(key, i.name) && strings.Compare(key, currents[0]) < 0 { - logrus.WithField("index", key).Info("removing old index") + slog.Info("removing old index", "index", key) _, err = utils.MakeJSONRequest(http.MethodDelete, fmt.Sprintf("%s/%s", i.elasticURL, key), nil, nil) if err != nil { return err @@ -275,27 +275,26 @@ func (i *baseIndexer) indexBatch(index string, batch []byte) (int, int, error) { createdCount, deletedCount, conflictedCount := 0, 0, 0 for _, item := range response.Items { if item.Index.ID != "" { - logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Trace("index response") + slog.Debug("index response", "id", item.Index.ID, "status", item.Index.Status) if item.Index.Status == 200 || item.Index.Status == 201 { createdCount++ } else if item.Index.Status == 409 { conflictedCount++ } else { - logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).WithField("result", item.Index.Result).Error("error indexing document") + slog.Error("error indexing document", "id", item.Index.ID, "status", item.Index.Status, "result", item.Index.Result) } } else if item.Delete.ID != "" { - logrus.WithField("id", item.Index.ID).WithField("status", item.Index.Status).Trace("delete response") + slog.Debug("delete response", "id", item.Index.ID, "status", item.Index.Status) if item.Delete.Status == 200 { deletedCount++ } else if item.Delete.Status == 409 { conflictedCount++ } } else { - logrus.Error("unparsed item in response") + slog.Error("unparsed item in response") } } - logrus.WithField("created", createdCount).WithField("deleted", deletedCount).WithField("conflicted", conflictedCount).Debug("indexed batch") - + slog.Debug("indexed batch", "created", createdCount, "deleted", deletedCount, "conflicted", conflictedCount) return createdCount, deletedCount, nil } diff --git a/indexers/base_test.go b/indexers/base_test.go index 9def23f..882171c 100644 --- a/indexers/base_test.go +++ b/indexers/base_test.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "log" + "log/slog" "os" "sort" "strconv" @@ -12,7 +13,6 @@ import ( "github.com/nyaruka/rp-indexer/v8/indexers" "github.com/olivere/elastic/v7" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -44,7 +44,7 @@ func setup(t *testing.T) (*sql.DB, *elastic.Client) { } } - logrus.SetLevel(logrus.DebugLevel) + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}))) return db, es } diff --git a/indexers/contacts.go b/indexers/contacts.go index 779fe00..0692844 100644 --- a/indexers/contacts.go +++ b/indexers/contacts.go @@ -2,13 +2,14 @@ package indexers import ( "bytes" + "context" "database/sql" _ "embed" "fmt" + "log/slog" "time" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) //go:embed contacts.index.json @@ -33,6 +34,7 @@ func NewContactIndexer(elasticURL, name string, shards, replicas, batchSize int) // Index indexes modified contacts and returns the name of the concrete index func (i *ContactIndexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error) { + ctx := context.TODO() var err error // find our physical index @@ -52,7 +54,7 @@ func (i *ContactIndexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error if err != nil { return "", errors.Wrap(err, "error creating new index") } - i.log().WithField("index", physicalIndex).Info("created new physical index") + i.log().Info("created new physical index", "index", physicalIndex) remapAlias = true } @@ -61,11 +63,11 @@ func (i *ContactIndexer) Index(db *sql.DB, rebuild, cleanup bool) (string, error return "", errors.Wrap(err, "error finding last modified") } - i.log().WithField("index", physicalIndex).WithField("last_modified", lastModified).Debug("indexing newer than last modified") + i.log().Debug("indexing newer than last modified", "index", physicalIndex, "last_modified", lastModified) // now index our docs start := time.Now() - indexed, deleted, err := i.indexModified(db, physicalIndex, lastModified.Add(-5*time.Second), rebuild) + indexed, deleted, err := i.indexModified(ctx, db, physicalIndex, lastModified.Add(-5*time.Second), rebuild) if err != nil { return "", errors.Wrap(err, "error indexing documents") } @@ -154,7 +156,7 @@ SELECT org_id, id, modified_on, is_active, row_to_json(t) FROM ( ` // IndexModified queries and indexes all contacts with a lastModified greater than or equal to the passed in time -func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified time.Time, rebuild bool) (int, int, error) { +func (i *ContactIndexer) indexModified(ctx context.Context, db *sql.DB, index string, lastModified time.Time, rebuild bool) (int, int, error) { totalFetched, totalCreated, totalDeleted := 0, 0, 0 var modifiedOn time.Time @@ -186,7 +188,7 @@ func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified ti return nil } - rows, err := db.Query(sqlSelectModifiedContacts, lastModified) + rows, err := db.QueryContext(ctx, sqlSelectModifiedContacts, lastModified) queryModified := lastModified @@ -209,14 +211,14 @@ func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified ti lastModified = modifiedOn if isActive { - logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).WithField("contact", contactJSON).Trace("modified contact") + slog.Debug("modified contact", "id", id, "modifiedOn", modifiedOn, "contact", contactJSON) subBatch.WriteString(fmt.Sprintf(indexCommand, id, modifiedOn.UnixNano(), orgID)) subBatch.WriteString("\n") subBatch.WriteString(contactJSON) subBatch.WriteString("\n") } else { - logrus.WithField("id", id).WithField("modifiedOn", modifiedOn).Trace("deleted contact") + slog.Debug("deleted contact", "id", id, "modifiedOn", modifiedOn) subBatch.WriteString(fmt.Sprintf(deleteCommand, id, modifiedOn.UnixNano(), orgID)) subBatch.WriteString("\n") @@ -246,16 +248,16 @@ func (i *ContactIndexer) indexModified(db *sql.DB, index string, lastModified ti batchTime := time.Since(batchStart) batchRate := int(float32(batchFetched) / (float32(batchTime) / float32(time.Second))) - log := i.log().WithField("index", index).WithFields(logrus.Fields{ - "rate": batchRate, - "batch_fetched": batchFetched, - "batch_created": batchCreated, - "batch_elapsed": batchTime, - "batch_elapsed_es": batchESTime, - "total_fetched": totalFetched, - "total_created": totalCreated, - "total_elapsed": totalTime, - }) + log := i.log().With("index", index, + "rate", batchRate, + "batch_fetched", batchFetched, + "batch_created", batchCreated, + "batch_elapsed", batchTime, + "batch_elapsed_es", batchESTime, + "total_fetched", totalFetched, + "total_created", totalCreated, + "total_elapsed", totalTime, + ) // if we're rebuilding, always log batch progress if rebuild { diff --git a/testdb.sql b/testdb.sql index c34ac70..0996fdf 100644 --- a/testdb.sql +++ b/testdb.sql @@ -33,9 +33,9 @@ CREATE TABLE contacts_contacturn ( priority integer NOT NULL, path character varying(255) NOT NULL, channel_id integer, - auth text, display character varying(255), - identity character varying(255) NOT NULL + identity character varying(255) NOT NULL, + auth_tokens jsonb ); DROP TABLE IF EXISTS contacts_contactgroup CASCADE; diff --git a/utils/http.go b/utils/http.go index 6871ade..f9ef2d0 100644 --- a/utils/http.go +++ b/utils/http.go @@ -4,13 +4,13 @@ import ( "bytes" "encoding/json" "fmt" - "io/ioutil" + "io" + "log/slog" "net/http" "time" "github.com/getsentry/sentry-go" "github.com/nyaruka/gocommon/httpx" - log "github.com/sirupsen/logrus" ) var retryConfig *httpx.RetryConfig @@ -44,62 +44,61 @@ func shouldRetry(request *http.Request, response *http.Response, withDelay time. } // check for unexpected EOF - bodyBytes, err := ioutil.ReadAll(response.Body) + bodyBytes, err := io.ReadAll(response.Body) response.Body.Close() if err != nil { - log.WithError(err).Error("error reading ES response, retrying") + slog.Error("error reading ES response, retrying", "error", err) return true } - response.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes)) + response.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) return false } // MakeJSONRequest is a utility function to make a JSON request, optionally decoding the response into the passed in struct func MakeJSONRequest(method string, url string, body []byte, dest any) (*http.Response, error) { - l := log.WithField("url", url).WithField("method", method) + l := slog.With("url", url, "method", method) req, _ := httpx.NewRequest(method, url, bytes.NewReader(body), map[string]string{"Content-Type": "application/json"}) resp, err := httpx.Do(http.DefaultClient, req, retryConfig, nil) - + if err != nil { + l.Error("error making request", "error", err) + return resp, err + } originalSize := len(body) - l.WithField("request header", req.Header).WithField("response header", resp.Header) + l = l.With("request header", req.Header, "response header", resp.Header) var formattedJSON bytes.Buffer if err := json.Indent(&formattedJSON, body, "", " "); err != nil { formattedJSON.Write(body) } formattedSize := formattedJSON.Len() - l.WithField("request", formattedJSON.String()) + l = l.With("request", formattedJSON.String()) - if err != nil { - l.WithError(err).Error("error making request") - return resp, err - } defer resp.Body.Close() // if we have a body, try to decode it - respBody, err := ioutil.ReadAll(resp.Body) + respBody, err := io.ReadAll(resp.Body) if err != nil { - l.WithError(err).Error("error reading response") + l.Error("error reading response", "error", err) return resp, err } - l = l.WithField("response", string(respBody)).WithField("status", resp.StatusCode) + l = l.With("response", string(respBody), "status", resp.StatusCode) // error if we got a non-200 if resp.StatusCode != http.StatusOK { - l.WithError(err).Error("error reaching ES") + l.Error("error reaching ES", "error", err) time.Sleep(time.Second * 5) sendFileToSentry(formattedJSON, originalSize, formattedSize, fmt.Errorf("received non 200 response %d: %s", resp.StatusCode, respBody), resp) - return resp, fmt.Errorf("received non 200 response %d: %s", resp.StatusCode, respBody) + return resp, fmt.Errorf("received non-200 response %d: %s", resp.StatusCode, respBody) } if dest != nil { err = json.Unmarshal(respBody, dest) if err != nil { - l.WithError(err).Error("error unmarshalling response") + l.Error("error unmarshalling response", "error", err) return resp, err } } diff --git a/utils/logrus.go b/utils/logrus.go new file mode 100644 index 0000000..2650bf9 --- /dev/null +++ b/utils/logrus.go @@ -0,0 +1,92 @@ +// Structured logging handler for logrus so we can rewrite code to use slog package incrementally. Once all logging is +// happening via slog, we just need to hook up Sentry directly to that, and then we can get rid of this file. +package utils + +import ( + "context" + "log/slog" + "slices" + "strings" + + "github.com/sirupsen/logrus" +) + +var levels = map[slog.Level]logrus.Level{ + slog.LevelError: logrus.ErrorLevel, + slog.LevelWarn: logrus.WarnLevel, + slog.LevelInfo: logrus.InfoLevel, + slog.LevelDebug: logrus.DebugLevel, +} + +type LogrusHandler struct { + logger *logrus.Logger + groups []string + attrs []slog.Attr +} + +func NewLogrusHandler(logger *logrus.Logger) *LogrusHandler { + return &LogrusHandler{logger: logger} +} + +func (l *LogrusHandler) clone() *LogrusHandler { + return &LogrusHandler{ + logger: l.logger, + groups: slices.Clip(l.groups), + attrs: slices.Clip(l.attrs), + } +} + +func (l *LogrusHandler) Enabled(ctx context.Context, level slog.Level) bool { + return levels[level] <= l.logger.GetLevel() +} + +func (l *LogrusHandler) Handle(ctx context.Context, r slog.Record) error { + log := logrus.NewEntry(l.logger) + if r.Time.IsZero() { + log = log.WithTime(r.Time) + } + + f := logrus.Fields{} + for _, a := range l.attrs { + if a.Key != "" { + f[a.Key] = a.Value + } + } + log = log.WithFields(f) + + r.Attrs(func(attr slog.Attr) bool { + if attr.Key == "" { + return true + } + log = log.WithField(attr.Key, attr.Value) + return true + }) + log.Logf(levels[r.Level], r.Message) + return nil +} + +func (l *LogrusHandler) groupPrefix() string { + if len(l.groups) > 0 { + return strings.Join(l.groups, ":") + ":" + } + return "" +} + +func (l *LogrusHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + newHandler := l.clone() + for _, a := range attrs { + newHandler.attrs = append(newHandler.attrs, slog.Attr{ + Key: l.groupPrefix() + a.Key, + Value: a.Value, + }) + } + return newHandler +} + +func (l *LogrusHandler) WithGroup(name string) slog.Handler { + newHandler := l.clone() + newHandler.groups = append(newHandler.groups, name) + return newHandler +} + +var _ slog.Handler = &LogrusHandler{}