Merge tag 'v9.2.0' into update/v9.2.0

 * Test against postgresql 15
 * Update dependencies
 * Test against Elasticsearch 8

.github/workflows/ci.yml

@@ -1,15 +1,15 @@
 name: CI
 on: [push, pull_request]
 env:
-  go-version: "1.21.x"
+  go-version: "1.22.x"
 jobs:
   test:
     name: Test
     runs-on: ubuntu-latest
 
     services:
       postgres:
-        image: postgres:14-alpine
+        image: postgres:15-alpine
         env:
           POSTGRES_DB: indexer_test
           POSTGRES_USER: indexer_test
@@ -18,19 +18,20 @@ jobs:
           - 5432:5432
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
       elastic:
-          image: elasticsearch:7.17.9
-          ports:
-            - 9200:9200
-            - 9300:9300
-          env:
-            discovery.type: single-node
+        image: elasticsearch:8.13.4
+        env:
+          discovery.type: single-node
+          xpack.security.enabled: false
+        ports:
+          - 9200:9200
+        options: --health-cmd "curl http://localhost:9200/_cluster/health" --health-interval 10s --health-timeout 5s --health-retries 5
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.go-version }}
 
@@ -39,7 +40,7 @@ jobs:
 
       - name: Upload coverage
         if: success()
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@v4
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           fail_ci_if_error: true
@@ -51,17 +52,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Install Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ env.go-version }}
 
       - name: Publish release
-        uses: goreleaser/goreleaser-action@v4
+        uses: goreleaser/goreleaser-action@v6
         with:
           args: release --clean
         env:

.gitignore

@@ -5,7 +5,7 @@
 *.dylib
 fabfile.py
 fabfile.pyc
-deploy/
+deploy
 rp-indexer
 
 # Test binary, build with `go test -c`
@@ -19,4 +19,5 @@ rp-indexer
 
 *~
 
-.DS_Store
+.DS_Store
+dist/

.goreleaser.yaml

@@ -0,0 +1,20 @@
+version: 2
+builds:
+  - main: ./cmd/rp-indexer/main.go
+    binary: rp-indexer
+    goos:
+      - darwin
+      - linux
+    goarch:
+      - amd64
+      - arm64
+
+changelog:
+  filters:
+    exclude:
+      - "^Update CHANGELOG.md"
+
+archives:
+  - files:
+      - LICENSE
+      - README.md

CHANGELOG.md

@@ -1,3 +1,58 @@
+v9.2.0 (2024-07-17)
+-------------------------
+ * Test against postgresql 15
+ * Update dependencies
+ * Test against Elasticsearch 8
+
+v9.1.9 (2024-06-10)
+-------------------------
+ * Record stats inside indexing batch loop
+ * Split up created vs updated in progress logging
+ * Add track_total_hits to GetESLastModified
+
+v9.1.8 (2024-06-05)
+-------------------------
+ * Update github actions versions
+ * Add healthcheck for elastic service in CI tests
+ * Update goreleaser config to v2
+
+v9.1.7 (2024-06-05)
+-------------------------
+ * Remove multi-search-db CI testing because it's unreliable
+
+v9.1.6 (2024-06-04)
+-------------------------
+ * Remove opensearch from CI
+
+v9.1.5 (2024-06-04)
+-------------------------
+ * Update slog-multi
+
+v9.1.4 (2024-06-03)
+-------------------------
+ * Update deps
+ * Use std lib errors
+
+v9.1.3 (2024-05-16)
+-------------------------
+ * Reduce batch size to 100,000
+ * Test with OpenSearch 2.13 as well
+
+v9.1.2 (2024-05-14)
+-------------------------
+ * Remove deprecated _doc from mapping to support ES8 / OS2
+ * Test against Elastic 7.x and 8.x and OpenSearch 1.x
+ * Rewrite tests to make raw HTTP calls to elastic and not use the old library
+
+v9.1.1 (2024-04-25)
+-------------------------
+ * Add reporting of lag as a metric
+
+v9.1.0 (2024-04-15)
+-------------------------
+ * Update deps and bump main package to v9
+ * Finish switching to use slog for logging
+
 v9.0.0 (2024-01-05)
 -------------------------
  * Update dependencies

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.21
+FROM golang:1.22
 
 WORKDIR /usr/src/app
 
@@ -7,6 +7,6 @@ COPY go.mod go.sum ./
 RUN go mod download && go mod verify
 
 COPY . .
-RUN go build -v -o /usr/local/bin/app github.com/nyaruka/rp-indexer/v8/cmd/rp-indexer
+RUN go build -v -o /usr/local/bin/app github.com/nyaruka/rp-indexer/v9/cmd/rp-indexer
 
 CMD ["app"]

README.md

@@ -1,4 +1,4 @@
-# Indexer
+# 🗃️ Indexer
 
 [![Build Status](https://github.com/nyaruka/rp-indexer/workflows/CI/badge.svg)](https://github.com/nyaruka/rp-indexer/actions?query=workflow%3ACI) 
 [![codecov](https://codecov.io/gh/nyaruka/rp-indexer/branch/main/graph/badge.svg)](https://codecov.io/gh/nyaruka/rp-indexer) 
@@ -46,40 +46,7 @@ For use with RapidPro, you will want to configure these settings:
 
 Recommended settings for error reporting:
 
- * `INDEXER_SENTRY_DSN`: The DSN to use when logging errors to Sentry
-
-### Reference
-
-These are the configuration options that can be provided as parameters or environment variables. If using environment 
-varibles, convert to uppercase, replace dashes with underscores and prefix the name with `INDEXER_`, e.g. `-log-level` 
-becomes `INDEXER_LOG_LEVEL`.
-
-```
-  -cleanup
-      whether to remove old indexes after a rebuild
-  -db string
-      the connection string for our database (default "postgres://localhost/rapidpro?sslmode=disable")
-  -debug-conf
-      print where config values are coming from
-  -elastic-url string
-      the url for our elastic search instance (default "http://localhost:9200")
-  -help
-      print usage information
-  -index string
-      the alias for our contact index (default "contacts")
-  -librato-username
-      the Librato username for metrics reporting
-  -librato-token
-      the Librato token for metrics reporting
-  -log-level string
-      the log level, one of error, warn, info, debug (default "info")
-  -poll int
-      the number of seconds to wait between checking for updated contacts (default 5)
-  -rebuild
-      whether to rebuild the index, swapping it when complete, then exiting (default false)
-  -sentry-dsn string
-      the sentry configuration to log errors to, if any
-```
+ * `INDEXER_SENTRY_DSN`: DSN to use when logging errors to Sentry
 
 ## Development
 

cmd/rp-indexer/main.go

@@ -3,21 +3,19 @@ package main
 import (
 	"database/sql"
 	"log"
-
 	"log/slog"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
 
-	"github.com/evalphobia/logrus_sentry"
 	"github.com/getsentry/sentry-go"
 	_ "github.com/lib/pq"
 	"github.com/nyaruka/ezconf"
-	indexer "github.com/nyaruka/rp-indexer/v8"
-	"github.com/nyaruka/rp-indexer/v8/indexers"
-	"github.com/nyaruka/rp-indexer/v8/utils"
-	"github.com/sirupsen/logrus"
+	indexer "github.com/nyaruka/rp-indexer/v9"
+	"github.com/nyaruka/rp-indexer/v9/indexers"
+	slogmulti "github.com/samber/slog-multi"
+	slogsentry "github.com/samber/slog-sentry"
 )
 
 var (
@@ -31,41 +29,46 @@ func main() {
 	loader := ezconf.NewLoader(cfg, "indexer", "Indexes RapidPro contacts to ElasticSearch", []string{"indexer.toml"})
 	loader.MustLoad()
 
-	level, err := logrus.ParseLevel(cfg.LogLevel)
+	var level slog.Level
+	err := level.UnmarshalText([]byte(cfg.LogLevel))
 	if err != nil {
-		logrus.Fatalf("Invalid log level '%s'", level)
+		log.Fatalf("invalid log level %s", level)
+		os.Exit(1)
 	}
 
-	logrus.SetLevel(level)
-	logrus.SetOutput(os.Stdout)
-	logrus.SetFormatter(&logrus.TextFormatter{})
-	logrus.WithField("version", version).WithField("released", date).Info("starting indexer")
-
-	// configure golang std structured logging to route to logrus
-	slog.SetDefault(slog.New(utils.NewLogrusHandler(logrus.StandardLogger())))
+	// configure our logger
+	logHandler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: level})
+	slog.SetDefault(slog.New(logHandler))
 
 	logger := slog.With("comp", "main")
 	logger.Info("starting indexer", "version", version, "released", date)
 
 	// if we have a DSN entry, try to initialize it
 	if cfg.SentryDSN != "" {
-		hook, err := logrus_sentry.NewSentryHook(cfg.SentryDSN, []logrus.Level{logrus.PanicLevel, logrus.FatalLevel, logrus.ErrorLevel})
-		hook.Timeout = 0
-		hook.StacktraceConfiguration.Enable = true
-		hook.StacktraceConfiguration.Skip = 4
-		hook.StacktraceConfiguration.Context = 5
+		err := sentry.Init(sentry.ClientOptions{
+			Dsn:           cfg.SentryDSN,
+			EnableTracing: false,
+		})
 		if err != nil {
-			logger.Error("invalid sentry DSN: '%s': %s", cfg.SentryDSN, err)
+			log.Fatalf("error initiating sentry client, error %s, dsn %s", err, cfg.SentryDSN)
+			os.Exit(1)
 		}
-		logrus.StandardLogger().Hooks.Add(hook)
-
 		err = sentry.Init(sentry.ClientOptions{
 			Dsn: cfg.SentryDSN,
 		})
 		if err != nil {
 			log.Fatalf("Sentry init failed: %v", err)
 		}
+
 		defer sentry.Flush(2 * time.Second)
+		logger = slog.New(
+			slogmulti.Fanout(
+				logHandler,
+				slogsentry.Option{Level: slog.LevelError}.NewSentryHandler(),
+			),
+		)
+		logger = logger.With("release", version)
+		slog.SetDefault(logger)
 	}
 
 	db, err := sql.Open("postgres", cfg.DB)