Merge pull request #2 from nyaruka/master

Up merge with base repository

.travis.yml

@@ -1,14 +1,25 @@
 language: go
 
 addons:
-  postgresql: '9.6'
+  postgresql: '10'
+  apt:
+    packages:
+      - postgresql-10
+      - postgresql-client-10
 
 go:
-- "1.10"
+- "1.11"
+
+env:
+  global:
+    - GO111MODULE=on
 
 before_install:
   - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.3.deb && sudo dpkg -i --force-confnew elasticsearch-6.2.3.deb
   - sudo service elasticsearch start
+  - sudo sed -i -e '/local.*peer/s/postgres/all/' -e 's/peer\|md5/trust/g' /etc/postgresql/*/main/pg_hba.conf
+  - sudo sed -i 's/port = 5433/port = 5432/' /etc/postgresql/10/main/postgresql.conf
+  - sudo service postgresql restart 10
   - sleep 10
 
 before_script:

CHANGELOG.md

@@ -1,3 +1,19 @@
+v2.0.0
+----------
+ * Ignore value of is_test on contacts
+
+v1.0.27
+----------
+ * update ES shards to match current ES best-practice guidance
+
+v1.0.26
+----------
+ * move to go module, dont ignore any keywords
+
+v1.0.25
+----------
+ * Changes to support both PG 10 and 9.6
+
 v1.0.24
 ----------
  * increase batch size to 500k
@@ -28,7 +44,6 @@ v1.0.18
 
 v1.0.17
 ----------
-
  * change to number instead of decimal field
  * add example not exists query
 
@@ -51,7 +66,7 @@ v1.0.13
 v1.0.12
 ----------
  * add modified_on_mu for sorting / index creation
- * add prefix name for index building 
+ * add prefix name for index building
 
 v1.0.11
 ----------
@@ -99,4 +114,3 @@ v1.0.2
 v1.0.1
 ----------
  * Add changelog, move to fancy revving
-

README.md

@@ -2,7 +2,14 @@
 
 Simple service for indexing RapidPro contacts into ElasticSearch.
 
-This service can run in two modes:
+# Deploying
+
+As Indexer is a go application, it compiles to a binary and that binary along with the config file is all
+you need to run it on your server. You can find bundles for each platform in the
+[releases directory](https://github.com/nyaruka/rp-indexer/releases). You should only run a single indexer
+instance for a deployment.
+
+Indexer can run in two modes:
 
 1) the default mode, which simply queries the ElasticSearch database, finds the most recently
 modified contact, then on a schedule queries the `contacts_contact` table on the RapidPro
@@ -14,36 +21,90 @@ all contacts on RapidPro. Once complete, this switches out the alias for the con
 with the newly build index. This can be run on a cron (in parallel with the mode above) to rebuild
 your index occasionally to get rid of bloat.
 
-## Usage
+# Configuration
+
+Indexer uses a tiered configuration system, each option takes precendence over the ones above it:
+ 1. The configuration file
+ 2. Environment variables starting with `INDEXER_` 
+ 3. Command line parameters
+
+We recommend running Indexer with no changes to the configuration and no parameters, using only
+environment variables to configure it. You can use `% rp-indexer --help` to see a list of the
+environment variables and parameters and for more details on each option.
 
-It is recommended to run the service with two environment variables set:
+## RapidPro Configuration
+
+For use with RapidPro, you will want to configure these settings:
 
  * `INDEXER_DB`: a URL connection string for your RapidPro database
  * `INDEXER_ELASTIC_URL`: the URL for your ElasticSearch endpoint
+
+Recommended settings for error reporting:
+
+ * `INDEXER_SENTRY_DSN`: The DSN to use when logging errors to Sentry
+
+# Development
+
+Install Indexer source in your workspace with:
+
+```
+go get github.com/nyaruka/rp-indexer
+```
+
+Build Indexer with:
+
+```
+go build github.com/nyaruka/rp-indexer/cmd/rp-indexer
+```
+
+This will create a new executable in your current directory `rp-indexer`
+
+To run the tests you need to create the test database:
+
+```
+$ createdb elastic_test
+```
+
+To run all of the tests:
+
+```
+go test github.com/nyaruka/rp-indexer/... -p=1
+```
+
+# Usage
 
 ```
 Indexes RapidPro contacts to ElasticSearch
 
 Usage of indexer:
+  -cleanup
+    	whether to remove old indexes after a rebuild
   -db string
-        the connection string for our database (default "postgres://localhost/rapidpro")
+    	the connection string for our database (default "postgres://localhost/rapidpro?sslmode=disable")
   -debug-conf
-        print where config values are coming from
+    	print where config values are coming from
   -elastic-url string
-        the url for our elastic search instance (default "http://localhost:9200")
+    	the url for our elastic search instance (default "http://localhost:9200")
   -help
-        print usage information
+    	print usage information
   -index string
-        the alias for our contact index (default "contacts")
+    	the alias for our contact index (default "contacts")
+  -log-level string
+    	the log level, one of error, warn, info, debug (default "info")
   -poll int
-        the number of seconds to wait between checking for updated contacts (default 5)
+    	the number of seconds to wait between checking for updated contacts (default 5)
   -rebuild
-        whether to rebuild the index, swapping it when complete, then exiting (default false)
+    	whether to rebuild the index, swapping it when complete, then exiting (default false)
+  -sentry-dsn string
+    	the sentry configuration to log errors to, if any
 
 Environment variables:
+                             INDEXER_CLEANUP - bool
                                   INDEXER_DB - string
                          INDEXER_ELASTIC_URL - string
                                INDEXER_INDEX - string
+                           INDEXER_LOG_LEVEL - string
                                 INDEXER_POLL - int
                              INDEXER_REBUILD - bool
-```
+                          INDEXER_SENTRY_DSN - string
+                          ```

go.mod

@@ -0,0 +1,24 @@
+module github.com/nyaruka/rp-indexer
+
+require (
+	github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261
+	github.com/davecgh/go-spew v1.1.0
+	github.com/evalphobia/logrus_sentry v0.4.5
+	github.com/fatih/structs v1.0.0
+	github.com/fortytw2/leaktest v1.3.0 // indirect
+	github.com/getsentry/raven-go v0.0.0-20180405121644-d1470f50d3a3
+	github.com/lib/pq v0.0.0-20180327071824-d34b9ff171c2
+	github.com/mailru/easyjson v0.0.0-20180323154445-8b799c424f57
+	github.com/naoina/go-stringutil v0.1.0
+	github.com/naoina/toml v0.1.1
+	github.com/nyaruka/ezconf v0.2.1
+	github.com/olivere/elastic v6.1.14+incompatible
+	github.com/pkg/errors v0.8.0
+	github.com/pmezard/go-difflib v1.0.0
+	github.com/sirupsen/logrus v1.0.5
+	github.com/stretchr/testify v1.2.1
+	golang.org/x/crypto v0.0.0-20180322175230-88942b9c40a4
+	golang.org/x/sys v0.0.0-20180326154331-13d03a9a82fb
+	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
+	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
+)