added client ip hashing

README.md

@@ -11,7 +11,8 @@ Features
 * Health checks for each ES node is enabled by default.
 * Uses bulk indexing for better performance with large amount of requests.
 * GeoIP lookup of each request, adds long/lat, city, and country for each client.
-* more to come eventually....
+* Allows to obfuscate client ip if needed.
+* ...
 
 ## Getting started
 

f5elastic.go

@@ -1,6 +1,8 @@
 package main
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"errors"
 	"flag"
 	"io/ioutil"
@@ -49,6 +51,7 @@ type Config struct {
 	Buffer  int
 	Timeout int
 	Geoip   string
+	Salt    string
 }
 
 type Worker struct {
@@ -111,6 +114,18 @@ func (w Worker) NewRequest(msg string) (Request, error) {
 		request.Location = strconv.FormatFloat(record.Location.Latitude, 'f', 6, 64) + "," + strconv.FormatFloat(record.Location.Longitude, 'f', 6, 64)
 	}
 	request.Timestamp = time.Now().UTC().Format("2006-01-02T15:04:05Z")
+	if config.Salt != "" {
+		// hash client ip with secret salt.
+		if val, ok := hashcache.Get(request.Client); ok {
+			request.Client = val.(string)
+		} else {
+			h := sha256.New()
+			h.Write([]byte(request.Client + config.Salt))
+			hexhash := hex.EncodeToString(h.Sum(nil))[0:16]
+			hashcache.Add(request.Client, hexhash)
+			request.Client = hexhash
+		}
+	}
 	return request, nil
 }
 
@@ -149,6 +164,7 @@ func (w Worker) Start() {
 
 var wg sync.WaitGroup
 var geocache *lru.Cache
+var hashcache *lru.Cache
 var geodb *geoip2.Reader
 var config Config
 
@@ -183,7 +199,9 @@ func main() {
 	}
 	defer geodb.Close()
 	// init our lru cache of geodb
-	geocache, _ = lru.New(10000)
+	geocache, _ = lru.New(100000)
+	// init our lru cache of sha256 hashes
+	hashcache, _ = lru.New(100000)
 	// init our elastic client
 	c, err := elastic.NewClient(
 		elastic.SetURL(config.Nodes...),

f5elastic.toml-example

@@ -16,3 +16,5 @@ buffer = 50000
 timeout = 5
 # Geoip2/GeoLite2 database
 geoip = "GeoLite2-City.mmdb"
+# generate sha256 hash to hide client ip (disabled if empty)
+salt = ""