Merge pull request #55 from Security-Onion-Solutions/kilo

Favor non-aggregatable type when a field has multiple data types

server/modules/elastic/elasticeventstore.go

@@ -396,7 +396,15 @@ func (store *ElasticEventstore) cacheFields(name gjson.Result, details gjson.Res
 			aggregatable: field["aggregatable"].(bool),
 			searchable:   field["searchable"].(bool),
 		}
-		store.fieldDefs[fieldName] = fieldDef
+
+		// If there are multiple types for this field prefer the non-aggregatable since
+		// we cannot reliably aggregate across all indices. In most, or maybe all cases,
+		// there will be a .keyword subfield across both indices which will be used
+		// for aggregation purposes until all ingested data is fully ECS data type
+		// compliant.
+		if store.fieldDefs[fieldName] == nil || !fieldDef.aggregatable {
+			store.fieldDefs[fieldName] = fieldDef
+		}
 
 		log.WithFields(log.Fields{
 			"name":         name,

server/modules/elastic/elasticeventstore_test.go

@@ -51,6 +51,34 @@ func TestFieldMapping(tester *testing.T) {
 	assert.Equal(tester, "event.acknowledged", actual)
 }
 
+func TestFieldMappingCollisions(tester *testing.T) {
+	store := &ElasticEventstore{}
+
+	json, err := ioutil.ReadFile("fieldcaps_response.json")
+	assert.Nil(tester, err)
+	store.cacheFieldsFromJson(string(json))
+
+	var testTable = []struct {
+		given    string
+		expected string
+	}{
+		{"event.module", "event.module.keyword"},
+		{"event.category", "event.category.keyword"},
+		{"event.dataset", "event.dataset.keyword"},
+		{"event.kind", "event.kind.keyword"},
+		{"event.outcome", "event.outcome.keyword"},
+		{"event.type", "event.type.keyword"},
+		{"event.timezone", "event.timezone.keyword"},
+	}
+
+	for _, test := range testTable {
+		tester.Run("given="+test.given, func(t *testing.T) {
+			actual := store.mapElasticField(test.given)
+			assert.Equal(tester, test.expected, actual)
+		})
+	}
+}
+
 func TestFieldMappingCache(tester *testing.T) {
 	store := &ElasticEventstore{}