Remove use of .kibana index in favor of Elastic fieldcaps API to dete…

…rmine indexed field capabilities. Resolves Security-Onion-Solutions/securityonion#3502.
Security-Onion-Solutions · Mar 17, 2021 · 43378a8 · 43378a8
1 parent 639667e
commit 43378a8
Show file tree

Hide file tree

Showing 4 changed files with 8,900 additions and 89 deletions.
diff --git a/server/modules/elastic/elasticeventstore.go b/server/modules/elastic/elasticeventstore.go
@@ -330,47 +330,59 @@ func (store *ElasticEventstore) refreshCache() {
   store.cacheLock.Lock()
   defer store.cacheLock.Unlock()
   if store.cacheTime.IsZero() || time.Now().Sub(store.cacheTime) > store.cacheMs {
-    err := store.refreshCacheFromIndexPatterns()
+    err := store.refreshCacheFromFieldCaps()
     if err == nil {
       store.cacheTime = time.Now()
     }
   }
 }
 
-func (store *ElasticEventstore) refreshCacheFromIndexPatterns() error {
-  query := fmt.Sprintf(`{"query" : { "bool": { "must": [ { "match": { "index-pattern.title" : "` + store.index + `" }}, { "match" : { "type" : "index-pattern" }} ] }}}`)
-  json, err := store.indexSearch(query, []string{".kibana*"})
-  if err != nil {
-    log.WithError(err).Error("Failed to refresh cache from index patterns")
-  } else {
+func (store *ElasticEventstore) refreshCacheFromFieldCaps() error {
+  log.Info("Fetching Field Capabilities from Elasticsearch")
+  indexes := strings.Split(store.index, ",")
+  var json string
+  res, err := store.esClient.FieldCaps(
+    store.esClient.FieldCaps.WithContext(context.Background()),
+    store.esClient.FieldCaps.WithIndex(indexes...),
+    store.esClient.FieldCaps.WithFields("*"),
+    store.esClient.FieldCaps.WithPretty(),
+  )
+  if err == nil {
+    defer res.Body.Close()
+    json, err = store.readJsonFromResponse(res)
+    log.WithFields(log.Fields{"response": json}).Debug("Fetch finished")
     store.cacheFieldsFromJson(json)
+  } else {
+    log.WithError(err).Error("Failed to refresh cache from index patterns")
   }
   return err
 }
 
 func (store *ElasticEventstore) cacheFieldsFromJson(json string) {
   store.fieldDefs = make(map[string]*FieldDefinition)
-  gjson.Get(json, "hits.hits.#._source.index-pattern.fields").ForEach(store.cacheFields)
+  gjson.Get(json, "fields").ForEach(store.cacheFields)
 }
 
-func (store *ElasticEventstore) cacheFields(name gjson.Result, fields gjson.Result) bool {
-  fieldList := make([]map[string]interface{}, 0, 0)
-  json.NewDecoder(strings.NewReader(fields.String())).Decode(&fieldList)
-  for _, field := range fieldList {
-    name := field["name"].(string)
+func (store *ElasticEventstore) cacheFields(name gjson.Result, details gjson.Result) bool {
+  fieldName := name.String()
+  detailsMap := make(map[string]map[string]interface{})
+  json.NewDecoder(strings.NewReader(details.String())).Decode(&detailsMap)
+  for _, field := range detailsMap {
     fieldType := field["type"].(string)
 
     fieldDef := &FieldDefinition {
-      name: name, 
+      name: fieldName, 
       fieldType: fieldType, 
       aggregatable: field["aggregatable"].(bool), 
       searchable: field["searchable"].(bool),
     }
-    store.fieldDefs[name] = fieldDef
+    store.fieldDefs[fieldName] = fieldDef
 
     log.WithFields(log.Fields {
       "name": name,
       "type": fieldType,
+      "aggregatable": fieldDef.aggregatable,
+      "searchable": fieldDef.searchable,
     }).Debug("Added field definition")
   }
   return true

diff --git a/server/modules/elastic/elasticeventstore_test.go b/server/modules/elastic/elasticeventstore_test.go
@@ -20,76 +20,90 @@ import (
 func TestFieldMapping(tester *testing.T) {
   store := &ElasticEventstore{}
 
-  json, err := ioutil.ReadFile("indexpattern_response.json")
+  json, err := ioutil.ReadFile("fieldcaps_response.json")
   if err != nil {
     tester.Errorf("Unexpected error while loading test resource: %v", err)
   }
   store.cacheFieldsFromJson(string(json))
 
-  // Exists as keyword
-  actual := store.mapElasticField("ack")
-  if actual != "ack.keyword" {
-    tester.Errorf("expected mapped field %s but got %s", "ack.keyword", actual)
+  // Exists as keyword and not already aggregatable
+  actual := store.mapElasticField("smb.service")
+  if actual != "smb.service.keyword" {
+    tester.Errorf("expected mapped field %s but got %s", "smb.service.keyword", actual)
+  }
+
+  // Exists as keyword but already aggregatable
+  actual = store.mapElasticField("agent.ip")
+  if actual != "agent.ip" {
+    tester.Errorf("expected mapped field %s but got %s", "agent.ip", actual)
   }
 
   // Does not exist as valid keyword
-  actual = store.mapElasticField("foo")
-  if actual != "foo" {
-    tester.Errorf("expected unmapped field %s but got %s", "foo", actual)
+  actual = store.mapElasticField("event.acknowledged")
+  if actual != "event.acknowledged" {
+    tester.Errorf("expected unmapped field %s but got %s", "event.acknowledged", actual)
+  }
+
+  // Both non-keyword and keyword variants are aggregatable
+  actual = store.unmapElasticField("agent.ip.keyword")
+  if actual != "agent.ip.keyword" {
+    tester.Errorf("expected unmapped field %s but got %s", "agent.ip.keyword", actual)
   }
 
-  actual = store.unmapElasticField("ack.keyword")
-  if actual != "ack" {
-    tester.Errorf("expected unmapped field %s but got %s", "ack", actual)
+  // Only keyword variant is aggregatable
+  actual = store.unmapElasticField("smb.service.keyword")
+  if actual != "smb.service" {
+    tester.Errorf("expected unmapped field %s but got %s", "smb.service", actual)
   }
 
-  actual = store.unmapElasticField("foo.keyword")
-  if actual != "foo.keyword" {
-    tester.Errorf("expected unmapped field %s but got %s", "foo.keyword", actual)
+  // Neither are aggregatable
+  actual = store.unmapElasticField("event.acknowledged")
+  if actual != "event.acknowledged" {
+    tester.Errorf("expected unmapped field %s but got %s", "event.acknowledged", actual)
   }
 }
 
 func TestFieldMappingCache(tester *testing.T) {
   store := &ElasticEventstore{}
 
-  json, err := ioutil.ReadFile("indexpattern_response.json")
+  json, err := ioutil.ReadFile("fieldcaps_response.json")
   if err != nil {
     tester.Errorf("Unexpected error while loading test resource: %v", err)
   }
   store.cacheFieldsFromJson(string(json))
 
-  ack := store.fieldDefs["ack"]
-  if ack == nil {
+  field := store.fieldDefs["smb.service"]
+  if field == nil {
     tester.Errorf("expected field definition")
   }
-  if ack.name != "ack" {
-    tester.Errorf("expected name %s but got %s", "ack", ack.name)
+  if field.name != "smb.service" {
+    tester.Errorf("expected name %s but got %s", "ack", field.name)
   }
-  if ack.fieldType != "string" {
-    tester.Errorf("expected fieldType %s but got %s", "string", ack.fieldType)
+  if field.fieldType != "text" {
+    tester.Errorf("expected fieldType %s but got %s", "text", field.fieldType)
   }
-  if ack.aggregatable != false {
-    tester.Errorf("expected aggregatable %t but got %t", false, ack.aggregatable)
+  if field.aggregatable != false {
+    tester.Errorf("expected aggregatable %t but got %t", false, field.aggregatable)
   }
-  if ack.searchable != true {
-    tester.Errorf("expected searchable %t but got %t", true, ack.searchable)
+  if field.searchable != true {
+    tester.Errorf("expected searchable %t but got %t", true, field.searchable)
   }
 
-  ackKeyword := store.fieldDefs["ack.keyword"]
-  if ackKeyword == nil {
+  fieldKeyword := store.fieldDefs["smb.service.keyword"]
+  if fieldKeyword == nil {
     tester.Errorf("expected field definition")
   }
-  if ackKeyword.name != "ack.keyword" {
-    tester.Errorf("expected name %s but got %s", "ackKeyword", ackKeyword.name)
+  if fieldKeyword.name != "smb.service.keyword" {
+    tester.Errorf("expected name %s but got %s", "smb.service.keyword", fieldKeyword.name)
   }
-  if ackKeyword.fieldType != "string" {
-    tester.Errorf("expected fieldType %s but got %s", "string", ackKeyword.fieldType)
+  if fieldKeyword.fieldType != "keyword" {
+    tester.Errorf("expected fieldType %s but got %s", "keyword", fieldKeyword.fieldType)
   }
-  if ackKeyword.aggregatable != true {
-    tester.Errorf("expected aggregatable %t but got %t", true, ackKeyword.aggregatable)
+  if fieldKeyword.aggregatable != true {
+    tester.Errorf("expected aggregatable %t but got %t", true, fieldKeyword.aggregatable)
   }
-  if ackKeyword.searchable != true {
-    tester.Errorf("expected searchable %t but got %t", true, ackKeyword.searchable)
+  if fieldKeyword.searchable != true {
+    tester.Errorf("expected searchable %t but got %t", true, fieldKeyword.searchable)
   }
 }