[Agent] Inject index with sane defaults to configuration (elastic#16903)

leehinman · Mar 11, 2020 · 96e4506 · 96e4506
1 parent b82a427
commit 96e4506
Show file tree

Hide file tree

Showing 39 changed files with 921 additions and 456 deletions.
diff --git a/x-pack/agent/CHANGELOG.asciidoc b/x-pack/agent/CHANGELOG.asciidoc
@@ -15,3 +15,5 @@
 - Fixed tests on windows {pull}16922[16922]
 
 ==== New features
+
+- Generate index name in a format type-dataset-namespace {pull}16903[16903]
diff --git a/x-pack/agent/_meta/agent.docker.yml b/x-pack/agent/_meta/agent.docker.yml
@@ -8,12 +8,20 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: default
+    use_output: default
+    inputs:
+      - type: system/metrics
+        streams:
+          - metricset: cpu
+            dataset: system.cpu
+          - metricset: memory
+            dataset: system.memory
+          - metricset: network
+            dataset: system.network
+          - metricset: filesystem
+            dataset: system.filesystem
 
 management:
   # Mode of management, the agent support two modes of operation:

diff --git a/x-pack/agent/_meta/agent.yml b/x-pack/agent/_meta/agent.yml
@@ -8,12 +8,20 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: default
+    use_output: default
+    inputs:
+      - type: system/metrics
+        streams:
+          - metricset: cpu
+            dataset: system.cpu
+          - metricset: memory
+            dataset: system.memory
+          - metricset: network
+            dataset: system.network
+          - metricset: filesystem
+            dataset: system.filesystem
 
 management:
   # Mode of management, the agent support two modes of operation:

diff --git a/x-pack/agent/_meta/common.p2.yml b/x-pack/agent/_meta/common.p2.yml
@@ -8,12 +8,20 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: default
+    use_output: default
+    inputs:
+      - type: system/metrics
+        streams:
+          - metricset: cpu
+            dataset: system.cpu
+          - metricset: memory
+            dataset: system.memory
+          - metricset: network
+            dataset: system.network
+          - metricset: filesystem
+            dataset: system.filesystem
 
 management:
   # Mode of magement, the agent support two modes of operation:

diff --git a/x-pack/agent/_meta/common.reference.p2.yml b/x-pack/agent/_meta/common.reference.p2.yml
@@ -8,12 +8,20 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: default
+    use_output: default
+    inputs:
+      - type: system/metrics
+        streams:
+          - metricset: cpu
+            dataset: system.cpu
+          - metricset: memory
+            dataset: system.memory
+          - metricset: network
+            dataset: system.network
+          - metricset: filesystem
+            dataset: system.filesystem
 
 management:
   # Mode of management, the agent currently only support the following mode:

diff --git a/x-pack/agent/agent.docker.yml b/x-pack/agent/agent.docker.yml
@@ -8,12 +8,20 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: default
+    use_output: default
+    inputs:
+      - type: system/metrics
+        streams:
+          - metricset: cpu
+            dataset: system.cpu
+          - metricset: memory
+            dataset: system.memory
+          - metricset: network
+            dataset: system.network
+          - metricset: filesystem
+            dataset: system.filesystem
 
 management:
   # Mode of management, the agent support two modes of operation:

diff --git a/x-pack/agent/agent.reference.yml b/x-pack/agent/agent.reference.yml
@@ -13,12 +13,20 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: default
+    use_output: default
+    inputs:
+      - type: system/metrics
+        streams:
+          - metricset: cpu
+            dataset: system.cpu
+          - metricset: memory
+            dataset: system.memory
+          - metricset: network
+            dataset: system.network
+          - metricset: filesystem
+            dataset: system.filesystem
 
 management:
   # Mode of management, the agent currently only support the following mode:

diff --git a/x-pack/agent/agent.yml b/x-pack/agent/agent.yml
@@ -13,12 +13,20 @@ outputs:
     username: elastic
     password: changeme
 
-streams:
-  - type: event/file
-    enabled: true
-    paths:
-      - /var/log/hello1.log
-      - /var/log/hello2.log
+datasources:
+  - namespace: default
+    use_output: default
+    inputs:
+      - type: system/metrics
+        streams:
+          - metricset: cpu
+            dataset: system.cpu
+          - metricset: memory
+            dataset: system.memory
+          - metricset: network
+            dataset: system.network
+          - metricset: filesystem
+            dataset: system.filesystem
 
 management:
   # Mode of magement, the agent support two modes of operation:

diff --git a/x-pack/agent/dev-tools/cmd/fakewebapi/action_example.json b/x-pack/agent/dev-tools/cmd/fakewebapi/action_example.json
@@ -3,9 +3,9 @@
   "success": true,
   "actions": [
     {
-      "type": "POLICY_CHANGE",
+      "type": "CONFIG_CHANGE",
       "data": {
-        "policy": {
+        "config": {
           "id": "default",
           "outputs": {
             "default": {

diff --git a/x-pack/agent/pkg/agent/application/action_store.go b/x-pack/agent/pkg/agent/application/action_store.go
@@ -33,14 +33,14 @@ func newActionStore(log *logger.Logger, store storeLoad) (*actionStore, error) {
 		return &actionStore{log: log, store: store}, nil
 	}
 
-	var action actionPolicyChangeSerializer
+	var action actionConfigChangeSerializer
 
 	dec := yaml.NewDecoder(reader)
 	if err := dec.Decode(&action); err != nil {
 		return nil, err
 	}
 
-	apc := fleetapi.ActionPolicyChange(action)
+	apc := fleetapi.ActionConfigChange(action)
 
 	return &actionStore{
 		log:    log,
@@ -53,7 +53,7 @@ func newActionStore(log *logger.Logger, store storeLoad) (*actionStore, error) {
 // any other type of action will be silently ignored.
 func (s *actionStore) Add(a action) {
 	switch v := a.(type) {
-	case *fleetapi.ActionPolicyChange:
+	case *fleetapi.ActionConfigChange:
 		// Only persist the action if the action is different.
 		if s.action != nil && s.action.ID() == v.ID() {
 			return
@@ -69,12 +69,12 @@ func (s *actionStore) Save() error {
 		return nil
 	}
 
-	apc, ok := s.action.(*fleetapi.ActionPolicyChange)
+	apc, ok := s.action.(*fleetapi.ActionConfigChange)
 	if !ok {
 		return fmt.Errorf("incompatible type, expected ActionPolicyChange and received %T", s.action)
 	}
 
-	serialize := actionPolicyChangeSerializer(*apc)
+	serialize := actionConfigChangeSerializer(*apc)
 
 	reader, err := yamlToReader(&serialize)
 	if err != nil {
@@ -98,7 +98,7 @@ func (s *actionStore) Actions() []action {
 	return []action{s.action}
 }
 
-// actionPolicyChangeSerializer is a struct that add YAML serialization, I don't think serialization
+// actionConfigChangeSerializer is a struct that add YAML serialization, I don't think serialization
 // is a concern of the fleetapi package. I went this route so I don't have to do much refactoring.
 //
 // There are four ways to achieve the same results:
@@ -108,14 +108,14 @@ func (s *actionStore) Actions() []action {
 // 4. We have two sets of type.
 //
 // This could be done in a refactoring.
-type actionPolicyChangeSerializer struct {
+type actionConfigChangeSerializer struct {
 	ActionID   string                 `yaml:"action_id"`
 	ActionType string                 `yaml:"action_type"`
-	Policy     map[string]interface{} `yaml:"policy"`
+	Config     map[string]interface{} `yaml:"config"`
 }
 
 // Add a guards between the serializer structs and the original struct.
-var _ actionPolicyChangeSerializer = actionPolicyChangeSerializer(fleetapi.ActionPolicyChange{})
+var _ actionConfigChangeSerializer = actionConfigChangeSerializer(fleetapi.ActionConfigChange{})
 
 // actionStoreAcker wraps an existing acker and will send any acked event to the action store,
 // its up to the action store to decide if we need to persist the event for future replay or just

diff --git a/x-pack/agent/pkg/agent/application/action_store_test.go b/x-pack/agent/pkg/agent/application/action_store_test.go
@@ -57,10 +57,10 @@ func TestActionStore(t *testing.T) {
 
 	t.Run("can save to disk known action type",
 		withFile(func(t *testing.T, file string) {
-			actionPolicyChange := &fleetapi.ActionPolicyChange{
+			actionConfigChange := &fleetapi.ActionConfigChange{
 				ActionID:   "abc123",
-				ActionType: "POLICY_CHANGE",
-				Policy: map[string]interface{}{
+				ActionType: "CONFIG_CHANGE",
+				Config: map[string]interface{}{
 					"hello": "world",
 				},
 			}
@@ -70,7 +70,7 @@ func TestActionStore(t *testing.T) {
 			require.NoError(t, err)
 
 			require.Equal(t, 0, len(store.Actions()))
-			store.Add(actionPolicyChange)
+			store.Add(actionConfigChange)
 			err = store.Save()
 			require.NoError(t, err)
 			require.Equal(t, 1, len(store.Actions()))
@@ -82,12 +82,12 @@ func TestActionStore(t *testing.T) {
 			actions := store1.Actions()
 			require.Equal(t, 1, len(actions))
 
-			require.Equal(t, actionPolicyChange, actions[0])
+			require.Equal(t, actionConfigChange, actions[0])
 		}))
 
 	t.Run("when we ACK we save to disk",
 		withFile(func(t *testing.T, file string) {
-			actionPolicyChange := &fleetapi.ActionPolicyChange{
+			actionConfigChange := &fleetapi.ActionConfigChange{
 				ActionID: "abc123",
 			}
 
@@ -98,7 +98,7 @@ func TestActionStore(t *testing.T) {
 			acker := newActionStoreAcker(&testAcker{}, store)
 			require.Equal(t, 0, len(store.Actions()))
 
-			require.NoError(t, acker.Ack(context.Background(), actionPolicyChange))
+			require.NoError(t, acker.Ack(context.Background(), actionConfigChange))
 			require.Equal(t, 1, len(store.Actions()))
 		}))
 }
diff --git a/x-pack/agent/pkg/agent/application/fleet_gateway_test.go b/x-pack/agent/pkg/agent/application/fleet_gateway_test.go
@@ -206,10 +206,10 @@ func TestFleetGateway(t *testing.T) {
 {
 	"actions": [
 		{
-			"type": "POLICY_CHANGE",
+			"type": "CONFIG_CHANGE",
 			"id": "id1",
 			"data": {
-				"policy": {
+				"config": {
 					"id": "policy-id"
 				}
 			}

diff --git a/x-pack/agent/pkg/agent/application/handler_action_policy_change.go b/x-pack/agent/pkg/agent/application/handler_action_policy_change.go
@@ -14,24 +14,24 @@ import (
 	"github.com/elastic/beats/v7/x-pack/agent/pkg/fleetapi"
 )
 
-type handlerPolicyChange struct {
+type handlerConfigChange struct {
 	log     *logger.Logger
 	emitter emitterFunc
 }
 
-func (h *handlerPolicyChange) Handle(ctx context.Context, a action, acker fleetAcker) error {
-	h.log.Debugf("HandlerPolicyChange: action '%+v' received", a)
-	action, ok := a.(*fleetapi.ActionPolicyChange)
+func (h *handlerConfigChange) Handle(ctx context.Context, a action, acker fleetAcker) error {
+	h.log.Debugf("handlerConfigChange: action '%+v' received", a)
+	action, ok := a.(*fleetapi.ActionConfigChange)
 	if !ok {
-		return fmt.Errorf("invalid type, expected ActionPolicyChange and received %T", a)
+		return fmt.Errorf("invalid type, expected ActionConfigChange and received %T", a)
 	}
 
-	c, err := config.NewConfigFrom(action.Policy)
+	c, err := config.NewConfigFrom(action.Config)
 	if err != nil {
 		return errors.New(err, "could not parse the configuration from the policy", errors.TypeConfig)
 	}
 
-	h.log.Debugf("HandlerPolicyChange: emit configuration for action %+v", a)
+	h.log.Debugf("handlerConfigChange: emit configuration for action %+v", a)
 	if err := h.emitter(c); err != nil {
 		return err
 	}
Original file line number	Diff line number	Diff line change
Expand Up		@@ -15,3 +15,5 @@
		- Fixed tests on windows {pull}16922[16922]

		==== New features

		- Generate index name in a format type-dataset-namespace {pull}16903[16903]