Add migrate_to_ignored_device_id config flag to filebeat

Additional config flag, which allows to automatically migrate the existing state with device_id enabled to disabled. This flag is only effective if ignore_device_id is enabled as well.
breml · Aug 30, 2019 · 6737c58 · 6737c58
1 parent 178ac05
commit 6737c58
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 11 deletions.
diff --git a/filebeat/input/log/config.go b/filebeat/input/log/config.go
@@ -82,17 +82,18 @@ type config struct {
 	CleanInactive time.Duration `config:"clean_inactive" validate:"min=0"`
 
 	// Input
-	Enabled        bool            `config:"enabled"`
-	ExcludeFiles   []match.Matcher `config:"exclude_files"`
-	IgnoreOlder    time.Duration   `config:"ignore_older"`
-	Paths          []string        `config:"paths"`
-	ScanFrequency  time.Duration   `config:"scan_frequency" validate:"min=0,nonzero"`
-	CleanRemoved   bool            `config:"clean_removed"`
-	HarvesterLimit uint32          `config:"harvester_limit" validate:"min=0"`
-	Symlinks       bool            `config:"symlinks"`
-	TailFiles      bool            `config:"tail_files"`
-	RecursiveGlob  bool            `config:"recursive_glob.enabled"`
-	IgnoreDeviceID bool            `config:"ignore_device_id"`
+	Enabled                  bool            `config:"enabled"`
+	ExcludeFiles             []match.Matcher `config:"exclude_files"`
+	IgnoreOlder              time.Duration   `config:"ignore_older"`
+	Paths                    []string        `config:"paths"`
+	ScanFrequency            time.Duration   `config:"scan_frequency" validate:"min=0,nonzero"`
+	CleanRemoved             bool            `config:"clean_removed"`
+	HarvesterLimit           uint32          `config:"harvester_limit" validate:"min=0"`
+	Symlinks                 bool            `config:"symlinks"`
+	TailFiles                bool            `config:"tail_files"`
+	RecursiveGlob            bool            `config:"recursive_glob.enabled"`
+	IgnoreDeviceID           bool            `config:"ignore_device_id"`
+	MigrateToIgnoredDeviceID bool            `config:"migrate_to_ignored_device_id"`
 
 	// Harvester
 	BufferSize int    `config:"harvester_buffer_size"`

diff --git a/filebeat/input/log/input.go b/filebeat/input/log/input.go
@@ -168,6 +168,23 @@ func (p *Input) loadStates(states []file.State) error {
 		if p.matchesFile(state.Source) && p.matchesMeta(state.Meta) {
 			state.TTL = -1
 
+			if p.config.IgnoreDeviceID && p.config.MigrateToIgnoredDeviceID && state.FileStateOS.Device != 0 {
+				// Migrate existing device ID if device ID are ignored
+				// and migration is enabled.
+				logp.Debug("input", "migrate state for inode: %d from %d to device ID 0", state.FileStateOS.Inode, state.FileStateOS.Device)
+
+				// Remove old entry
+				p.removeState(state)
+
+				// Setup properties for the migrated state
+				state.TTL = -1
+				state.FileStateOS.Device = 0
+
+				// Reset ID because device ID as part of the ID has changed
+				state.Id = ""
+				state.ID()
+			}
+
 			// In case a input is tried to be started with an unfinished state matching the glob pattern
 			if !state.Finished {
 				return fmt.Errorf("Can only start an input when all related states are finished: %+v", state)