Add ignore_device_id config flag to filebeat

Additional config flag, which allows to disable the device_id as part of the
unique file identification for situations, where the device ID is not
stable.

filebeat/input/file/state.go

@@ -43,15 +43,19 @@ type State struct {
 }
 
 // NewState creates a new file state
-func NewState(fileInfo os.FileInfo, path string, t string, meta map[string]string) State {
+func NewState(fileInfo os.FileInfo, path string, t string, meta map[string]string, ignoreDeviceID bool) State {
 	if len(meta) == 0 {
 		meta = nil
 	}
+	fileStateOS := file.GetOSState(fileInfo)
+	if ignoreDeviceID {
+		fileStateOS.Device = 0
+	}
 	return State{
 		Fileinfo:    fileInfo,
 		Source:      path,
 		Finished:    false,
-		FileStateOS: file.GetOSState(fileInfo),
+		FileStateOS: fileStateOS,
 		Timestamp:   time.Now(),
 		TTL:         -1, // By default, state does have an infinite ttl
 		Type:        t,

filebeat/input/log/config.go

@@ -54,6 +54,7 @@ var (
 		ScanSort:       "",
 		ScanOrder:      "asc",
 		RecursiveGlob:  true,
+		IgnoreDeviceID: false,
 
 		// Harvester
 		BufferSize:     16 * humanize.KiByte,
@@ -91,6 +92,7 @@ type config struct {
 	Symlinks       bool            `config:"symlinks"`
 	TailFiles      bool            `config:"tail_files"`
 	RecursiveGlob  bool            `config:"recursive_glob.enabled"`
+	IgnoreDeviceID bool            `config:"ignore_device_id"`
 
 	// Harvester
 	BufferSize int    `config:"harvester_buffer_size"`

filebeat/input/log/harvester.go

@@ -505,6 +505,9 @@ func (h *Harvester) getState() file.State {
 
 	// refreshes the values in State with the values from the harvester itself
 	state.FileStateOS = file_helper.GetOSState(h.state.Fileinfo)
+	if h.config.IgnoreDeviceID {
+		state.FileStateOS.Device = 0
+	}
 	return state
 }
 

filebeat/input/log/input.go

@@ -227,7 +227,8 @@ func (p *Input) Run() {
 				}
 			} else {
 				// Check if existing source on disk and state are the same. Remove if not the case.
-				newState := file.NewState(stat, state.Source, p.config.Type, p.meta)
+				newState := file.NewState(stat, state.Source, p.config.Type, p.meta, p.config.IgnoreDeviceID)
+				// Could be done in `IsSame` as well, if the signature of file.NewState should stay untouched: newState.FileStateOS.Device = 0
 				if !newState.FileStateOS.IsSame(state.FileStateOS) {
 					p.removeState(state)
 					logp.Debug("input", "Remove state for file as file removed or renamed: %s", state.Source)
@@ -420,7 +421,7 @@ func getFileState(path string, info os.FileInfo, p *Input) (file.State, error) {
 	}
 	logp.Debug("input", "Check file for harvesting: %s", absolutePath)
 	// Create new state for comparison
-	newState := file.NewState(info, absolutePath, p.config.Type, p.meta)
+	newState := file.NewState(info, absolutePath, p.config.Type, p.meta, p.config.IgnoreDeviceID)
 	return newState, nil
 }