rename inode_deviceid to natice && remove support for inode_marker on…

… windows

filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl

@@ -52,7 +52,7 @@ filebeat.inputs:
 
   # Method to determine if two files are the same or not. By default
   # the Beat considers two files the same if their inode and device id are the same.
-  #file_identity.inode_deviceid: ~
+  #file_identity.native: ~
 
   # Optional additional fields. These fields can be freely picked
   # to add additional information to the crawled log files for filtering

filebeat/docs/inputs/input-common-file-options.asciidoc

@@ -409,12 +409,12 @@ Different `file_identity` methods can be configured to suit the
 environment where you are collecting log messages.
 
 
-*`inode_deviceid`*:: The default behaviour of {beatname_uc} is to differentiate
+*`native`*:: The default behaviour of {beatname_uc} is to differentiate
 between files using their inodes and device ids.
 
 [source,yaml]
 ----
-file_identity.inode_deviceid: ~
+file_identity.native: ~
 ----
 
 *`path`*:: To identify files based on their paths use this strategy.
@@ -433,7 +433,9 @@ file_identity.path: ~
 ----
 
 *`inode_marker`*:: If the device id changes from time to time, you must use
-this method to distinguish files. Set the location of the marker file the following way:
+this method to distinguish files. This option is not supported on Windows.
+
+Set the location of the marker file the following way:
 
 [source,yaml]
 ----

filebeat/filebeat.reference.yml

@@ -439,7 +439,7 @@ filebeat.inputs:
 
   # Method to determine if two files are the same or not. By default
   # the Beat considers two files the same if their inode and device id are the same.
-  #file_identity.inode_deviceid: ~
+  #file_identity.native: ~
 
   # Optional additional fields. These fields can be freely picked
   # to add additional information to the crawled log files for filtering

filebeat/input/file/identifier.go

@@ -19,33 +19,28 @@ package file
 
 import (
 	"fmt"
-	"io/ioutil"
-	"os"
-	"path/filepath"
 	"strconv"
 	"strings"
-	"time"
 
 	"github.com/mitchellh/hashstructure"
 
 	"github.com/elastic/beats/v7/libbeat/common"
-	"github.com/elastic/beats/v7/libbeat/logp"
 )
 
 const (
-	inodeDeviceIDName = "inode_deviceid"
-	pathName          = "path"
-	inodeMarkerName   = "inode_marker"
+	nativeName      = "native"
+	pathName        = "path"
+	inodeMarkerName = "inode_marker"
 
-	DefaultIdentifierName = inodeDeviceIDName
+	DefaultIdentifierName = nativeName
 	identitySep           = "::"
 )
 
 var (
 	identifierFactories = map[string]IdentifierFactory{
-		inodeDeviceIDName: newINodeDeviceIdentifier,
-		pathName:          newPathIdentifier,
-		inodeMarkerName:   newINodeMarkerIdentifier,
+		nativeName:      newINodeDeviceIdentifier,
+		pathName:        newPathIdentifier,
+		inodeMarkerName: newINodeMarkerIdentifier,
 	}
 )
 
@@ -78,7 +73,7 @@ type inodeDeviceIdentifier struct {
 
 func newINodeDeviceIdentifier(_ *common.Config) (StateIdentifier, error) {
 	return &inodeDeviceIdentifier{
-		name: inodeDeviceIDName,
+		name: nativeName,
 	}, nil
 }
 
@@ -102,73 +97,6 @@ func (p *pathIdentifier) GenerateID(s State) (id, identifierType string) {
 	return genIDWithHash(s.Meta, stateID), p.name
 }
 
-type inodeMarkerIdentifier struct {
-	log        *logp.Logger
-	name       string
-	markerPath string
-
-	markerFileLastModifitaion time.Time
-	markerTxt                 string
-}
-
-func newINodeMarkerIdentifier(cfg *common.Config) (StateIdentifier, error) {
-	var config struct {
-		MarkerPath string `config:"path" validate:"required"`
-	}
-	err := cfg.Unpack(&config)
-	if err != nil {
-		return nil, fmt.Errorf("error while reading configuration of INode + marker file configuration: %v", err)
-	}
-
-	fi, err := os.Stat(config.MarkerPath)
-	if err != nil {
-		return nil, fmt.Errorf("error while opening marker file at %s: %v", config.MarkerPath, err)
-	}
-	markerContent, err := ioutil.ReadFile(config.MarkerPath)
-	if err != nil {
-		return nil, fmt.Errorf("error while reading marker file at %s: %v", config.MarkerPath, err)
-	}
-	return &inodeMarkerIdentifier{
-		log:                       logp.NewLogger("inode_marker_identifier_" + filepath.Base(config.MarkerPath)),
-		name:                      inodeMarkerName,
-		markerPath:                config.MarkerPath,
-		markerFileLastModifitaion: fi.ModTime(),
-		markerTxt:                 string(markerContent),
-	}, nil
-}
-
-func (i *inodeMarkerIdentifier) markerContents() string {
-	f, err := os.Open(i.markerPath)
-	if err != nil {
-		i.log.Errorf("Failed to open marker file %s: %v", i.markerPath, err)
-		return ""
-	}
-	defer f.Close()
-
-	fi, err := f.Stat()
-	if err != nil {
-		i.log.Errorf("Failed to fetch file information for %s: %v", i.markerPath, err)
-		return ""
-	}
-	if i.markerFileLastModifitaion.Before(fi.ModTime()) {
-		contents, err := ioutil.ReadFile(i.markerPath)
-		if err != nil {
-			i.log.Errorf("Error while reading contents of marker file: %v", err)
-			return ""
-		}
-		i.markerTxt = string(contents)
-	}
-
-	return i.markerTxt
-}
-
-func (i *inodeMarkerIdentifier) GenerateID(s State) (id, identifierType string) {
-	m := i.markerContents()
-
-	stateID := fmt.Sprintf("%s%s%s-%s", i.name, identitySep, s.FileStateOS.InodeString(), m)
-	return genIDWithHash(s.Meta, stateID), i.name
-}
-
 func genIDWithHash(meta map[string]string, fileID string) string {
 	if len(meta) == 0 {
 		return fileID

filebeat/input/file/identifier_inode_deviceid.go

@@ -0,0 +1,98 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build !windows
+
+package file
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+	"github.com/elastic/beats/v7/libbeat/logp"
+)
+
+type inodeMarkerIdentifier struct {
+	log        *logp.Logger
+	name       string
+	markerPath string
+
+	markerFileLastModifitaion time.Time
+	markerTxt                 string
+}
+
+func newINodeMarkerIdentifier(cfg *common.Config) (StateIdentifier, error) {
+	var config struct {
+		MarkerPath string `config:"path" validate:"required"`
+	}
+	err := cfg.Unpack(&config)
+	if err != nil {
+		return nil, fmt.Errorf("error while reading configuration of INode + marker file configuration: %v", err)
+	}
+
+	fi, err := os.Stat(config.MarkerPath)
+	if err != nil {
+		return nil, fmt.Errorf("error while opening marker file at %s: %v", config.MarkerPath, err)
+	}
+	markerContent, err := ioutil.ReadFile(config.MarkerPath)
+	if err != nil {
+		return nil, fmt.Errorf("error while reading marker file at %s: %v", config.MarkerPath, err)
+	}
+	return &inodeMarkerIdentifier{
+		log:                       logp.NewLogger("inode_marker_identifier_" + filepath.Base(config.MarkerPath)),
+		name:                      inodeMarkerName,
+		markerPath:                config.MarkerPath,
+		markerFileLastModifitaion: fi.ModTime(),
+		markerTxt:                 string(markerContent),
+	}, nil
+}
+
+func (i *inodeMarkerIdentifier) markerContents() string {
+	f, err := os.Open(i.markerPath)
+	if err != nil {
+		i.log.Errorf("Failed to open marker file %s: %v", i.markerPath, err)
+		return ""
+	}
+	defer f.Close()
+
+	fi, err := f.Stat()
+	if err != nil {
+		i.log.Errorf("Failed to fetch file information for %s: %v", i.markerPath, err)
+		return ""
+	}
+	if i.markerFileLastModifitaion.Before(fi.ModTime()) {
+		contents, err := ioutil.ReadFile(i.markerPath)
+		if err != nil {
+			i.log.Errorf("Error while reading contents of marker file: %v", err)
+			return ""
+		}
+		i.markerTxt = string(contents)
+	}
+
+	return i.markerTxt
+}
+
+func (i *inodeMarkerIdentifier) GenerateID(s State) (id, identifierType string) {
+	m := i.markerContents()
+
+	stateID := fmt.Sprintf("%s%s%s-%s", i.name, identitySep, s.FileStateOS.InodeString(), m)
+	return genIDWithHash(s.Meta, stateID), i.name
+}

filebeat/input/file/identifier_inode_deviceid_windows.go

@@ -0,0 +1,30 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build windows
+
+package file
+
+import (
+	"fmt"
+
+	"github.com/elastic/beats/v7/libbeat/common"
+)
+
+func newINodeMarkerIdentifier(cfg *common.Config) (StateIdentifier, error) {
+	return nil, fmt.Errorf("inode_deviceid is not supported on Windows")
+}

filebeat/input/file/identifier_test_windows.go

@@ -0,0 +1,29 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// +build windows
+
+package file
+
+import "testing"
+
+func TestInodeMarkerError(t *testing.T) {
+	_, err := newINodeMarkerIdentifier(nil)
+	if err == nil {
+		t.Fatal("inode_marker should not be supported on windows")
+	}
+}

filebeat/tests/system/test_harvester.py

@@ -79,7 +79,7 @@ def test_close_renamed(self):
 
     def test_close_removed(self):
         """
-        Checks that a file is closed if removed with inode_deviceid file identifier
+        Checks that a file is closed if removed with native file identifier
         """
         self.render_config_template(
             path=os.path.abspath(self.working_dir) + "/log/test.log",

x-pack/filebeat/filebeat.reference.yml

@@ -1547,7 +1547,7 @@ filebeat.inputs:
 
   # Method to determine if two files are the same or not. By default
   # the Beat considers two files the same if their inode and device id are the same.
-  #file_identity.inode_deviceid: ~
+  #file_identity.native: ~
 
   # Optional additional fields. These fields can be freely picked
   # to add additional information to the crawled log files for filtering