diff --git a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl index 310c0fe4abc..c920b7dbec8 100644 --- a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl +++ b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl @@ -52,7 +52,7 @@ filebeat.inputs: # Method to determine if two files are the same or not. By default # the Beat considers two files the same if their inode and device id are the same. - #file_identity.inode_deviceid: ~ + #file_identity.native: ~ # Optional additional fields. These fields can be freely picked # to add additional information to the crawled log files for filtering diff --git a/filebeat/docs/inputs/input-common-file-options.asciidoc b/filebeat/docs/inputs/input-common-file-options.asciidoc index b52e4660d44..9bd93e97efc 100644 --- a/filebeat/docs/inputs/input-common-file-options.asciidoc +++ b/filebeat/docs/inputs/input-common-file-options.asciidoc @@ -409,12 +409,12 @@ Different `file_identity` methods can be configured to suit the environment where you are collecting log messages. -*`inode_deviceid`*:: The default behaviour of {beatname_uc} is to differentiate +*`native`*:: The default behaviour of {beatname_uc} is to differentiate between files using their inodes and device ids. [source,yaml] ---- -file_identity.inode_deviceid: ~ +file_identity.native: ~ ---- *`path`*:: To identify files based on their paths use this strategy. @@ -433,7 +433,9 @@ file_identity.path: ~ ---- *`inode_marker`*:: If the device id changes from time to time, you must use -this method to distinguish files. Set the location of the marker file the following way: +this method to distinguish files. This option is not supported on Windows. + +Set the location of the marker file the following way: [source,yaml] ---- diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index decc9d584e5..f52b731140f 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -439,7 +439,7 @@ filebeat.inputs: # Method to determine if two files are the same or not. By default # the Beat considers two files the same if their inode and device id are the same. - #file_identity.inode_deviceid: ~ + #file_identity.native: ~ # Optional additional fields. These fields can be freely picked # to add additional information to the crawled log files for filtering diff --git a/filebeat/input/file/identifier.go b/filebeat/input/file/identifier.go index c2c49f297b0..c16535f3e19 100644 --- a/filebeat/input/file/identifier.go +++ b/filebeat/input/file/identifier.go @@ -19,33 +19,28 @@ package file import ( "fmt" - "io/ioutil" - "os" - "path/filepath" "strconv" "strings" - "time" "github.com/mitchellh/hashstructure" "github.com/elastic/beats/v7/libbeat/common" - "github.com/elastic/beats/v7/libbeat/logp" ) const ( - inodeDeviceIDName = "inode_deviceid" - pathName = "path" - inodeMarkerName = "inode_marker" + nativeName = "native" + pathName = "path" + inodeMarkerName = "inode_marker" - DefaultIdentifierName = inodeDeviceIDName + DefaultIdentifierName = nativeName identitySep = "::" ) var ( identifierFactories = map[string]IdentifierFactory{ - inodeDeviceIDName: newINodeDeviceIdentifier, - pathName: newPathIdentifier, - inodeMarkerName: newINodeMarkerIdentifier, + nativeName: newINodeDeviceIdentifier, + pathName: newPathIdentifier, + inodeMarkerName: newINodeMarkerIdentifier, } ) @@ -78,7 +73,7 @@ type inodeDeviceIdentifier struct { func newINodeDeviceIdentifier(_ *common.Config) (StateIdentifier, error) { return &inodeDeviceIdentifier{ - name: inodeDeviceIDName, + name: nativeName, }, nil } @@ -102,73 +97,6 @@ func (p *pathIdentifier) GenerateID(s State) (id, identifierType string) { return genIDWithHash(s.Meta, stateID), p.name } -type inodeMarkerIdentifier struct { - log *logp.Logger - name string - markerPath string - - markerFileLastModifitaion time.Time - markerTxt string -} - -func newINodeMarkerIdentifier(cfg *common.Config) (StateIdentifier, error) { - var config struct { - MarkerPath string `config:"path" validate:"required"` - } - err := cfg.Unpack(&config) - if err != nil { - return nil, fmt.Errorf("error while reading configuration of INode + marker file configuration: %v", err) - } - - fi, err := os.Stat(config.MarkerPath) - if err != nil { - return nil, fmt.Errorf("error while opening marker file at %s: %v", config.MarkerPath, err) - } - markerContent, err := ioutil.ReadFile(config.MarkerPath) - if err != nil { - return nil, fmt.Errorf("error while reading marker file at %s: %v", config.MarkerPath, err) - } - return &inodeMarkerIdentifier{ - log: logp.NewLogger("inode_marker_identifier_" + filepath.Base(config.MarkerPath)), - name: inodeMarkerName, - markerPath: config.MarkerPath, - markerFileLastModifitaion: fi.ModTime(), - markerTxt: string(markerContent), - }, nil -} - -func (i *inodeMarkerIdentifier) markerContents() string { - f, err := os.Open(i.markerPath) - if err != nil { - i.log.Errorf("Failed to open marker file %s: %v", i.markerPath, err) - return "" - } - defer f.Close() - - fi, err := f.Stat() - if err != nil { - i.log.Errorf("Failed to fetch file information for %s: %v", i.markerPath, err) - return "" - } - if i.markerFileLastModifitaion.Before(fi.ModTime()) { - contents, err := ioutil.ReadFile(i.markerPath) - if err != nil { - i.log.Errorf("Error while reading contents of marker file: %v", err) - return "" - } - i.markerTxt = string(contents) - } - - return i.markerTxt -} - -func (i *inodeMarkerIdentifier) GenerateID(s State) (id, identifierType string) { - m := i.markerContents() - - stateID := fmt.Sprintf("%s%s%s-%s", i.name, identitySep, s.FileStateOS.InodeString(), m) - return genIDWithHash(s.Meta, stateID), i.name -} - func genIDWithHash(meta map[string]string, fileID string) string { if len(meta) == 0 { return fileID diff --git a/filebeat/input/file/identifier_inode_deviceid.go b/filebeat/input/file/identifier_inode_deviceid.go new file mode 100644 index 00000000000..f5e191744d6 --- /dev/null +++ b/filebeat/input/file/identifier_inode_deviceid.go @@ -0,0 +1,98 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build !windows + +package file + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/elastic/beats/v7/libbeat/common" + "github.com/elastic/beats/v7/libbeat/logp" +) + +type inodeMarkerIdentifier struct { + log *logp.Logger + name string + markerPath string + + markerFileLastModifitaion time.Time + markerTxt string +} + +func newINodeMarkerIdentifier(cfg *common.Config) (StateIdentifier, error) { + var config struct { + MarkerPath string `config:"path" validate:"required"` + } + err := cfg.Unpack(&config) + if err != nil { + return nil, fmt.Errorf("error while reading configuration of INode + marker file configuration: %v", err) + } + + fi, err := os.Stat(config.MarkerPath) + if err != nil { + return nil, fmt.Errorf("error while opening marker file at %s: %v", config.MarkerPath, err) + } + markerContent, err := ioutil.ReadFile(config.MarkerPath) + if err != nil { + return nil, fmt.Errorf("error while reading marker file at %s: %v", config.MarkerPath, err) + } + return &inodeMarkerIdentifier{ + log: logp.NewLogger("inode_marker_identifier_" + filepath.Base(config.MarkerPath)), + name: inodeMarkerName, + markerPath: config.MarkerPath, + markerFileLastModifitaion: fi.ModTime(), + markerTxt: string(markerContent), + }, nil +} + +func (i *inodeMarkerIdentifier) markerContents() string { + f, err := os.Open(i.markerPath) + if err != nil { + i.log.Errorf("Failed to open marker file %s: %v", i.markerPath, err) + return "" + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + i.log.Errorf("Failed to fetch file information for %s: %v", i.markerPath, err) + return "" + } + if i.markerFileLastModifitaion.Before(fi.ModTime()) { + contents, err := ioutil.ReadFile(i.markerPath) + if err != nil { + i.log.Errorf("Error while reading contents of marker file: %v", err) + return "" + } + i.markerTxt = string(contents) + } + + return i.markerTxt +} + +func (i *inodeMarkerIdentifier) GenerateID(s State) (id, identifierType string) { + m := i.markerContents() + + stateID := fmt.Sprintf("%s%s%s-%s", i.name, identitySep, s.FileStateOS.InodeString(), m) + return genIDWithHash(s.Meta, stateID), i.name +} diff --git a/filebeat/input/file/identifier_inode_deviceid_windows.go b/filebeat/input/file/identifier_inode_deviceid_windows.go new file mode 100644 index 00000000000..9fb1152a33c --- /dev/null +++ b/filebeat/input/file/identifier_inode_deviceid_windows.go @@ -0,0 +1,30 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package file + +import ( + "fmt" + + "github.com/elastic/beats/v7/libbeat/common" +) + +func newINodeMarkerIdentifier(cfg *common.Config) (StateIdentifier, error) { + return nil, fmt.Errorf("inode_deviceid is not supported on Windows") +} diff --git a/filebeat/input/file/identifier_test_windows.go b/filebeat/input/file/identifier_test_windows.go new file mode 100644 index 00000000000..544dbad2546 --- /dev/null +++ b/filebeat/input/file/identifier_test_windows.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// +build windows + +package file + +import "testing" + +func TestInodeMarkerError(t *testing.T) { + _, err := newINodeMarkerIdentifier(nil) + if err == nil { + t.Fatal("inode_marker should not be supported on windows") + } +} diff --git a/filebeat/tests/system/test_harvester.py b/filebeat/tests/system/test_harvester.py index a6e7004f8d0..cb30dc4976b 100644 --- a/filebeat/tests/system/test_harvester.py +++ b/filebeat/tests/system/test_harvester.py @@ -79,7 +79,7 @@ def test_close_renamed(self): def test_close_removed(self): """ - Checks that a file is closed if removed with inode_deviceid file identifier + Checks that a file is closed if removed with native file identifier """ self.render_config_template( path=os.path.abspath(self.working_dir) + "/log/test.log", diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 276476d83f5..879676c267c 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -1547,7 +1547,7 @@ filebeat.inputs: # Method to determine if two files are the same or not. By default # the Beat considers two files the same if their inode and device id are the same. - #file_identity.inode_deviceid: ~ + #file_identity.native: ~ # Optional additional fields. These fields can be freely picked # to add additional information to the crawled log files for filtering