Merge pull request elastic#13 from ruflin/integration-tests

Integration tests

.gitignore

@@ -1,3 +1,4 @@
+/coverage
 /etc/filebeat.dev.yml
 .idea
 filebeat

.travis.yml

@@ -5,6 +5,13 @@ go:
 
 sudo: false
 
+addons:
+  apt:
+    packages:
+      - libpcap-dev
+      - python-virtualenv
+      - geoip-database
+
 before_install:
   # Redo the travis setup but with the elastic/filebeat path. This is needed so the package path is correct
   - mkdir -p $HOME/gopath/src/github.com/elastic/filebeat
@@ -17,13 +24,12 @@ install:
 
 script:
   - make check
-  - make cover
+  - make full-coverage
 
 notifications:
   email:
     - [email protected]
 
 after_success:
   # Copy profile.cov to coverage.txt because codecov.io requires this file
-  - cp profile.cov coverage.txt
-  - bash <(curl -s https://codecov.io/bash)
+  - bash <(curl -s https://codecov.io/bash) -f coverage/full.cov

Makefile

@@ -1,6 +1,6 @@
 ARCH?=$(shell uname -m)
 GODEP=$(GOPATH)/bin/godep
-
+GOFILES = $(shell find . -type f -name '*.go')
 
 filebeat: $(GOFILES)
 	# first make sure we have godep
@@ -17,9 +17,10 @@ check:
 clean:
 	gofmt -w .
 	-rm filebeat
+	-rm filebeat.test
 	-rm .filebeat
 	-rm profile.cov
-	-rm -r cover
+	-rm -r coverage
 
 .PHONY: run
 run: filebeat
@@ -29,18 +30,27 @@ run: filebeat
 test:
 	$(GODEP) go test -short ./...
 
-.PHONY: cover
-cover:
+.PHONY: coverage
+coverage:
 	# gotestcover is needed to fetch coverage for multiple packages
 	go get github.com/pierrre/gotestcover
-	GOPATH=$(shell $(GODEP) path):$(GOPATH) $(GOPATH)/bin/gotestcover -coverprofile=profile.cov -covermode=count github.com/elastic/filebeat/...
-	mkdir -p cover
-	$(GODEP) go tool cover -html=profile.cov -o cover/coverage.html
+	mkdir -p coverage
+	GOPATH=$(shell $(GODEP) path):$(GOPATH) $(GOPATH)/bin/gotestcover -coverprofile=coverage/unit.cov -covermode=count github.com/elastic/filebeat/...
+	$(GODEP) go tool cover -html=coverage/unit.cov -o coverage/unit.html
 
 # Command used by CI Systems
 .PHONE: testsuite
 testsuite: filebeat
-	make cover
+	make coverage
 
 filebeat.test: $(GOFILES)
 	$(GODEP) go test -c -cover -covermode=count -coverpkg ./...
+
+full-coverage:
+	make coverage
+	make -C ./tests/integration coverage
+	# Writes count mode on top of file
+	echo 'mode: count' > ./coverage/full.cov
+	# Collects all integration coverage files and skips top line with mode
+	tail -q -n +2 ./coverage/*.cov >> ./coverage/full.cov
+	$(GODEP) go tool cover -html=./coverage/full.cov -o coverage/full.html

beat/filebeat.go

@@ -115,6 +115,13 @@ func (fb *Filebeat) Cleanup(b *beat.Beat) error {
 
 func (fb *Filebeat) Stop() {
 
+	// Stop harvesters
+	// Stop prospectors
+	// Flush what is in spooler
+	// Write state
+
+	fb.stopSpooler()
+
 	// FIXME: Improve to first write state and then close channels
 	close(fb.SpoolChan)
 	close(fb.publisherChan)

beat/spooler.go

@@ -6,6 +6,8 @@ import (
 	"time"
 )
 
+var stopSpooler = false
+
 // startSpooler Starts up the spooler and starts listening on the spool channel from the harvester
 // Sends then bulk updates to the publisher channel
 func (fb *Filebeat) startSpooler(options *cfg.FilebeatConfig) {
@@ -58,5 +60,13 @@ func (fb *Filebeat) startSpooler(options *cfg.FilebeatConfig) {
 				}
 			}
 		}
+
+		if stopSpooler {
+			break
+		}
 	}
 }
+
+func (fb *Filebeat) stopSpooler() {
+	stopSpooler = true
+}

changelog.md

@@ -47,6 +47,7 @@ Questions:
 * Getting config from env was removed. I think a better method like getting it from es should be used: https://github.com/elastic/logstash-forwarder/pull/435
 * What should we do about multiple configs? Just provide some docs? https://github.com/elastic/logstash-forwarder/issues/136 currently working with -c for beat -config for dirs
 * Command line config option -config was renamed to configDir. Should also be introduced as config file param in case we want to keep it
+* Rethink dead-time: https://github.com/elastic/logstash-forwarder/issues/460
 
 Notes:
 * Should every config entry have a name -> make it possible to know from which config entry something comes.

crawler/harvester.go

@@ -91,6 +91,7 @@ func (h *Harvester) handleReadlineError(lastTimeRead time.Time, err error) error
 			logp.Debug("harvester", "File truncated, seeking to beginning: %s", h.Path)
 			h.file.Seek(0, os.SEEK_SET)
 			h.Offset = 0
+
 		} else if age := time.Since(lastTimeRead); age > h.FileConfig.IgnoreOlderDuration {
 			// if lastTimeRead was more than ignore older and ignore older is set, this file is probably dead. Stop watching it.
 			logp.Debug("harvester", "Stopping harvest of ", h.Path, "last change was: ", age)
@@ -157,6 +158,7 @@ func (h *Harvester) open() *os.File {
 
 	// Check we are not following a rabbit hole (symlinks, etc.)
 	if !file.IsRegularFile() {
+		// TODO: This should be replaced by a normal error
 		panic(fmt.Errorf("Harvester file error"))
 	}
 

crawler/harvester_test.go

@@ -0,0 +1,20 @@
+package crawler
+
+import (
+	"github.com/stretchr/testify/assert"
+	"testing"
+)
+
+// Most harvester tests need real files to tes that can be modified. These tests are implemented with
+// integration tests
+
+func TestExampleTest(t *testing.T) {
+
+	h := Harvester{
+		Path:   "/var/log/",
+		Offset: 0,
+	}
+
+	assert.Equal(t, "/var/log/", h.Path)
+
+}

crawler/prospector.go

@@ -309,3 +309,7 @@ func (p *Prospector) calculateResume(file string, fileinfo os.FileInfo, resume *
 	// New file so just start from an automatic position
 	return 0, false
 }
+
+func (p *Prospector) Stop() {
+
+}

tests/integration/.gitignore

@@ -0,0 +1,5 @@
+env
+last_run
+run
+
+*.pyc

tests/integration/Makefile

@@ -0,0 +1,30 @@
+SHELL := /bin/bash
+PROCESSES ?= 4
+TIMEOUT ?= 60
+
+env: env/bin/activate
+env/bin/activate: requirements.txt
+	test -d env || virtualenv env > /dev/null
+	. env/bin/activate && pip install -Ur requirements.txt > /dev/null
+	touch env/bin/activate
+
+.PHONY: test
+test: env
+	make -C ../.. filebeat.test
+	. env/bin/activate; nosetests --processes=${PROCESSES} --process-timeout=$(TIMEOUT)
+
+.PHONY: coverage
+coverage: test
+	mkdir -p ../../coverage
+	# Writes count mode on top of file
+	echo 'mode: count' > ../../coverage/integration.cov
+	# Collects all integration coverage files and skips top line with mode
+	tail -q -n +2 ./run/**/*.cov >> ../../coverage/integration.cov
+	# Generates HTML output
+	go tool cover -html=../../coverage/integration.cov -o ../../coverage/integration.html
+
+.PHONY: clean
+clean:
+	-rm -r env
+	-rm -r run
+	-rm last_run

tests/integration/config/filebeat.yml.j2

@@ -0,0 +1,86 @@
+############################# Filbeat ######################################
+filebeat:
+  files:
+    -
+      # Paths that should be crawled and fetched
+      paths:
+        - "{{ path }}"
+      # Type of the files. Annotated in every documented
+      type: log
+  spoolSize:
+  harvesterBufferSize:
+  cpuProfileFile:
+  idleTimeout:
+  tailOnRotate:
+  quiet:
+  registryfile: {{ fb.working_dir + '/' }}.filebeat
+
+
+
+############################# Shipper ############################################
+shipper:
+
+ # The name of the shipper that publishes the network data. It can be used to group
+ # all the transactions sent by a single shipper in the web interface.
+ # If this options is not defined, the hostname is used.
+ name:
+
+ # The tags of the shipper are included in their own field with each
+ # transaction published. Tags make it easy to group servers by different
+ # logical properties.
+ tags: [
+    {%- if agent_tags -%}
+        {%- for tag in agent_tags -%}
+            "{{ tag }}"
+            {%- if not loop.last %}, {% endif -%}
+        {%- endfor -%}
+    {%- endif -%}]
+
+
+ # Uncomment the following if you want to ignore transactions created
+ # by the server on which the shipper is installed. This option is useful
+ # to remove duplicates if shippers are installed on multiple servers.
+ # ignore_outgoing: true
+
+
+
+############################# Output ############################################
+
+# Configure what outputs to use when sending the data collected by packetbeat.
+# You can enable one or multiple outputs by setting enabled option to true.
+output:
+
+  # Elasticsearch as output
+  # Options:
+  # host, port: where Elasticsearch is listening on
+  # save_topology: specify if the topology is saved in Elasticsearch
+  #elasticsearch:
+  #  enabled: false
+  #  host: localhost
+  #  port: 9200
+  #  save_topology: true
+
+  # Redis as output
+  # Options:
+  # host, port: where Redis is listening on
+  # save_topology: specify if the topology is saved in Redis
+  #redis:
+  #  enabled: false
+  #  host: localhost
+  #  port: 6379
+  #  save_topology: true
+
+  # File as output
+  # Options
+  # path: where to save the files
+  # filename: name of the files
+  # rotate_every_kb: maximum size of the files in path
+  # number of files: maximum number of files in path
+  file:
+    enabled: true
+    path: "{{ output_file_path|default(fb.working_dir + "/output") }}"
+    filename: "{{ output_file_filename|default("filebeat") }}"
+    rotate_every_kb: 1000
+    #number_of_files: 7
+
+# vim: set ft=jinja: