From cce82846bce9baa42319530d5ee807f2faef7bd0 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Wed, 15 Jul 2020 17:10:26 -0400 Subject: [PATCH] Favor direct mage invocation on CI This changes Jenkins and Travis to directly invoke mage where possible instead of going through make. Some of the remaining make don't yet have a mage equivalant (mainly crosscompile). For Packetbeat this add Jenkins stages to test on darwin and Windows. There were a few fixes I had to make related to these changes: - Add some mage target dependencies to ensure fields and dashboards are ready when tests use them. - Swap the order of the Go imports for dev-tools/mage/targets/integtest and unittest so that unit tests run before integ tests when running the 'mage test' target. - chown the shared Python venv that is in the root of the repo after Dockerized integ tests exit to ensure there are no permissions issues caused by root owned files. - I found a few Python string encoding issues that caused tests failures. I thought we fixed these during the Python 3 conversion, but something here exposed a few that we didn't address. One thing of note that I did not correct. Journalbeat has system tests but they are not executed. The existing/old Makefile has SYSTEM_TESTS=false so they are not executed. So when I switched it to mage I left it as 'mage goUnitTest' to avoid the Python when failed when I tried it. --- .travis.yml | 124 ++++++++++++++++++++------ Jenkinsfile | 66 +++++++++----- auditbeat/magefile.go | 6 +- dev-tools/mage/integtest.go | 2 +- dev-tools/mage/integtest_docker.go | 8 ++ dev-tools/mage/pytest.go | 26 +++--- filebeat/tests/system/test_input.py | 1 + heartbeat/magefile.go | 4 +- libbeat/tests/system/test_http.py | 4 +- magefile.go | 1 - packetbeat/magefile.go | 3 +- packetbeat/tests/system/packetbeat.py | 2 +- x-pack/libbeat/magefile.go | 4 +- 13 files changed, 181 insertions(+), 70 deletions(-) diff --git a/.travis.yml b/.travis.yml index b9f903b023b2..1fd774492ba8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -38,41 +38,55 @@ jobs: # Filebeat - os: linux before_install: .ci/scripts/travis_has_changes.sh filebeat libbeat || travis_terminate 0 - env: TARGETS="-C filebeat testsuite" + env: + - PROJECT=filebeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test - os: osx before_install: .ci/scripts/travis_has_changes.sh filebeat libbeat || travis_terminate 0 - env: TARGETS="TEST_ENVIRONMENT=0 -C filebeat testsuite" + env: + - PROJECT=filebeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/filebeat filebeat libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/filebeat testsuite" + env: + - PROJECT=x-pack/filebeat + - MAGE='build test' go: $(GO_VERSION) stage: test # Heartbeat - os: linux before_install: .ci/scripts/travis_has_changes.sh heartbeat libbeat || travis_terminate 0 - env: TARGETS="-C heartbeat testsuite" + env: + - PROJECT=heartbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test - os: osx before_install: .ci/scripts/travis_has_changes.sh heartbeat libbeat || travis_terminate 0 - env: TARGETS="TEST_ENVIRONMENT=0 -C heartbeat testsuite" + env: + - PROJECT=heartbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test # Auditbeat - os: linux before_install: .ci/scripts/travis_has_changes.sh auditbeat libbeat || travis_terminate 0 - env: TARGETS="-C auditbeat testsuite" + env: + - PROJECT=auditbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test - os: osx before_install: .ci/scripts/travis_has_changes.sh auditbeat libbeat || travis_terminate 0 - env: TARGETS="TEST_ENVIRONMENT=0 -C auditbeat testsuite" + env: + - PROJECT=auditbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux @@ -82,14 +96,27 @@ jobs: stage: test - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/auditbeat auditbeat libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/auditbeat testsuite" + env: + - PROJECT=x-pack/auditbeat + - MAGE='build test' + go: $TRAVIS_GO_VERSION + stage: test + - os: osx + before_install: .ci/scripts/travis_has_changes.sh x-pack/auditbeat auditbeat libbeat || travis_terminate 0 + env: + - PROJECT=x-pack/auditbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test # Libbeat - os: linux before_install: .ci/scripts/travis_has_changes.sh libbeat || travis_terminate 0 - env: TARGETS="-C libbeat testsuite" + env: + - PROJECT=libbeat + - MAGE='build test' + # The libbeat tests are so verbose that they exceed the maximum allowed log length of Travis CI. + - MAGEFILE_VERBOSE=false go: $TRAVIS_GO_VERSION stage: test - os: linux @@ -104,35 +131,44 @@ jobs: stage: test - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/libbeat libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/libbeat testsuite" + env: + - PROJECT=x-pack/libbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test # Metricbeat - os: linux before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0 - env: TARGETS="-C metricbeat unit-tests" + env: + - PROJECT=metricbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0 - install: + install: - .ci/scripts/install-kind.sh - .ci/scripts/install-kubectl.sh env: - - TARGETS="-C metricbeat integration-tests" - K8S_VERSION=v1.17.2 - KIND_VERSION=v0.7.0 + - PROJECT=metricbeat + - MAGE='goIntegTest' go: $TRAVIS_GO_VERSION stage: test - os: linux before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0 - env: TARGETS="-C metricbeat system-tests" + env: + - PROJECT=metricbeat + - MAGE='pythonIntegTest' go: $TRAVIS_GO_VERSION stage: test - os: osx before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0 - env: TARGETS="-C metricbeat testsuite" + env: + - PROJECT=metricbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux @@ -142,29 +178,39 @@ jobs: stage: test - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/metricbeat unit-tests" + env: + - PROJECT=x-pack/metricbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/metricbeat integration-tests" + env: + - PROJECT=x-pack/metricbeat + - MAGE='goIntegTest' go: $TRAVIS_GO_VERSION stage: test - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/metricbeat metricbeat libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/metricbeat system-tests" + env: + - PROJECT=x-pack/metricbeat + - MAGE='pythonIntegTest' go: $TRAVIS_GO_VERSION stage: test - os: osx before_install: .ci/scripts/travis_has_changes.sh metricbeat libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/metricbeat testsuite" + env: + - PROJECT=x-pack/metricbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test # Packetbeat - os: linux before_install: .ci/scripts/travis_has_changes.sh packetbeat libbeat || travis_terminate 0 - env: TARGETS="-C packetbeat testsuite" + env: + - PROJECT=packetbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test @@ -178,43 +224,57 @@ jobs: # Functionbeat - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/functionbeat libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/functionbeat testsuite" + env: + - PROJECT=x-pack/functionbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test - os: osx before_install: .ci/scripts/travis_has_changes.sh x-pack/functionbeat libbeat || travis_terminate 0 - env: TARGETS="TEST_ENVIRONMENT=0 -C x-pack/functionbeat testsuite" + env: + - PROJECT=x-pack/functionbeat + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/functionbeat libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/functionbeat test-gcp-functions" + env: + - PROJECT=x-pack/functionbeat + - MAGE='testGCPFunctions' go: 1.13.1 stage: test # Docker Log Driver - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/dockerlogbeat libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/dockerlogbeat testsuite" + env: + - PROJECT=x-pack/dockerlogbeat + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test # Journalbeat - os: linux before_install: .ci/scripts/travis_has_changes.sh journalbeat libbeat || travis_terminate 0 - env: TARGETS="-C journalbeat testsuite" + env: + - PROJECT=journalbeat + - MAGE='build goUnitTest' go: $TRAVIS_GO_VERSION stage: test # Agent - os: linux before_install: .ci/scripts/travis_has_changes.sh x-pack/elastic-agent libbeat || travis_terminate 0 - env: TARGETS="-C x-pack/elastic-agent testsuite" + env: + - PROJECT=x-pack/elastic-agent + - MAGE='build test' go: $TRAVIS_GO_VERSION stage: test - os: osx before_install: .ci/scripts/travis_has_changes.sh x-pack/elastic-agent libbeat || travis_terminate 0 - env: TARGETS="TEST_ENVIRONMENT=0 -C x-pack/elastic-agent testsuite" + env: + - PROJECT=x-pack/elastic-agent + - MAGE='build unitTest' go: $TRAVIS_GO_VERSION stage: test @@ -318,13 +378,21 @@ before_script: - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - if [ $TRAVIS_OS_NAME = osx ]; then pip install virtualenv==16.7.9; fi + - make mage script: # Replacement for travis_wait which doesn't print output in real time. # Default Travis timeout is 10min, so this workaround prints timestamps every 9min to reset the counter. # Using seconds (540s = 9min) instead of minutes for shell compatibility reasons. - while sleep 540; do echo "=====[ ${SECONDS} seconds still running ]====="; done & - - make $TARGETS + - if [[ -n "$MAGE" ]]; then + echo ">> mage $MAGE from $PROJECT"; + cd "$PROJECT"; + mage $MAGE; + else + echo ">> make $TARGETS"; + make $TARGETS; + fi - kill %1 notifications: diff --git a/Jenkinsfile b/Jenkinsfile index e0f12058de46..23437e93e2e0 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -153,7 +153,7 @@ pipeline { } } steps { - makeTarget(context: "Filebeat oss Linux", target: "-C filebeat testsuite", withModule: true) + mageTarget(context: "Filebeat oss Linux", directory: "filebeat", target: "build test", withModule: true) } } stage('Filebeat x-pack'){ @@ -166,7 +166,7 @@ pipeline { } } steps { - mageTarget(context: "Filebeat x-pack Linux", directory: "x-pack/filebeat", target: "update build test", withModule: true) + mageTarget(context: "Filebeat x-pack Linux", directory: "x-pack/filebeat", target: "build test", withModule: true) } } stage('Filebeat Mac OS X'){ @@ -243,7 +243,7 @@ pipeline { stages { stage('Heartbeat oss'){ steps { - makeTarget(context: "Heartbeat oss Linux", target: "-C heartbeat testsuite") + mageTarget(context: "Heartbeat oss Linux", directory: "heartbeat", target: "build test") } } stage('Heartbeat Mac OS X'){ @@ -289,7 +289,7 @@ pipeline { } } steps { - makeTarget(context: "Auditbeat oss Linux", target: "-C auditbeat testsuite", withModule: true) + mageTarget(context: "Auditbeat oss Linux", directory: "auditbeat", target: "build test") } } stage('Auditbeat crosscompile'){ @@ -387,7 +387,7 @@ pipeline { stages { stage('Libbeat oss'){ steps { - makeTarget(context: "Libbeat oss Linux", target: "-C libbeat testsuite") + mageTarget(context: "Libbeat oss Linux", directory: "libbeat", target: "build test") } } stage('Libbeat crosscompile'){ @@ -395,11 +395,6 @@ pipeline { makeTarget(context: "Libbeat oss crosscompile", target: "-C libbeat crosscompile") } } - stage('Libbeat stress-tests'){ - steps { - makeTarget(context: "Libbeat stress-tests", target: "STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests") - } - } } } stage('Libbeat x-pack'){ @@ -412,7 +407,7 @@ pipeline { } } steps { - makeTarget(context: "Libbeat x-pack Linux", target: "-C x-pack/libbeat testsuite") + mageTarget(context: "Libbeat x-pack Linux", directory: "x-pack/libbeat", target: "build test") } } stage('Metricbeat OSS Unit tests'){ @@ -428,7 +423,7 @@ pipeline { mageTarget(context: "Metricbeat OSS linux/amd64 (unitTest)", directory: "metricbeat", target: "build unitTest") } } - stage('Metricbeat OSS Integration tests'){ + stage('Metricbeat OSS Go Integration tests'){ agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } when { @@ -441,7 +436,7 @@ pipeline { mageTarget(context: "Metricbeat OSS linux/amd64 (goIntegTest)", directory: "metricbeat", target: "goIntegTest", withModule: true) } } - stage('Metricbeat Python integration tests'){ + stage('Metricbeat OSS Python Integration tests'){ agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } when { @@ -559,7 +554,7 @@ pipeline { mageTargetWin(context: "Metricbeat x-pack Windows", directory: "x-pack/metricbeat", target: "build unitTest") } } - stage('Packetbeat'){ + stage('Packetbeat OSS'){ agent { label 'ubuntu && immutable' } options { skipDefaultCheckout() } when { @@ -569,9 +564,40 @@ pipeline { } } stages { - stage('Packetbeat oss'){ + stage('Packetbeat Linux'){ + steps { + mageTarget(context: "Packetbeat OSS Linux", directory: "packetbeat", target: "build test") + } + } + stage('Packetbeat Mac OS X'){ + agent { label 'macosx' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.macosTest + } + } + steps { + mageTarget(context: "Packetbeat OSS Mac OS X", directory: "packetbeat", target: "build unitTest") + } + post { + always { + delete() + } + } + } + stage('Packetbeat Windows'){ + agent { label 'windows-immutable && windows-2019' } + options { skipDefaultCheckout() } + when { + beforeAgent true + expression { + return params.windowsTest + } + } steps { - makeTarget(context: "Packetbeat oss Linux", target: "-C packetbeat testsuite") + mageTargetWin(context: "Packetbeat OSS Windows", directory: "packetbeat", target: "build unitTest") } } } @@ -588,7 +614,7 @@ pipeline { stages { stage('Dockerlogbeat'){ steps { - mageTarget(context: "Elastic Docker Logging Driver Plugin unit tests", directory: "x-pack/dockerlogbeat", target: "update build test") + mageTarget(context: "Elastic Docker Logging Driver Plugin unit tests", directory: "x-pack/dockerlogbeat", target: "build test") } } } @@ -650,7 +676,7 @@ pipeline { steps { mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "update build test") withEnv(["GO_VERSION=1.13.1"]){ - makeTarget(context: "Functionbeat x-pack Linux", target: "-C x-pack/functionbeat test-gcp-functions") + mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "testGCPFunctions") } } } @@ -699,7 +725,7 @@ pipeline { stages { stage('Journalbeat oss'){ steps { - makeTarget(context: "Journalbeat Linux", target: "-C journalbeat testsuite") + mageTarget(context: "Journalbeat Linux", directory: "journalbeat", target: "build goUnitTest") } } } @@ -1296,7 +1322,7 @@ def loadConfigEnvVars(){ } /** - This method gathers the module name, if required, in order to run the ITs only if + This method gathers the module name, if required, in order to run the ITs only if the changeset affects a specific module. For such, it's required to look for changes under the module folder and exclude anything else diff --git a/auditbeat/magefile.go b/auditbeat/magefile.go index 115547619406..73110b173546 100644 --- a/auditbeat/magefile.go +++ b/auditbeat/magefile.go @@ -31,16 +31,18 @@ import ( // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/common" // mage:import - _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" - // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import + "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" + // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" ) func init() { common.RegisterCheckDeps(Update) unittest.RegisterGoTestDeps(fieldsYML) + integtest.RegisterGoTestDeps(fieldsYML) + integtest.RegisterPythonTestDeps(Dashboards) devtools.BeatDescription = "Audit the activities of users and processes on your system." } diff --git a/dev-tools/mage/integtest.go b/dev-tools/mage/integtest.go index 396c010dbc20..16321fa90326 100644 --- a/dev-tools/mage/integtest.go +++ b/dev-tools/mage/integtest.go @@ -213,7 +213,7 @@ func NewIntegrationRunners(path string, passInEnv map[string]string) (Integratio return runners, nil } -// NewDockerIntegrationRunner returns an intergration runner configured only for docker. +// NewDockerIntegrationRunner returns an integration runner configured only for docker. func NewDockerIntegrationRunner(passThroughEnvVars ...string) (*IntegrationRunner, error) { cwd, err := os.Getwd() if err != nil { diff --git a/dev-tools/mage/integtest_docker.go b/dev-tools/mage/integtest_docker.go index afc05a16dc1a..da7ecf33c375 100644 --- a/dev-tools/mage/integtest_docker.go +++ b/dev-tools/mage/integtest_docker.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "os" + "path" "path/filepath" "runtime" "strings" @@ -156,6 +157,13 @@ func (d *DockerIntegrationTester) Test(_ string, mageTarget string, env map[stri func (d *DockerIntegrationTester) InsideTest(test func() error) error { // Fix file permissions after test is done writing files as root. if runtime.GOOS != "windows" { + repo, err := GetProjectRepoInfo() + if err != nil { + return err + } + + // Handle virtualenv and the current project dir. + defer DockerChown(path.Join(repo.RootDir, "build")) defer DockerChown(".") } return test() diff --git a/dev-tools/mage/pytest.go b/dev-tools/mage/pytest.go index 9c15e1f7e5f4..c8a12fdfab62 100644 --- a/dev-tools/mage/pytest.go +++ b/dev-tools/mage/pytest.go @@ -132,9 +132,7 @@ func PythonNoseTest(params PythonTestArgs) error { nosetestsOptions := []string{ "--process-timeout=90", "--with-timer", - } - if mg.Verbose() { - nosetestsOptions = append(nosetestsOptions, "-v") + "-v", } if params.XUnitReportFile != "" { nosetestsOptions = append(nosetestsOptions, @@ -188,7 +186,7 @@ func PythonNoseTestForModule(params PythonTestArgs) error { // PythonVirtualenv constructs a virtualenv that contains the given modules as // defined in the requirements file pointed to by requirementsTxt. It returns -// the path to the virutalenv. +// the path to the virtualenv. func PythonVirtualenv() (string, error) { pythonVirtualenvLock.Lock() defer pythonVirtualenvLock.Unlock() @@ -221,20 +219,28 @@ func PythonVirtualenv() (string, error) { } pip := virtualenvPath(ve, "pip") - args := []string{"install"} - if !mg.Verbose() { - args = append(args, "--quiet") + pipUpgrade := func(pkg string) error { + return sh.RunWith(env, pip, "install", "-U", pkg) } - for _, req := range reqs { - args = append(args, "-Ur", req) + + // Ensure we are using the latest pip version. + if err = pipUpgrade("pip"); err != nil { + fmt.Printf("warn: failed to upgrade pip (ignoring): %v", err) } // First ensure that wheel is installed so that bdists build cleanly. - if err = sh.RunWith(env, pip, "install", "-U", "wheel"); err != nil { + if err = pipUpgrade("wheel"); err != nil { return "", err } // Execute pip to install the dependencies. + args := []string{"install"} + if !mg.Verbose() { + args = append(args, "--quiet") + } + for _, req := range reqs { + args = append(args, "-Ur", req) + } if err := sh.RunWith(env, pip, args...); err != nil { return "", err } diff --git a/filebeat/tests/system/test_input.py b/filebeat/tests/system/test_input.py index 8aebe49f4d0a..7192fd956d54 100644 --- a/filebeat/tests/system/test_input.py +++ b/filebeat/tests/system/test_input.py @@ -713,6 +713,7 @@ def test_path_based_identity_tracking(self): # assert that renaming of the file went undetected assert not self.log_contains("File rename was detected:" + testfile + " -> " + renamedfile) + @unittest.skip("Skipped as flaky: https://github.com/elastic/beats/issues/20010") @unittest.skipIf(sys.platform.startswith("win"), "inode_marker is not supported on windows") def test_inode_marker_based_identity_tracking(self): """ diff --git a/heartbeat/magefile.go b/heartbeat/magefile.go index 298ee0443d1a..6d2bbf8c2f34 100644 --- a/heartbeat/magefile.go +++ b/heartbeat/magefile.go @@ -33,10 +33,10 @@ import ( // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/common" // mage:import - "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" - // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import + "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" + // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" ) diff --git a/libbeat/tests/system/test_http.py b/libbeat/tests/system/test_http.py index c2379676da59..5c1baa81bd22 100644 --- a/libbeat/tests/system/test_http.py +++ b/libbeat/tests/system/test_http.py @@ -24,7 +24,7 @@ def test_root(self): r = requests.get("http://localhost:5066") assert r.status_code == 200 - data = json.loads(r.content) + data = json.loads(r.content.decode('utf_8')) assert data["beat"] == "mockbeat" assert data["version"] == "9.9.9" @@ -36,7 +36,7 @@ def test_stats(self): r = requests.get("http://localhost:5066/stats") assert r.status_code == 200 - data = json.loads(r.content) + data = json.loads(r.content.decode('utf_8')) # Test one data point assert data["libbeat"]["config"]["scans"] == 0 diff --git a/magefile.go b/magefile.go index ac4614e2cdee..75c6d60faef6 100644 --- a/magefile.go +++ b/magefile.go @@ -135,7 +135,6 @@ func CheckLicenseHeaders() error { licenser.Exclude("x-pack"), licenser.Exclude("generator/_templates/beat/{beat}"), licenser.Exclude("generator/_templates/metricbeat/{beat}"), - licenser.Exclude("generator/_templates/beat/{beat}"), ), licenser( licenser.Check(), diff --git a/packetbeat/magefile.go b/packetbeat/magefile.go index 00622696bc48..fd00c932ab46 100644 --- a/packetbeat/magefile.go +++ b/packetbeat/magefile.go @@ -35,7 +35,7 @@ import ( // mage:import "github.com/elastic/beats/v7/dev-tools/mage/target/common" // mage:import - _ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" + "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest/notests" // mage:import @@ -44,6 +44,7 @@ import ( func init() { common.RegisterCheckDeps(Update) + unittest.RegisterPythonTestDeps(fieldsYML, Dashboards) devtools.BeatDescription = "Packetbeat analyzes network traffic and sends the data to Elasticsearch." } diff --git a/packetbeat/tests/system/packetbeat.py b/packetbeat/tests/system/packetbeat.py index 54d7404d928d..2243d433c4c9 100644 --- a/packetbeat/tests/system/packetbeat.py +++ b/packetbeat/tests/system/packetbeat.py @@ -126,7 +126,7 @@ def read_output(self, types=None, required_fields=None): jsons = [] - with open(os.path.join(self.working_dir, output_file), "r") as f: + with open(os.path.join(self.working_dir, output_file), "r", encoding='utf_8') as f: for line in f: document = self.flatten_object(json.loads(line), self.dict_fields) if not types or document["type"] in types: diff --git a/x-pack/libbeat/magefile.go b/x-pack/libbeat/magefile.go index 01e9b4d7f2d4..2b8d8f35d8ee 100644 --- a/x-pack/libbeat/magefile.go +++ b/x-pack/libbeat/magefile.go @@ -12,10 +12,10 @@ import ( // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/common" // mage:import - _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" - // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest" // mage:import + _ "github.com/elastic/beats/v7/dev-tools/mage/target/integtest" + // mage:import _ "github.com/elastic/beats/v7/dev-tools/mage/target/test" )