From 2ba7161fc52d66c14e2f999667155895696bb955 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Tue, 24 Aug 2021 10:23:38 +0200 Subject: [PATCH] feat: add a mage command to package system tests (#27295) (#27551) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: add a mage command to package system tests * chore: delegate prints to the caller * chore: do not keep writing tar file in memory * chore: update comment Co-authored-by: Jaime Soriano Pastor * chore: handle error while traversing the filesystem Co-authored-by: Jaime Soriano Pastor (cherry picked from commit 7f086e554a307c8c406a769e8a2133de325dfb2e) Co-authored-by: Manuel de la Peña --- .ci/scripts/search_system_tests.py | 10 ---- Jenkinsfile | 25 +++++--- dev-tools/mage/common.go | 67 +++++++++++++++++++++ dev-tools/mage/target/common/package.go | 78 +++++++++++++++++++++++++ 4 files changed, 162 insertions(+), 18 deletions(-) delete mode 100755 .ci/scripts/search_system_tests.py create mode 100644 dev-tools/mage/target/common/package.go diff --git a/.ci/scripts/search_system_tests.py b/.ci/scripts/search_system_tests.py deleted file mode 100755 index 0e3896d9ff4c..000000000000 --- a/.ci/scripts/search_system_tests.py +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env python3 - -import os - - -if __name__ == "__main__": - - for root, dirs, files in os.walk('build'): - if root.endswith(('system-tests')): - print(root.replace(".{}".format(os.sep), '')) diff --git a/Jenkinsfile b/Jenkinsfile index 1d826ff32c9b..47fb6f603e81 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -668,7 +668,7 @@ def withBeatsEnv(Map args = [:], Closure body) { error("Error '${err.toString()}'") } finally { if (archive) { - archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) + archiveTestOutput(directory: directory, testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) } tearDown() } @@ -766,6 +766,8 @@ def getCommonModuleInTheChangeSet(String directory) { * to bypass some issues when working with big repositories. */ def archiveTestOutput(Map args = [:]) { + def directory = args.get('directory', '') + catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') { if (isUnix()) { fixPermissions("${WORKSPACE}") @@ -789,13 +791,20 @@ def archiveTestOutput(Map args = [:]) { } if (args.upload) { catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') { - def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim() - log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball") - if (folder.trim()) { - // TODO: nodeOS() should support ARM - def os_suffix = isArm() ? 'linux' : nodeOS() - def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix - tarAndUploadArtifacts(file: "${name}.tgz", location: folder) + withMageEnv(version: "${env.GO_VERSION}"){ + dir(directory){ + cmd(label: "Archive system tests files", script: 'mage packageSystemTests') + } + } + def fileName = 'build/system-tests-*.tar.gz' // see dev-tools/mage/target/common/package.go#PackageSystemTests method + dir("${BASE_DIR}"){ + cmd(label: "List files to upload", script: "ls -l ${BASE_DIR}/${fileName}") + googleStorageUploadExt( + bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}", + credentialsId: "${JOB_GCS_EXT_CREDENTIALS}", + pattern: "${BASE_DIR}/${fileName}", + sharedPublicly: true + ) } } } diff --git a/dev-tools/mage/common.go b/dev-tools/mage/common.go index dd0a2fd56c2c..208ae02d974c 100644 --- a/dev-tools/mage/common.go +++ b/dev-tools/mage/common.go @@ -353,6 +353,73 @@ func unzip(sourceFile, destinationDir string) error { return nil } +// Tar compress a directory using tar + gzip algorithms +func Tar(src string, targetFile string) error { + fmt.Printf(">> creating TAR file from directory: %s, target: %s\n", src, targetFile) + + f, err := os.Create(targetFile) + if err != nil { + return fmt.Errorf("error creating tar file: %w", err) + } + defer f.Close() + + // tar > gzip > file + zr := gzip.NewWriter(f) + tw := tar.NewWriter(zr) + + // walk through every file in the folder + filepath.Walk(src, func(file string, fi os.FileInfo, errFn error) error { + if errFn != nil { + return fmt.Errorf("error traversing the file system: %w", errFn) + } + + // if a symlink, skip file + if fi.Mode().Type() == os.ModeSymlink { + fmt.Printf(">> skipping symlink: %s\n", file) + return nil + } + + // generate tar header + header, err := tar.FileInfoHeader(fi, file) + if err != nil { + return fmt.Errorf("error getting file info header: %w", err) + } + + // must provide real name + // (see https://golang.org/src/archive/tar/common.go?#L626) + header.Name = filepath.ToSlash(file) + + // write header + if err := tw.WriteHeader(header); err != nil { + return fmt.Errorf("error writing header: %w", err) + } + + // if not a dir, write file content + if !fi.IsDir() { + data, err := os.Open(file) + if err != nil { + return fmt.Errorf("error opening file: %w", err) + } + defer data.Close() + if _, err := io.Copy(tw, data); err != nil { + return fmt.Errorf("error compressing file: %w", err) + } + } + return nil + }) + + // produce tar + if err := tw.Close(); err != nil { + return fmt.Errorf("error closing tar file: %w", err) + } + // produce gzip + if err := zr.Close(); err != nil { + return fmt.Errorf("error closing gzip file: %w", err) + } + + return nil +} + func untar(sourceFile, destinationDir string) error { file, err := os.Open(sourceFile) if err != nil { diff --git a/dev-tools/mage/target/common/package.go b/dev-tools/mage/target/common/package.go new file mode 100644 index 000000000000..e8849adb26d2 --- /dev/null +++ b/dev-tools/mage/target/common/package.go @@ -0,0 +1,78 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package common + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + devtools "github.com/elastic/beats/v7/dev-tools/mage" +) + +// PackageSystemTests packages the python system tests results +func PackageSystemTests() error { + excludeds := []string{".ci", ".git", ".github", "vendor", "dev-tools"} + + // include run as it's the directory we want to compress + systemTestsDir := filepath.Join("build", "system-tests", "run") + files, err := devtools.FindFilesRecursive(func(path string, _ os.FileInfo) bool { + base := filepath.Base(path) + for _, excluded := range excludeds { + if strings.HasPrefix(base, excluded) { + return false + } + } + + return strings.HasPrefix(path, systemTestsDir) + }) + if err != nil { + return err + } + + if len(files) == 0 { + fmt.Printf(">> there are no system test files under %s", systemTestsDir) + return nil + } + + // create a plain directory layout for all beats + beat := devtools.MustExpand("{{ repo.SubDir }}") + beat = strings.ReplaceAll(beat, string(os.PathSeparator), "-") + + targetFile := devtools.MustExpand("{{ elastic_beats_dir }}/build/system-tests-" + beat + ".tar.gz") + parent := filepath.Dir(targetFile) + if !fileExists(parent) { + fmt.Printf(">> creating parent dir: %s", parent) + os.Mkdir(parent, 0750) + } + + err = devtools.Tar(systemTestsDir, targetFile) + if err != nil { + fmt.Printf(">> %s", err) + return err + } + + return nil +} + +// fileExists returns true if the specified file exists. +func fileExists(file string) bool { + _, err := os.Stat(file) + return !os.IsNotExist(err) +}