merge master

.ci/Jenkinsfile

@@ -14,6 +14,7 @@ pipeline {
     KIND_VERSION = "v0.11.1"
     K8S_VERSION = "v1.20.2"
     JOB_GCS_BUCKET = 'beats-ci-temp'
+    JOB_GCS_BUCKET_INTERNAL = 'beats-ci-temp-internal'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
     JOB_GCS_EXT_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials'
     STACK_VERSION = "${params.stackVersion}"
@@ -44,8 +45,8 @@ pipeline {
     }
     stage('Check Go sources') {
       steps {
-        withMageEnv(){
-          dir("${BASE_DIR}"){
+        dir("${BASE_DIR}"){
+          withMageEnv(){
             sh(label: 'Checks and builds Go sources', script: 'mage -debug check')
             checkGitDiff()
           }
@@ -95,7 +96,8 @@ pipeline {
                           junit(allowEmptyResults: true, keepLongStdio: true, testResults: "build/test-results/*.xml")
                           sh(label: "Collect Elastic stack logs", script: "build/elastic-package stack dump -v --output build/elastic-stack-dump/${it}")
                           archiveArtifacts(allowEmptyArchive: true, artifacts: "build/elastic-stack-dump/${it}/logs/*.log, build/elastic-stack-dump/${it}/logs/fleet-server-internal/*")
-                          archiveArtifactsSafe(it, "build/elastic-stack-dump/${it}/logs/elastic-agent-internal/*")
+                          archiveArtifactsSafe("insecure-logs/${it}", "build/elastic-stack-dump/${it}/logs/elastic-agent-internal/*")
+                          archiveArtifactsSafe("insecure-logs/${it}/container-logs", "build/container-logs/*.log")
                           sh(label: "Take down the Elastic stack", script: 'build/elastic-package stack down -v')
                           stashCoverageReport()
                         }
@@ -271,15 +273,15 @@ def getCoveragePathPrefix() {
   return "${env.JOB_NAME}-${env.BUILD_ID}/test-coverage/"
 }
 
-def archiveArtifactsSafe(integrationName, artifacts) {
+def archiveArtifactsSafe(remotePath, artifacts) {
   r = sh(label: "areArtifactsPresent", script: "ls ${artifacts}", returnStatus: true)
   if (r != 0) {
     echo "areArtifactsPresent: artifacts files not found, nothing will be archived"
     return
   }
 
   googleStorageUploadExt(
-    bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}/insecure-logs/${integrationName}",
+    bucket: "gs://${JOB_GCS_BUCKET_INTERNAL}/${env.JOB_NAME}-${env.BUILD_ID}/${remotePath}",
     credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
     pattern: artifacts)
-}
+}

.ci/schedule-daily.groovy

@@ -5,6 +5,8 @@ pipeline {
   environment {
     NOTIFY_TO = credentials('notify-to')
     PIPELINE_LOG_LEVEL = 'INFO'
+    SLACK_CHANNEL = "#beats-build"
+    INTEGRATION_JOB = 'Ingest-manager/integrations/master'
   }
   options {
     timeout(time: 2, unit: 'HOURS')
@@ -23,18 +25,18 @@ pipeline {
     stage('Daily integration builds') {
       steps {
         build(
-          job: 'Ingest-manager/integrations/master',
+          job: env.INTEGRATION_JOB,
           parameters: [stringParam(name: 'stackVersion', value: '7.x-SNAPSHOT')],
           quietPeriod: 0,
-          wait: false,
-          propagate: false,
+          wait: true,
+          propagate: true,
         )
       }
     }
   }
   post {
     cleanup {
-      notifyBuildResult(prComment: false)
+      notifyBuildResult(prComment: false, slackHeader: "Integration job failed ${env.JENKINS_URL}search/?q=${env.INTEGRATION_JOB.replaceAll('/','+')}")
     }
   }
 }

.go-version

@@ -0,0 +1 @@
+1.17.1

dev/import-beats/agent.go

@@ -9,7 +9,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/elastic/package-registry/util"
+	"github.com/elastic/package-registry/packages"
 )
 
 type agentContent struct {
@@ -26,7 +26,7 @@ func extractInputConfigFilename(configFilePath string) string {
 	return configFilePath[i+1:]
 }
 
-func createAgentContentForMetrics(moduleName, dataStreamName string, streams []util.Stream) (agentContent, error) {
+func createAgentContentForMetrics(moduleName, dataStreamName string, streams []packages.Stream) (agentContent, error) {
 	inputName := moduleName + "/metrics"
 	vars := extractVarsFromStream(streams, inputName)
 
@@ -60,11 +60,11 @@ func createAgentContentForMetrics(moduleName, dataStreamName string, streams []u
 	}, nil
 }
 
-func extractVarsFromStream(streams []util.Stream, inputName string) []util.Variable {
+func extractVarsFromStream(streams []packages.Stream, inputName string) []packages.Variable {
 	for _, stream := range streams {
 		if stream.Input == inputName {
 			return stream.Vars
 		}
 	}
-	return []util.Variable{}
+	return []packages.Variable{}
 }

dev/import-beats/conditions.go

@@ -7,13 +7,13 @@ package main
 import (
 	"github.com/blang/semver"
 
-	"github.com/elastic/package-registry/util"
+	"github.com/elastic/package-registry/packages"
 )
 
 var zeroVersion = semver.MustParse("0.0.0")
 
-func createConditions() *util.Conditions {
-	return &util.Conditions{
-		KibanaVersion: "^7.9.0",
+func createConditions() *packages.Conditions {
+	return &packages.Conditions{
+		Kibana: &packages.KibanaConditions{Version: "^7.15.0"},
 	}
 }

dev/import-beats/datasets.go

@@ -13,14 +13,14 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/elastic/package-registry/util"
+	"github.com/elastic/package-registry/packages"
 )
 
 type dataStreamContent struct {
 	name     string
 	beatType string
 
-	manifest util.DataStream
+	manifest packages.DataStream
 
 	agent         agentContent
 	elasticsearch elasticsearchContent
@@ -119,7 +119,7 @@ func createDataStreams(beatType, modulePath, moduleName, moduleTitle string, mod
 		}
 
 		// manifest
-		manifest := util.DataStream{
+		manifest := packages.DataStream{
 			Title:   fmt.Sprintf("%s %s %s", moduleTitle, dataStreamName, beatType),
 			Release: "experimental",
 			Type:    beatType,

dev/import-beats/icons.go

@@ -98,12 +98,11 @@ func retrieveIconPathFromTutorials(kibanaDir string) (map[string]string, error)
 
 		// Extracting module name from tutorials path
 		// e.g. ./src/plugins/home/server/tutorials//php_fpm_metrics/index.ts -> php_fpm
-		moduleName := tutorialFilePath[len(tutorialsPath)+1:]
-		moduleName = moduleName[:strings.Index(moduleName, "/")]
+		moduleName := filepath.Base(filepath.Dir(tutorialFilePath))
 		moduleName = moduleName[:strings.LastIndex(moduleName, "_")]
 
-		if val[0] == '/' {
-			iconFileName := val[strings.LastIndex(val, "/")+1:]
+		if filepath.IsAbs(val) {
+			iconFileName := filepath.Base(val)
 			val = path.Join(kibanaDir, kibanaLogosPath, iconFileName)
 			refs[moduleName] = val
 		}

dev/import-beats/images.go

@@ -18,7 +18,7 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/elastic/package-registry/util"
+	"github.com/elastic/package-registry/packages"
 )
 
 var (
@@ -91,8 +91,8 @@ func extractImages(beatDocsPath string, docsFile []byte) []imageContent {
 	return contents
 }
 
-func (images imageContentArray) toManifestImages() ([]util.Image, error) {
-	var imgs []util.Image
+func (images imageContentArray) toManifestImages() ([]packages.Image, error) {
+	var imgs []packages.Image
 	for _, image := range images {
 		i := strings.LastIndex(image.source, "/")
 		sourceFileName := image.source[i:]
@@ -107,7 +107,7 @@ func (images imageContentArray) toManifestImages() ([]util.Image, error) {
 			return nil, errors.Wrapf(err, "extracting image type failed")
 		}
 
-		imgs = append(imgs, util.Image{
+		imgs = append(imgs, packages.Image{
 			Src:   path.Join("/img", sourceFileName),
 			Title: toImageTitle(sourceFileName),
 			Size:  imageSize,

dev/import-beats/kibana.go

@@ -12,6 +12,7 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -170,6 +171,10 @@ func createKibanaContent(kibanaMigrator *kibanaMigrator, modulePath string, modu
 	kibanaPath := filepath.Join(modulePath, "_meta", "kibana", "7")
 	dashboardIDMap := make(map[string]string, 0)
 	err := filepath.Walk(kibanaPath, func(path string, info fs.FileInfo, err error) error {
+		if errors.Is(err, os.ErrNotExist) {
+			return nil // elasticsearch module: _meta/kibana/7 doesn't exist
+		}
+
 		if info.IsDir() {
 			return nil
 		}

dev/import-beats/packages.go

@@ -18,13 +18,13 @@ import (
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v2"
 
-	"github.com/elastic/package-registry/util"
+	"github.com/elastic/package-registry/packages"
 )
 
 var ignoredModules = map[string]bool{"apache2": true}
 
 type packageContent struct {
-	manifest       util.Package
+	manifest       packages.Package
 	dataStreams    dataStreamContentArray
 	images         []imageContent
 	kibana         kibanaContent
@@ -34,18 +34,18 @@ type packageContent struct {
 
 func newPackageContent(name string) packageContent {
 	return packageContent{
-		manifest: util.Package{
+		manifest: packages.Package{
 			FormatVersion: "1.0.0",
-			BasePackage: util.BasePackage{
+			BasePackage: packages.BasePackage{
 				Name:    name,
 				Version: "0.0.1", // TODO
 				Type:    "integration",
 				Release: "experimental",
+				Owner: &packages.Owner{
+					Github: "elastic/integrations",
+				},
 			},
 			License: "basic",
-			Owner: &util.Owner{
-				Github: "elastic/integrations",
-			},
 		},
 		kibana: kibanaContent{
 			files: map[string]map[string][]byte{},
@@ -325,7 +325,7 @@ func (r *packageRepository) save(outputDir string) error {
 
 			// dataStream/elasticsearch
 			if len(dataStream.elasticsearch.ingestPipelines) > 0 {
-				ingestPipelinesPath := filepath.Join(dataStreamPath, "elasticsearch", util.DirIngestPipeline)
+				ingestPipelinesPath := filepath.Join(dataStreamPath, "elasticsearch", packages.DirIngestPipeline)
 				err := os.MkdirAll(ingestPipelinesPath, 0755)
 				if err != nil {
 					return errors.Wrapf(err, "cannot make directory for data stream ingest pipelines: '%s'", ingestPipelinesPath)

dev/import-beats/policy_templates.go

@@ -9,7 +9,7 @@ import (
 	"sort"
 	"strings"
 
-	"github.com/elastic/package-registry/util"
+	"github.com/elastic/package-registry/packages"
 )
 
 type policyTemplateContent struct {
@@ -23,10 +23,10 @@ type policyTemplateInput struct {
 	dataStreamNames []string
 	packageType     string
 	inputType       string
-	vars            []util.Variable
+	vars            []packages.Variable
 }
 
-func (ptc policyTemplateContent) toMetadataPolicyTemplates() []util.PolicyTemplate {
+func (ptc policyTemplateContent) toMetadataPolicyTemplates() []packages.PolicyTemplate {
 	var inputTypes []string
 	var packageTypes []string
 	for k, input := range ptc.inputs {
@@ -46,11 +46,11 @@ func (ptc policyTemplateContent) toMetadataPolicyTemplates() []util.PolicyTempla
 		description = toPolicyTemplateDescription(ptc.moduleTitle, packageTypes[0])
 	}
 
-	var inputs []util.Input
+	var inputs []packages.Input
 	for _, packageType := range packageTypes {
 		for inputType, input := range ptc.inputs {
 			if input.packageType == packageType {
-				inputs = append(inputs, util.Input{
+				inputs = append(inputs, packages.Input{
 					Type:        input.inputType,
 					Title:       toPolicyTemplateInputTitle(ptc.moduleTitle, packageType, ptc.inputs[inputType].dataStreamNames, inputType),
 					Description: toPolicyTemplateInputDescription(ptc.moduleTitle, packageType, ptc.inputs[inputType].dataStreamNames, inputType),
@@ -59,7 +59,7 @@ func (ptc policyTemplateContent) toMetadataPolicyTemplates() []util.PolicyTempla
 			}
 		}
 	}
-	return []util.PolicyTemplate{
+	return []packages.PolicyTemplate{
 		{
 			Name:        ptc.moduleName,
 			Title:       title,
@@ -75,7 +75,7 @@ type updatePolicyTemplateParameters struct {
 	packageType string
 
 	dataStreams dataStreamContentArray
-	inputVars   map[string][]util.Variable
+	inputVars   map[string][]packages.Variable
 }
 
 func updatePolicyTemplate(dsc policyTemplateContent, params updatePolicyTemplateParameters) (policyTemplateContent, error) {

dev/import-beats/streams.go

@@ -14,14 +14,14 @@ import (
 
 	"github.com/pkg/errors"
 
-	"github.com/elastic/package-registry/util"
+	"github.com/elastic/package-registry/packages"
 )
 
 // createStreams method builds a set of stream inputs including configuration variables.
 // Stream definitions depend on a beat type - log or metric.
 // At the moment, the array returns only one stream.
-func createStreams(modulePath, moduleName, moduleTitle, dataStreamName, beatType string) ([]util.Stream, agentContent, error) {
-	var streams []util.Stream
+func createStreams(modulePath, moduleName, moduleTitle, dataStreamName, beatType string) ([]packages.Stream, agentContent, error) {
+	var streams []packages.Stream
 	var agent agentContent
 	var err error
 
@@ -46,7 +46,7 @@ func createStreams(modulePath, moduleName, moduleTitle, dataStreamName, beatType
 
 // createLogStreams method builds a set of stream inputs for logs oriented dataStream.
 // The method unmarshals "manifest.yml" file and picks all configuration variables.
-func createLogStreams(modulePath, moduleTitle, dataStreamName string) ([]util.Stream, agentContent, error) {
+func createLogStreams(modulePath, moduleTitle, dataStreamName string) ([]packages.Stream, agentContent, error) {
 	manifestPath := filepath.Join(modulePath, dataStreamName, "manifest.yml")
 	manifestFile, err := ioutil.ReadFile(manifestPath)
 	if err != nil {
@@ -67,7 +67,7 @@ func createLogStreams(modulePath, moduleTitle, dataStreamName string) ([]util.St
 		return nil, agentContent{}, fmt.Errorf("expected at least one config file (modulePath: %s, dataStreamName: %s)", modulePath, dataStreamName)
 	}
 
-	var streams []util.Stream
+	var streams []packages.Stream
 	var agent agentContent
 	for _, configFilePath := range configFilePaths {
 		fileName := extractInputConfigFilename(configFilePath)
@@ -102,7 +102,7 @@ func createLogStreams(modulePath, moduleTitle, dataStreamName string) ([]util.St
 				body:           inputConfig,
 			})
 
-			streams = append(streams, util.Stream{
+			streams = append(streams, packages.Stream{
 				Input:        aType,
 				Title:        fmt.Sprintf("%s %s logs (%s)", moduleTitle, dataStreamName, inputType),
 				Description:  fmt.Sprintf("Collect %s %s logs using %s input", moduleTitle, dataStreamName, inputType),
@@ -120,7 +120,7 @@ func createLogStreams(modulePath, moduleTitle, dataStreamName string) ([]util.St
 //
 // The method skips commented variables, but keeps arrays of structures (even if it's not possible to render them using
 // UI).
-func createMetricStreams(modulePath, moduleName, moduleTitle, dataStreamName string) ([]util.Stream, agentContent, error) {
+func createMetricStreams(modulePath, moduleName, moduleTitle, dataStreamName string) ([]packages.Stream, agentContent, error) {
 	merged, err := mergeMetaConfigFiles(modulePath)
 	if err != nil {
 		return nil, agentContent{}, errors.Wrapf(err, "merging config files failed")
@@ -130,7 +130,7 @@ func createMetricStreams(modulePath, moduleName, moduleTitle, dataStreamName str
 	if err != nil {
 		return nil, agentContent{}, errors.Wrapf(err, "creating metric stream variables failed (modulePath: %s)", modulePath)
 	}
-	streams := []util.Stream{
+	streams := []packages.Stream{
 		{
 			Input:       moduleName + "/metrics",
 			Title:       fmt.Sprintf("%s %s metrics", moduleTitle, dataStreamName),