Add basic "import-beats" scripts

dev/import-beats/main.go

@@ -0,0 +1,55 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package main
+
+import (
+	"flag"
+	"log"
+
+	"github.com/pkg/errors"
+)
+
+func main() {
+	// Beats repository directory
+	var beatsDir string
+	// Target public directory where the generated packages should end up in
+	var outputDir string
+
+	flag.StringVar(&beatsDir, "beatsDir", "", "Path to the beats repository")
+	flag.StringVar(&outputDir, "outputDir", "", "Path to the output directory ")
+	flag.Parse()
+
+	if beatsDir == "" || outputDir == "" {
+		log.Fatal("beatsDir and outputDir must be set")
+	}
+
+	if err := build(beatsDir, outputDir); err != nil {
+		log.Fatal(err)
+	}
+}
+
+// build method visits all beats in beatsDir to collect configuration data for modules.
+// The package-registry groups integrations per target product not per module type. It's opposite to the beats project,
+// where logs and metrics are distributed with different beats (oriented either on logs or metrics - metricbeat,
+// filebeat, etc.).
+func build(beatsDir, outputDir string) error {
+	packages := packageMap{}
+
+	for _, beatName := range logSources {
+		err := packages.loadFromSource(beatsDir, beatName, "logs")
+		if err != nil {
+			return errors.Wrap(err, "loading logs source failed")
+		}
+	}
+
+	for _, beatName := range metricSources {
+		err := packages.loadFromSource(beatsDir, beatName, "metrics")
+		if err != nil {
+			return errors.Wrap(err, "loading metrics source failed")
+		}
+	}
+
+	return packages.writePackages(outputDir)
+}

dev/import-beats/packages.go

@@ -0,0 +1,73 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package main
+
+import (
+	"io/ioutil"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+
+	"github.com/elastic/package-registry/util"
+)
+
+type packageMap map[string]*util.Package
+
+func (p packageMap) loadFromSource(beatsDir, beatName, packageType string) error {
+	path := filepath.Join(beatsDir, beatName, "module")
+	moduleDirs, err := ioutil.ReadDir(path)
+	if err != nil {
+		return errors.Wrapf(err, "cannot read directory '%s'", path)
+	}
+
+	for _, moduleDir := range moduleDirs {
+		if !moduleDir.IsDir() {
+			continue
+		}
+
+		log.Printf("Visit '%s:%s'\n", beatName, moduleDir.Name())
+
+		_, ok := p[moduleDir.Name()]
+		if !ok {
+			p[moduleDir.Name()] = &util.Package{
+				FormatVersion: "1.0.0",
+				Name:          moduleDir.Name(),
+				Version:       "0.0.1", // TODO
+				Type:          "integration",
+				Categories:    []string{},
+			}
+		}
+
+		p[moduleDir.Name()].Categories = append(p[moduleDir.Name()].Categories, packageType)
+	}
+	return nil
+}
+
+func (p packageMap) writePackages(outputDir string) error {
+	for name, content := range p {
+		log.Printf("Writing package '%s' (version: %s)\n", name, content.Version)
+
+		path := filepath.Join(outputDir, name+"-"+content.Version)
+		err := os.MkdirAll(path, 0755)
+		if err != nil {
+			return errors.Wrapf(err, "cannot make directory '%s'", path)
+		}
+
+		m, err := yaml.Marshal(content)
+		if err != nil {
+			return errors.Wrapf(err, "marshaling package content failed (package name: %s)", name)
+		}
+
+		manifestFilePath := filepath.Join(path, "manifest.yml")
+		err = ioutil.WriteFile(manifestFilePath, m, 0644)
+		if err != nil {
+			return errors.Wrapf(err, "writing manifest file failed (path: %s)", manifestFilePath)
+		}
+	}
+	return nil
+}

dev/import-beats/sources.go

@@ -0,0 +1,15 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package main
+
+var metricSources = []string{
+	"metricbeat",
+	"x-pack/metricbeat",
+}
+
+var logSources = []string{
+	"filebeat",
+	"x-pack/filebeat",
+}

dev/package-beats/activemq-0.0.1/manifest.yml

@@ -0,0 +1,11 @@
+format_version: 1.0.0
+name: activemq
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- logs
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/aerospike-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: aerospike
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/apache-0.0.1/manifest.yml

@@ -0,0 +1,11 @@
+format_version: 1.0.0
+name: apache
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- logs
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/apache2-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: apache2
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- logs
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/appsearch-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: appsearch
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/auditd-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: auditd
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- logs
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/aws-0.0.1/manifest.yml

@@ -0,0 +1,11 @@
+format_version: 1.0.0
+name: aws
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- logs
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/azure-0.0.1/manifest.yml

@@ -0,0 +1,11 @@
+format_version: 1.0.0
+name: azure
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- logs
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/beat-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: beat
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/cef-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: cef
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- logs
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/ceph-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: ceph
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/cisco-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: cisco
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- logs
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/cloudfoundry-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: cloudfoundry
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/cockroachdb-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: cockroachdb
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/consul-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: consul
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/coredns-0.0.1/manifest.yml

@@ -0,0 +1,11 @@
+format_version: 1.0.0
+name: coredns
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- logs
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/couchbase-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: couchbase
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/couchdb-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: couchdb
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/docker-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: docker
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/dropwizard-0.0.1/manifest.yml

@@ -0,0 +1,10 @@
+format_version: 1.0.0
+name: dropwizard
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/elasticsearch-0.0.1/manifest.yml

@@ -0,0 +1,11 @@
+format_version: 1.0.0
+name: elasticsearch
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- logs
+- metrics
+requirement:
+  kibana:
+    versions: ""

dev/package-beats/envoyproxy-0.0.1/manifest.yml

@@ -0,0 +1,11 @@
+format_version: 1.0.0
+name: envoyproxy
+version: 0.0.1
+description: ""
+type: integration
+categories:
+- logs
+- metrics
+requirement:
+  kibana:
+    versions: ""