apmpackage: remove version directory (#5414)

* apmpackage: remove version directory

The version directory should be added when copying
to package-storage, but it is not needed in this repo.
The version is maintained in manifest.yml.

* apmpackage: update package version to 0.3.0

* Update apmpackage instructions and script

* Makefile: update *-package targets

* add docker entrypoint to copy package to am/n.n.n

* apmpackage/cmd/gen-package: no version in path

* make fmt

(cherry picked from commit 1876181)

# Conflicts:
#	apmpackage/cmd/gen-package/main.go

Makefile

@@ -238,11 +238,11 @@ check-docker-compose: $(PYTHON_BIN)
 
 .PHONY: check-package format-package build-package
 check-package: $(ELASTICPACKAGE)
-	@for x in apmpackage/apm/*; do (cd $$x; echo "Checking $$x"; $(CURDIR)/$(ELASTICPACKAGE) check); done
+	@(cd apmpackage/apm; $(CURDIR)/$(ELASTICPACKAGE) check)
 format-package: $(ELASTICPACKAGE)
-	@for x in apmpackage/apm/*; do (cd $$x; echo "Formatting $$x"; $(CURDIR)/$(ELASTICPACKAGE) format); done
+	@(cd apmpackage/apm; $(CURDIR)/$(ELASTICPACKAGE) format)
 build-package: $(ELASTICPACKAGE)
-	@for x in apmpackage/apm/*; do (cd $$x; echo "Building $$x"; $(CURDIR)/$(ELASTICPACKAGE) build); done
+	@(cd apmpackage/apm; $(CURDIR)/$(ELASTICPACKAGE) build)
 
 .PHONY: check-gofmt check-autopep8 gofmt autopep8
 check-fmt: check-gofmt check-autopep8

apmpackage/README.md

@@ -10,12 +10,12 @@
 #### Update / fix a package
 
 1. Actual work
-    - Make changes in `apmpackage/apm/<version>` and/or code as needed
+    - Make changes in `apmpackage/apm` and/or code as needed
     - Run `make update`. That will update fields, pipeline and doc files based on apm-server fields and pipeline defs.
 
 2. Run the registry
     - Checkout a fresh master from the registry and run `mage build`
-    - Copy `apmpackage/apm` in apm-server to `build/package-storage/packages` in the registry
+    - Copy `apmpackage/apm` in apm-server to `build/package-storage/packages/apm/<version>` in the registry
     - `go run .`
 
 3. Run the stack
@@ -34,19 +34,17 @@
 
 5. Upload to the snapshot registry
     - When everything works and `apmpackage/apm/` changes have been merged to `master`, copy the new package to
-    `package-storage/packages` in the `package-storage` repo, `snapshot` branch.
+    `package-storage/packages/apm/<version>` in the `package-storage` repo, `snapshot` branch.
     Do *NOT* override any existing packages. Instead, bump the qualifier version (eg: `0.1.0-dev.1` to `0.1.0-dev.2`)
     both in the folder name and the content (`manifest.yml` and `default.json` pipelines)
-    - You can `cd script && python copy-package.py` for this.
+    - You can `cd script && python copy_package.py` for this.
 
 #### Create a new package version
 
 Follow steps described above, except:
 
 1. New local version
-    - Copy `apmpackage/apm/<latest-version>` to `apmpackage/apm/<next-version>`.
-    - Add a new version mapping in the `versionMapping` variable at the top of `apmpackage/cmd/gen-package/main.go`.
-      Most likely this won't be needed in the future.
+    - Bump the version in `apmpackage/apm/manifest.yml`
     - Then do any changes in the new folder. The rest of the steps are the same.
 
 2. First dev version

apmpackage/apm/changelog.yml

@@ -0,0 +1,24 @@
+# newer versions go on top
+#
+# change type can be one of: enhancement, bugfix, breaking-change
+- version: "0.3.0"
+  changes:
+    - description: added apm-server.url config
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/5332
+    - description: removed apm-server.kibana.api_key config
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/5380
+- version: "0.2.0"
+  changes:
+    - description: added support for apm-server.rum.allow_service_names
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/5030
+    - description: introduce a configurable default service environment
+      type: enhancement
+      link: https://github.com/elastic/apm-server/pull/4861
+- version: "0.1.0"
+  changes:
+    - description: initial release
+      type: enhancement
+      link: https://github.com/elastic/apm-server/

apmpackage/apm/0.3.0/manifest.yml → apmpackage/apm/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: apm
 title: Elastic APM
-version: 0.2.0
+version: 0.3.0
 license: basic
 description: Ingest APM data
 type: integration

apmpackage/cmd/gen-package/gendocs.go

@@ -37,25 +37,25 @@ func escapeReplacer(s ...string) *strings.Replacer {
 
 var markdownReplacer = escapeReplacer("\\", "`", "*", "_")
 
-func generateDocs(inputFields map[string][]field, version string) {
+func generateDocs(inputFields map[string][]field) {
 	data := docsData{
-		Traces:             prepareFields(inputFields, version, "traces"),
-		Metrics:            prepareFields(inputFields, version, "app_metrics"),
-		Logs:               prepareFields(inputFields, version, "error_logs"),
+		Traces:             prepareFields(inputFields, "traces"),
+		Metrics:            prepareFields(inputFields, "app_metrics"),
+		Logs:               prepareFields(inputFields, "error_logs"),
 		TransactionExample: loadExample("transactions.json"),
 		SpanExample:        loadExample("spans.json"),
 		MetricsExample:     loadExample("metricsets.json"),
 		ErrorExample:       loadExample("errors.json"),
 	}
-	t := template.New(docsTemplateFilePath(version))
+	t := template.New(docsTemplateFilePath())
 	tmpl, err := t.Funcs(map[string]interface{}{
 		"Trim":           strings.TrimSpace,
 		"EscapeMarkdown": markdownReplacer.Replace,
-	}).ParseFiles(docsTemplateFilePath(version))
+	}).ParseFiles(docsTemplateFilePath())
 	if err != nil {
 		panic(err)
 	}
-	path := docsFilePath(version)
+	path := docsFilePath()
 	file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
 	if err != nil {
 		panic(err)
@@ -77,10 +77,10 @@ type docsData struct {
 	ErrorExample       string
 }
 
-func prepareFields(inputFields map[string][]field, version, stream string) []field {
+func prepareFields(inputFields map[string][]field, stream string) []field {
 	extend := func(fs []field) []field {
 		var baseFields []field
-		for _, f := range loadFieldsFile(baseFieldsFilePath(version, stream)) {
+		for _, f := range loadFieldsFile(baseFieldsFilePath(stream)) {
 			f.IsECS = true
 			baseFields = append(baseFields, f)
 		}

apmpackage/cmd/gen-package/genfields.go

@@ -30,7 +30,7 @@ import (
 	"github.com/elastic/ecs/code/go/ecs"
 )
 
-func generateFields(version string) map[string][]field {
+func generateFields() map[string][]field {
 
 	ecsFlatFields := loadECSFields()
 
@@ -60,7 +60,7 @@ func generateFields(version string) map[string][]field {
 			if err != nil {
 				panic(err)
 			}
-			err = ioutil.WriteFile(filepath.Join(fieldsPath(version, streamType), fName), bytes, 0644)
+			err = ioutil.WriteFile(filepath.Join(fieldsPath(streamType), fName), bytes, 0644)
 			if err != nil {
 				panic(err)
 			}

apmpackage/cmd/gen-package/genpipelines.go

@@ -61,7 +61,7 @@ func generatePipelines(version, dataStream string) error {
 		return err
 	}
 
-	if err := os.MkdirAll(pipelinesPath(version, dataStream), 0755); err != nil {
+	if err := os.MkdirAll(pipelinesPath(dataStream), 0755); err != nil {
 		return err
 	}
 
@@ -71,7 +71,7 @@ func generatePipelines(version, dataStream string) error {
 			apmPipelineBody = definition.Body
 			continue
 		}
-		fName := filepath.Join(pipelinesPath(version, dataStream), definition.ID+".json")
+		fName := filepath.Join(pipelinesPath(dataStream), definition.ID+".json")
 		if err := writeFilePipelineJSON(fName, definition.Body); err != nil {
 			return err
 		}
@@ -91,7 +91,7 @@ func generatePipelines(version, dataStream string) error {
 		}
 		apmPipelineBody = out
 	}
-	fName := filepath.Join(pipelinesPath(version, dataStream), "default.json")
+	fName := filepath.Join(pipelinesPath(dataStream), "default.json")
 	return writeFilePipelineJSON(fName, apmPipelineBody)
 }
 

apmpackage/cmd/gen-package/main.go

@@ -18,24 +18,13 @@
 package main
 
 import (
-	"errors"
-	"fmt"
 	"io/ioutil"
 	"log"
 	"os"
 
-	"github.com/elastic/apm-server/cmd"
-	"github.com/elastic/beats/v7/libbeat/common"
+	"gopkg.in/yaml.v2"
 )
 
-var versionMapping = map[string]string{
-	"7.11": "0.1.0",
-	"7.12": "0.1.0",
-	"7.13": "0.2.0",
-	"7.14": "0.3.0",
-	"8.0":  "0.3.0",
-}
-
 // Some data streams may not have a counterpart template
 // in standalone apm-server, and so it does not make sense
 // to maintain a separate fields.yml.
@@ -44,29 +33,40 @@ var handwritten = map[string]bool{
 }
 
 func main() {
+<<<<<<< HEAD
 	stackVersion := common.MustNewVersion(cmd.DefaultSettings().Version)
 	shortVersion := fmt.Sprintf("%d.%d", stackVersion.Major, stackVersion.Minor)
 	packageVersion, ok := versionMapping[shortVersion]
 	if !ok {
 		panic(errors.New("integration package can't be generated for current apm-server version"))
+=======
+	manifestData, err := ioutil.ReadFile(manifestFilePath())
+	if err != nil {
+		log.Fatal(err)
+>>>>>>> 18761816 (apmpackage: remove version directory (#5414))
 	}
-	clear(packageVersion)
-	inputFields := generateFields(packageVersion)
+	var manifest struct {
+		Version string `yaml:"version"`
+	}
+	if err := yaml.Unmarshal(manifestData, &manifest); err != nil {
+		log.Fatal(err)
+	}
+
+	clear()
+	inputFields := generateFields()
 	for dataStream := range inputFields {
-		if err := generatePipelines(packageVersion, dataStream); err != nil {
+		if err := generatePipelines(manifest.Version, dataStream); err != nil {
 			log.Fatal(err)
 		}
 	}
 	// TODO(axw) rely on `elastic-package build` to build docs from a template, like in integrations.
-	generateDocs(inputFields, packageVersion)
-	log.Printf("Package fields and docs generated for version %s (stack %s)", packageVersion, stackVersion.String())
+	generateDocs(inputFields)
+	log.Printf("Package fields and docs generated for version %s", manifest.Version)
 }
 
-func clear(version string) {
-	fileInfo, err := ioutil.ReadDir(dataStreamPath(version))
+func clear() {
+	fileInfo, err := ioutil.ReadDir(dataStreamPath())
 	if err != nil {
-		log.Printf("NOTE: if you are adding a new package version, you must create the folder"+
-			" `apmpackage/apm/%s/` and copy all the contents from the previous version.", version)
 		panic(err)
 	}
 	for _, f := range fileInfo {
@@ -77,11 +77,11 @@ func clear(version string) {
 		if handwritten[name] {
 			continue
 		}
-		removeFile(ecsFilePath(version, name))
-		removeFile(fieldsFilePath(version, name))
-		removeDir(pipelinesPath(version, name))
+		removeFile(ecsFilePath(name))
+		removeFile(fieldsFilePath(name))
+		removeDir(pipelinesPath(name))
 	}
-	ioutil.WriteFile(docsFilePath(version), nil, 0644)
+	ioutil.WriteFile(docsFilePath(), nil, 0644)
 }
 
 func removeFile(path string) {

apmpackage/cmd/gen-package/paths.go

@@ -19,34 +19,38 @@ package main
 
 import "path/filepath"
 
-func docsTemplateFilePath(version string) string {
-	return filepath.Join("apmpackage", "apm", version, "README.template.md")
+func manifestFilePath() string {
+	return filepath.Join("apmpackage", "apm", "manifest.yml")
 }
 
-func docsFilePath(version string) string {
-	return filepath.Join("apmpackage", "apm", version, "docs/README.md")
+func docsTemplateFilePath() string {
+	return filepath.Join("apmpackage", "apm", "README.template.md")
 }
 
-func pipelinesPath(version, dataStream string) string {
-	return filepath.Join("apmpackage", "apm", version, "data_stream", dataStream, "elasticsearch", "ingest_pipeline")
+func docsFilePath() string {
+	return filepath.Join("apmpackage", "apm", "docs/README.md")
 }
 
-func dataStreamPath(version string) string {
-	return filepath.Join("apmpackage", "apm", version, "data_stream")
+func pipelinesPath(dataStream string) string {
+	return filepath.Join("apmpackage", "apm", "data_stream", dataStream, "elasticsearch", "ingest_pipeline")
 }
 
-func fieldsPath(version, dataStream string) string {
-	return filepath.Join(dataStreamPath(version), dataStream, "fields")
+func dataStreamPath() string {
+	return filepath.Join("apmpackage", "apm", "data_stream")
 }
 
-func ecsFilePath(version, dataStream string) string {
-	return filepath.Join(fieldsPath(version, dataStream), "ecs.yml")
+func fieldsPath(dataStream string) string {
+	return filepath.Join(dataStreamPath(), dataStream, "fields")
 }
 
-func fieldsFilePath(version, dataStream string) string {
-	return filepath.Join(fieldsPath(version, dataStream), "fields.yml")
+func ecsFilePath(dataStream string) string {
+	return filepath.Join(fieldsPath(dataStream), "ecs.yml")
 }
 
-func baseFieldsFilePath(version, dataStream string) string {
-	return filepath.Join(fieldsPath(version, dataStream), "base-fields.yml")
+func fieldsFilePath(dataStream string) string {
+	return filepath.Join(fieldsPath(dataStream), "fields.yml")
+}
+
+func baseFieldsFilePath(dataStream string) string {
+	return filepath.Join(fieldsPath(dataStream), "base-fields.yml")
 }

docker-compose.yml

@@ -108,6 +108,8 @@ services:
     image: docker.elastic.co/package-registry/distribution:snapshot
     ports:
       - 8080
+    entrypoint: /entrypoint.sh
     volumes:
+      - "./testing/docker/package-registry/entrypoint.sh:/entrypoint.sh"
       - "./testing/docker/package-registry/config.yml:/package-registry/config.yml"
-      - "./apmpackage/apm:/packages/local/apm"
+      - "./apmpackage:/apmpackage" # copied to /packages/local/apm/<version> by entrypoint.sh

script/copy_package.py

@@ -6,6 +6,8 @@
 import subprocess
 import sys
 
+import yaml
+
 
 def semver_sorter(a, b):
     a_list = a.split("-")
@@ -38,22 +40,18 @@ def bump(v):
     parser = argparse.ArgumentParser()
     parser.add_argument('--dst', help='directory of the package-storage repo', default="../../package-storage")
     parser.add_argument('--final', action='store_true')
-    parser.add_argument('-v', '--version', help='version of the package to copy, defaults to last one')
     parser.add_argument('--dry', action='store_true', help='dont copy data')
     args = parser.parse_args()
 
     src = "../apmpackage/apm/"
-    original_version = args.version
-    if not args.version:
-        # default to last version
-        versions = [os.path.basename(f) for f in os.listdir(src)]
-        versions.sort()
-        original_version = versions[-1]
-
-    src = os.path.join(src, original_version)
-    dst = os.path.join(args.dst, "packages/apm/")
+
+    # Get the version from manifest.yml
+    with open(src + 'manifest.yml', 'r') as f:
+        manifest = yaml.safe_load(f)
+    original_version = manifest['version']
 
     # find and sort published versions
+    dst = os.path.join(args.dst, "packages/apm/")
     published_versions = [os.path.basename(f) for f in os.listdir(dst)]
     published_versions.sort(key=cmp_to_key(semver_sorter))
     published_versions.reverse()
@@ -84,7 +82,7 @@ def bump(v):
     # copy over the package and replace version in manifest and pipeline names
     shutil.copytree(src, dst)
     subprocess.check_call('rm -rf {0}'.format(os.path.join(dst, 'README.template.md')), shell=True)
-    cmd = 'find {0} -not -name "*.png" -type f -print0 | xargs -0 sed -i "" "s/{1}/{2}/g"'.format(
+    cmd = 'find {0} -not -name "*.png" -type f -exec sed -i -e "s/{1}/{2}/g" {{}} \\;'.format(
         dst, original_version, next_version)
 
     out = subprocess.check_output(cmd, shell=True)