Change package path from {packagename}-{version} to {packagename}/{ve…

…rsion} (#300)


The current package path is `/package/{packagename}-{version}`. EPM will need some info about which packages exist. To make it possible to populate some generic info about a package, the path is changed to `/package/{packagename}/{version}`. No additional info is added yet but it gives you the option to add it.

This PR makes all changes in the code that are needed on the packaging side to make the above happen. Also all test files are updated.

This is a breaking change for EPM.

.ci/jenkins-go-agent/Dockerfile

@@ -1,9 +1,9 @@
-FROM golang:1.12
+FROM golang:1.13
 
 RUN \
     apt-get update \
       && apt-get install -y --no-install-recommends \
-         zip \
+         zip rsync\
       && rm -rf /var/lib/apt/lists/*
 
 RUN go get -u github.com/magefile/mage \

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Breaking changes
 
+* Change package path from /package/{packagename}-{version} to /package/{packagename}/{version} [#](https://github.com/elastic/integrations-registry/pull/)
 
 ### Bugfixes
 

Dockerfile

@@ -4,7 +4,7 @@ FROM golang:${GO_VERSION:-1.13.4}
 RUN \
     apt-get update \
       && apt-get install -y --no-install-recommends \
-         zip \
+         zip rsync \
       && rm -rf /var/lib/apt/lists/*
 
 COPY ./ /go/src/github.com/elastic/package-registry

Jenkinsfile

@@ -3,7 +3,7 @@
 @Library('apm@current') _
 
 pipeline {
-  agent { label 'docker && linux && immutable' }
+  agent { label 'ubuntu && immutable' }
   environment {
     BASE_DIR="src/github.com/elastic/package-registry"
     JOB_GIT_CREDENTIALS = "f6c7695a-671e-4f4f-a331-acdce44ff9ba"
@@ -27,8 +27,8 @@ pipeline {
   }
   stages {
     /**
-    Checkout the code and stash it, to use it on other stages.
-    */
+     Checkout the code and stash it, to use it on other stages.
+     */
     stage('Checkout') {
       steps {
         deleteDir()
@@ -37,8 +37,8 @@ pipeline {
       }
     }
     /**
-    Checks formatting / linting.
-    */
+     Checks formatting / linting.
+     */
     stage('Lint') {
       steps {
         deleteDir()
@@ -51,8 +51,8 @@ pipeline {
       }
     }
     /**
-    Build the project from code..
-    */
+     Build the project from code..
+     */
     stage('Build') {
       steps {
         deleteDir()
@@ -65,8 +65,8 @@ pipeline {
       }
     }
     /**
-    Execute unit tests.
-    */
+     Execute unit tests.
+     */
     stage('Test') {
       steps {
         deleteDir()
@@ -86,8 +86,8 @@ pipeline {
       }
     }
     /**
-      Publish Docker images.
-    */
+     Publish Docker images.
+     */
     stage('Publish Docker image'){
       environment {
         DOCKER_IMG_TAG = "${env.DOCKER_IMG}:${env.GIT_BASE_COMMIT}"
@@ -105,8 +105,8 @@ pipeline {
             script: "docker push ${env.DOCKER_IMG_TAG}")
           sh(label: 'Re-tag Docker image',
             script: "docker tag ${env.DOCKER_IMG_TAG} ${env.DOCKER_IMG_TAG_BRANCH}")
-            sh(label: 'Push Docker image name',
-              script: "docker push ${env.DOCKER_IMG_TAG_BRANCH}")
+          sh(label: 'Push Docker image name',
+            script: "docker push ${env.DOCKER_IMG_TAG_BRANCH}")
         }
       }
     }

README.md

@@ -9,8 +9,8 @@ Endpoints:
 * `/`: Info about the registry
 * `/search`: Search for packages. By default returns all the most recent packages available.
 * `/categories`: List of the existing package categories and how many packages are in each category.
-* `/package/{name}-{version}`: Info about a package
-* `/package/{name}-{version}.tar.gz`: Download a package
+* `/package/{name}/{version}`: Info about a package
+* `/epr/{name}/{name}-{version}.tar.gz`: Download a package
 
 Examples for each API endpoint can be found here: https://github.com/elastic/package-registry/tree/master/docs/api
 

dev/generator/main.go

@@ -12,6 +12,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/magefile/mage/sh"
 
@@ -75,9 +76,9 @@ func Build(sourceDir, publicDir string) error {
 
 // CopyPackage copies the files of a package to the public directory
 func CopyPackage(src, dst string) error {
-	fmt.Println(">> Copy package: " + src)
+	log.Println(">> Copy package: " + src)
 	os.MkdirAll(dst, 0755)
-	err := sh.RunV("cp", "-a", src, dst)
+	err := sh.RunV("rsync", "-a", src, dst)
 	if err != nil {
 		return err
 	}
@@ -100,14 +101,19 @@ func BuildPackages(sourceDir, packagesPath string) error {
 			continue
 		}
 
+		// Finds the last occurence of "-" and then below splits it up in 2 parts.
+		dashIndex := strings.LastIndex(packageName, "-")
+		dstDir := filepath.Join(packagesPath, packageName[0:dashIndex], packageName[dashIndex+1:])
+
 		if copy {
-			err := CopyPackage(filepath.Join(sourceDir, packageName), packagesPath)
+			// Trailing slash is to make sure content of package is copied
+			err := CopyPackage(filepath.Join(sourceDir, packageName)+"/", dstDir)
 			if err != nil {
 				return err
 			}
 		}
 
-		p, err := util.NewPackage(filepath.Join(packagesPath, packageName))
+		p, err := util.NewPackage(dstDir)
 		if err != nil {
 			return err
 		}
@@ -200,15 +206,35 @@ func buildPackage(packagesBasePath string, p util.Package) error {
 	}
 
 	if tarGz {
-		err = os.MkdirAll(filepath.Join(packagesBasePath, "..", "epr", p.Name), 0755)
+		tarGzDirPath := filepath.Join(packagesBasePath, "..", "epr", p.Name)
+		err = os.MkdirAll(tarGzDirPath, 0755)
+		if err != nil {
+			return err
+		}
+
+		tarGzName := p.Name + "-" + p.Version
+		copiedPackagePath := filepath.Join(tarGzDirPath, tarGzName)
+
+		// As the package directories are now {packagename}/{version} when just running tar, the dir inside
+		// the package had the wrong name. Using `-s` or `--transform` for some reason worked on the command line
+		// but not when run through Golang. So the hack for now is to just copy over all files with the correct name
+		// and then run tar on it.
+		// This could become even useful in the future as things like images or videos should potentially not be part of
+		// a tar.gz to keep it small.
+		err := CopyPackage(packagesBasePath+"/"+p.Name+"/"+p.Version+"/", copiedPackagePath)
 		if err != nil {
 			return err
 		}
 
-		err = sh.RunV("tar", "czf", filepath.Join(packagesBasePath, "..", "epr", p.Name, p.GetPath()+".tar.gz"), "-C", packagesBasePath+"/", filepath.Base(p.GetPath())+"/")
+		err = sh.RunV("tar", "czf", filepath.Join(packagesBasePath, "..", "epr", p.Name, tarGzName+".tar.gz"), "-C", tarGzDirPath, tarGzName+"/")
 		if err != nil {
 			return fmt.Errorf("Error creating package: %s: %s", p.GetPath(), err)
 		}
+
+		err = os.RemoveAll(copiedPackagePath)
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil

docs/api/package.json

@@ -33,25 +33,25 @@
     }
   ],
   "assets": [
-    "/package/example-1.0.0/manifest.yml",
-    "/package/example-1.0.0/docs/README.md",
-    "/package/example-1.0.0/img/icon.png",
-    "/package/example-1.0.0/img/kibana-envoyproxy.jpg",
-    "/package/example-1.0.0/dataset/foo/manifest.yml",
-    "/package/example-1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json",
-    "/package/example-1.0.0/kibana/index-pattern/filebeat-*.json",
-    "/package/example-1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json",
-    "/package/example-1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json",
-    "/package/example-1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json",
-    "/package/example-1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json",
-    "/package/example-1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json",
-    "/package/example-1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json",
-    "/package/example-1.0.0/dataset/foo/fields/stream.yml",
-    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-entry.json",
-    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-http.json",
-    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-json.json",
-    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json",
-    "/package/example-1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json"
+    "/package/example/1.0.0/manifest.yml",
+    "/package/example/1.0.0/docs/README.md",
+    "/package/example/1.0.0/img/icon.png",
+    "/package/example/1.0.0/img/kibana-envoyproxy.jpg",
+    "/package/example/1.0.0/dataset/foo/manifest.yml",
+    "/package/example/1.0.0/kibana/dashboard/0c610510-5cbd-11e9-8477-077ec9664dbd.json",
+    "/package/example/1.0.0/kibana/index-pattern/filebeat-*.json",
+    "/package/example/1.0.0/kibana/visualization/0a994af0-5c9d-11e9-8477-077ec9664dbd.json",
+    "/package/example/1.0.0/kibana/visualization/36f872a0-5c03-11e9-85b4-19d0072eb4f2.json",
+    "/package/example/1.0.0/kibana/visualization/38f96190-5c99-11e9-8477-077ec9664dbd.json",
+    "/package/example/1.0.0/kibana/visualization/7e4084e0-5c99-11e9-8477-077ec9664dbd.json",
+    "/package/example/1.0.0/kibana/visualization/80844540-5c97-11e9-8477-077ec9664dbd.json",
+    "/package/example/1.0.0/kibana/visualization/ab48c3f0-5ca6-11e9-8477-077ec9664dbd.json",
+    "/package/example/1.0.0/dataset/foo/fields/stream.yml",
+    "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-entry.json",
+    "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-http.json",
+    "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-json.json",
+    "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-plaintext.json",
+    "/package/example/1.0.0/dataset/foo/elasticsearch/ingest-pipeline/pipeline-tcp.json"
   ],
   "datasets": [
     {
@@ -106,5 +106,5 @@
     }
   ],
   "download": "/epr/example/example-1.0.0.tar.gz",
-  "path": "/package/example-1.0.0"
-}
+  "path": "/package/example/1.0.0"
+}

docs/api/search-category-logs.json

@@ -3,7 +3,7 @@
     "description": "Package with data sources",
     "download": "/epr/datasources/datasources-1.0.0.tar.gz",
     "name": "datasources",
-    "path": "/package/datasources-1.0.0",
+    "path": "/package/datasources/1.0.0",
     "title": "Default datasource Integration",
     "type": "integration",
     "version": "1.0.0"
@@ -12,7 +12,7 @@
     "description": "Tests if no pipeline is set, it defaults to the default one",
     "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
     "name": "default-pipeline",
-    "path": "/package/default-pipeline-0.0.2",
+    "path": "/package/default-pipeline/0.0.2",
     "title": "Default pipeline Integration",
     "type": "integration",
     "version": "0.0.2"
@@ -21,7 +21,7 @@
     "description": "This is the example integration",
     "download": "/epr/example/example-1.0.0.tar.gz",
     "name": "example",
-    "path": "/package/example-1.0.0",
+    "path": "/package/example/1.0.0",
     "title": "Example Integration",
     "type": "integration",
     "version": "1.0.0"

docs/api/search-category-metrics.json

@@ -3,7 +3,7 @@
     "description": "This is the example integration",
     "download": "/epr/example/example-1.0.0.tar.gz",
     "name": "example",
-    "path": "/package/example-1.0.0",
+    "path": "/package/example/1.0.0",
     "title": "Example Integration",
     "type": "integration",
     "version": "1.0.0"
@@ -12,7 +12,7 @@
     "description": "This is the foo integration",
     "download": "/epr/foo/foo-1.0.0.tar.gz",
     "name": "foo",
-    "path": "/package/foo-1.0.0",
+    "path": "/package/foo/1.0.0",
     "title": "Foo",
     "type": "solution",
     "version": "1.0.0"

docs/api/search-kibana652.json

@@ -3,7 +3,7 @@
     "description": "Package with data sources",
     "download": "/epr/datasources/datasources-1.0.0.tar.gz",
     "name": "datasources",
-    "path": "/package/datasources-1.0.0",
+    "path": "/package/datasources/1.0.0",
     "title": "Default datasource Integration",
     "type": "integration",
     "version": "1.0.0"
@@ -12,7 +12,7 @@
     "description": "Tests if no pipeline is set, it defaults to the default one",
     "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
     "name": "default-pipeline",
-    "path": "/package/default-pipeline-0.0.2",
+    "path": "/package/default-pipeline/0.0.2",
     "title": "Default pipeline Integration",
     "type": "integration",
     "version": "0.0.2"
@@ -21,7 +21,7 @@
     "description": "This is the example integration.",
     "download": "/epr/example/example-0.0.2.tar.gz",
     "name": "example",
-    "path": "/package/example-0.0.2",
+    "path": "/package/example/0.0.2",
     "title": "Example",
     "type": "integration",
     "version": "0.0.2"

docs/api/search-kibana721.json

@@ -3,7 +3,7 @@
     "description": "Package with data sources",
     "download": "/epr/datasources/datasources-1.0.0.tar.gz",
     "name": "datasources",
-    "path": "/package/datasources-1.0.0",
+    "path": "/package/datasources/1.0.0",
     "title": "Default datasource Integration",
     "type": "integration",
     "version": "1.0.0"
@@ -12,7 +12,7 @@
     "description": "Tests if no pipeline is set, it defaults to the default one",
     "download": "/epr/default-pipeline/default-pipeline-0.0.2.tar.gz",
     "name": "default-pipeline",
-    "path": "/package/default-pipeline-0.0.2",
+    "path": "/package/default-pipeline/0.0.2",
     "title": "Default pipeline Integration",
     "type": "integration",
     "version": "0.0.2"
@@ -21,7 +21,7 @@
     "description": "This is the example integration",
     "download": "/epr/example/example-1.0.0.tar.gz",
     "name": "example",
-    "path": "/package/example-1.0.0",
+    "path": "/package/example/1.0.0",
     "title": "Example Integration",
     "type": "integration",
     "version": "1.0.0"
@@ -30,7 +30,7 @@
     "description": "This is the foo integration",
     "download": "/epr/foo/foo-1.0.0.tar.gz",
     "name": "foo",
-    "path": "/package/foo-1.0.0",
+    "path": "/package/foo/1.0.0",
     "title": "Foo",
     "type": "solution",
     "version": "1.0.0"

docs/api/search-package-example.json

@@ -3,7 +3,7 @@
     "description": "This is the example integration.",
     "download": "/epr/example/example-0.0.2.tar.gz",
     "name": "example",
-    "path": "/package/example-0.0.2",
+    "path": "/package/example/0.0.2",
     "title": "Example",
     "type": "integration",
     "version": "0.0.2"
@@ -12,7 +12,7 @@
     "description": "This is the example integration",
     "download": "/epr/example/example-1.0.0.tar.gz",
     "name": "example",
-    "path": "/package/example-1.0.0",
+    "path": "/package/example/1.0.0",
     "title": "Example Integration",
     "type": "integration",
     "version": "1.0.0"