Cache all package manifest on startup

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 * Add validation check that Kibana min/max are valid semver versions. [#99](https://github.com/elastic/integrations-registry/pull/99)
 * Adding Cache-Control max-age headers to all http responses set to 1h. [#101](https://github.com/elastic/integrations-registry/pull/101)
 * Validate packages to guarantee only predefined categories can be used. [#100](https://github.com/elastic/integrations-registry/pull/100)
+* Cache all manifest on service startup for resource optimisation. [#103](https://github.com/elastic/integrations-registry/pull/103)
 
 ### Changed
 

categories.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package main
 
 import (
@@ -9,8 +13,6 @@ import (
 	"github.com/elastic/integrations-registry/util"
 )
 
-
-
 type Category struct {
 	Id    string `yaml:"id" json:"id"`
 	Title string `yaml:"title" json:"title"`
@@ -22,19 +24,14 @@ func categoriesHandler() func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		cacheHeaders(w)
 
-		packagePaths, err := util.GetPackagePaths(packagesBasePath)
+		packages, err := util.GetPackages(packagesBasePath)
 		if err != nil {
 			notFound(w, err)
 			return
 		}
-
-		packageList := map[string]*util.Package{}
+		packageList := map[string]util.Package{}
 		// Get unique list of newest packages
-		for _, i := range packagePaths {
-			p, err := util.NewPackage(packagesBasePath, i)
-			if err != nil {
-				return
-			}
+		for _, p := range packages {
 
 			// Check if the version exists and if it should be added or not.
 			if pp, ok := packageList[p.Name]; ok {

magefile.go

@@ -90,21 +90,21 @@ func BuildIntegrationPackages() error {
 		packagesBasePath = publicDir + "/" + packageDir + "/"
 	}
 
-	packagePaths, err := util.GetPackagePaths(packagesBasePath)
+	packages, err := util.GetPackages(packagesBasePath)
 	if err != nil {
 		return err
 	}
 
-	for _, path := range packagePaths {
-		err = buildPackage(packagesBasePath, path)
+	for _, p := range packages {
+		err = buildPackage(packagesBasePath, p)
 		if err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func buildPackage(packagesBasePath, path string) error {
+func buildPackage(packagesBasePath string, p util.Package) error {
 
 	// Change path to simplify tar command
 	currentPath, err := os.Getwd()
@@ -117,29 +117,23 @@ func buildPackage(packagesBasePath, path string) error {
 	}
 	defer os.Chdir(currentPath)
 
-	err = sh.RunV("tar", "cvzf", path+".tar.gz", filepath.Base(path)+"/")
+	err = sh.RunV("tar", "cvzf", p.GetPath()+".tar.gz", filepath.Base(p.GetPath())+"/")
 	if err != nil {
-		return err
-	}
-
-	// Build package endpoint
-	p, err := util.NewPackage(".", path)
-	if err != nil {
-		return fmt.Errorf("Error creating package: %s: %s", path, err)
+		return fmt.Errorf("Error creating package: %s: %s", p.GetPath(), err)
 	}
 
 	// Checks if the package is valid
 	err = p.Validate()
 	if err != nil {
-		return fmt.Errorf("Invalid package %s-%s: %s", p.Name, p.Version, err)
+		return fmt.Errorf("Invalid package: %s: %s", p.GetPath(), err)
 	}
 
-	err = p.LoadAssets(path)
+	err = p.LoadAssets(p.GetPath())
 	if err != nil {
 		return err
 	}
 
-	err = writeJsonFile(p, path+"/index.json")
+	err = writeJsonFile(p, p.GetPath()+"/index.json")
 	if err != nil {
 		return err
 	}

main.go

@@ -14,6 +14,8 @@ import (
 	"strconv"
 	"syscall"
 
+	"github.com/elastic/integrations-registry/util"
+
 	ucfgYAML "github.com/elastic/go-ucfg/yaml"
 
 	"github.com/gorilla/mux"
@@ -46,6 +48,14 @@ func main() {
 	}
 	packagesBasePath = config.PackagesPath
 
+	// Prefill the package cache
+	packages, err := util.GetPackages(packagesBasePath)
+	if err != nil {
+		log.Print(err)
+		os.Exit(1)
+	}
+	log.Printf("%v package manifests loaded into memory.\n", len(packages))
+
 	server := &http.Server{Addr: address, Handler: getRouter()}
 
 	go func() {

search.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package main
 
 import (
@@ -47,21 +51,15 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
-		packagePaths, err := util.GetPackagePaths(packagesBasePath)
+		packages, err := util.GetPackages(packagesBasePath)
 		if err != nil {
-			notFound(w, err)
+			notFound(w, fmt.Errorf("problem fetching packages: %s", err))
 			return
 		}
-
-		packagesList := map[string]map[string]*util.Package{}
+		packagesList := map[string]map[string]util.Package{}
 
 		// Checks that only the most recent version of an integration is added to the list
-		for _, path := range packagePaths {
-			p, err := util.NewPackage(packagesBasePath, path)
-			if err != nil {
-				notFound(w, err)
-				return
-			}
+		for _, p := range packages {
 
 			// Filter by category first as this could heavily reduce the number of packages
 			// It must happen before the version filtering as there only the newest version
@@ -100,7 +98,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
 			}
 
 			if _, ok := packagesList[p.Name]; !ok {
-				packagesList[p.Name] = map[string]*util.Package{}
+				packagesList[p.Name] = map[string]util.Package{}
 			}
 			packagesList[p.Name][p.Version] = p
 		}
@@ -116,7 +114,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func getPackageOutput(packagesList map[string]map[string]*util.Package) ([]byte, error) {
+func getPackageOutput(packagesList map[string]map[string]util.Package) ([]byte, error) {
 
 	separator := "@"
 	// Packages need to be sorted to be always outputted in the same order

util/package.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package util
 
 import (
@@ -8,10 +12,9 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
+	yaml "gopkg.in/yaml.v2"
 
 	"github.com/blang/semver"
-
-	"gopkg.in/yaml.v2"
 )
 
 const defaultType = "integration"
@@ -137,7 +140,7 @@ func (p *Package) HasKibanaVersion(version *semver.Version) bool {
 	return true
 }
 
-func (p *Package) IsNewer(pp *Package) bool {
+func (p *Package) IsNewer(pp Package) bool {
 	return p.versionSemVer.GT(pp.versionSemVer)
 }
 
@@ -230,3 +233,7 @@ func (p *Package) Validate() error {
 
 	return nil
 }
+
+func (p *Package) GetPath() string {
+	return p.Name + "-" + p.Version
+}

util/package_test.go

@@ -1,3 +1,7 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package util
 
 import (

util/packages.go

@@ -1,9 +1,41 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
 package util
 
-import "io/ioutil"
+import (
+	"io/ioutil"
+)
+
+var packageList []Package
+
+// GetPackages returns a slice with all existing packages.
+// The list is stored in memory and on the second request directly
+// served from memory. This assumes chnages to packages only happen on restart.
+// Caching the packages request many file reads every time this method is called.
+func GetPackages(packagesBasePath string) ([]Package, error) {
+	if packageList != nil {
+		return packageList, nil
+	}
+
+	packagePaths, err := getPackagePaths(packagesBasePath)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, i := range packagePaths {
+		p, err := NewPackage(packagesBasePath, i)
+		if err != nil {
+			return nil, err
+		}
+		packageList = append(packageList, *p)
+	}
+	return packageList, nil
+}
 
-// GetPackagePaths returns list of available packages, one for each version.
-func GetPackagePaths(packagesPath string) ([]string, error) {
+// getPackagePaths returns list of available packages, one for each version.
+func getPackagePaths(packagesPath string) ([]string, error) {
 
 	files, err := ioutil.ReadDir(packagesPath)
 	if err != nil {