Cache all package manifest on startup

So far in each request to search or categories all manifest had to be reread from disk each time. As the packages do not change during the runtime of the service, this is not necessary and all can be cached on startup of the service and kept in memory. Even if there are one day 10000 packages and each manifest is 1KB, this would only consume ~10MB of memory (currently manifest are only about 250 Bytes). Using 10MB of memory instead of 10000 file read on each request seems worth it.

The code was also modified to in general pass a copy of the packages instead of using a reference. This makes sure even if a request modifies the packages locally, the global package list is not modified.

CHANGELOG.md

@@ -10,6 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added
 
+* Cache all manifest on service startup for resource optimisation.
+
 ### Changed
 
 ### Deprecated

categories.go

@@ -24,24 +24,19 @@ type Category struct {
 func categoriesHandler() func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 
-		packagePaths, err := util.GetPackagePaths(packagesBasePath)
+		packages, err := util.GetPackages(packagesBasePath)
 		if err != nil {
 			notFound(w, err)
 			return
 		}
-
-		packageList := map[string]*util.Package{}
+		packageList := map[string]util.Package{}
 		// Get unique list of newest packages
-		for _, i := range packagePaths {
-			p, err := util.NewPackage(packagesBasePath, i)
-			if err != nil {
-				return
-			}
+		for _, p := range packages {
 
 			// Check if the version exists and if it should be added or not.
 			if pp, ok := packageList[p.Name]; ok {
 				// If the package in the list is newer, do nothing. Otherwise delete and later add the new one.
-				if pp.IsNewer(p) {
+				if pp.IsNewer(&p) {
 					continue
 				}
 			}

main.go

@@ -13,6 +13,8 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/elastic/integrations-registry/util"
+
 	ucfgYAML "github.com/elastic/go-ucfg/yaml"
 
 	"github.com/gorilla/mux"
@@ -45,6 +47,14 @@ func main() {
 	}
 	packagesBasePath = config.PackagesPath
 
+	// Prefill the package cache
+	packages, err := util.GetPackages(packagesBasePath)
+	if err != nil {
+		log.Print(err)
+		os.Exit(1)
+	}
+	log.Printf("%v package manifests loaded into memory.\n", len(packages))
+
 	server := &http.Server{Addr: address, Handler: getRouter()}
 
 	go func() {

search.go

@@ -46,21 +46,15 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
-		packagePaths, err := util.GetPackagePaths(packagesBasePath)
+		packages, err := util.GetPackages(packagesBasePath)
 		if err != nil {
-			notFound(w, err)
+			notFound(w, fmt.Errorf("problem fetching packages: %s", err))
 			return
 		}
-
-		packagesList := map[string]map[string]*util.Package{}
+		packagesList := map[string]map[string]util.Package{}
 
 		// Checks that only the most recent version of an integration is added to the list
-		for _, path := range packagePaths {
-			p, err := util.NewPackage(packagesBasePath, path)
-			if err != nil {
-				notFound(w, err)
-				return
-			}
+		for _, p := range packages {
 
 			// Filter by category first as this could heavily reduce the number of packages
 			// It must happen before the version filtering as there only the newest version
@@ -88,7 +82,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
 						if pp.Name == p.Name {
 
 							// If the package in the list is newer, do nothing. Otherwise delete and later add the new one.
-							if pp.IsNewer(p) {
+							if pp.IsNewer(&p) {
 								continue
 							}
 
@@ -99,7 +93,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
 			}
 
 			if _, ok := packagesList[p.Name]; !ok {
-				packagesList[p.Name] = map[string]*util.Package{}
+				packagesList[p.Name] = map[string]util.Package{}
 			}
 			packagesList[p.Name][p.Version] = p
 		}
@@ -115,7 +109,7 @@ func searchHandler() func(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
-func getPackageOutput(packagesList map[string]map[string]*util.Package) ([]byte, error) {
+func getPackageOutput(packagesList map[string]map[string]util.Package) ([]byte, error) {
 
 	separator := "@"
 	// Packages need to be sorted to be always outputted in the same order

util/packages.go

@@ -1,6 +1,10 @@
 package util
 
-import "io/ioutil"
+import (
+	"io/ioutil"
+)
+
+var packageList []Package
 
 // GetPackagePaths returns list of available packages, one for each version.
 func GetPackagePaths(packagesPath string) ([]string, error) {
@@ -21,3 +25,24 @@ func GetPackagePaths(packagesPath string) ([]string, error) {
 
 	return packages, nil
 }
+
+func GetPackages(packagesBasePath string) ([]Package, error) {
+	if packageList != nil {
+		return packageList, nil
+	}
+
+	packagePaths, err := GetPackagePaths(packagesBasePath)
+	if err != nil {
+		return nil, err
+	}
+
+	// Get unique list of newest packages
+	for _, i := range packagePaths {
+		p, err := NewPackage(packagesBasePath, i)
+		if err != nil {
+			return nil, err
+		}
+		packageList = append(packageList, *p)
+	}
+	return packageList, nil
+}