Proxy upstream support

.circleci/config.yml

@@ -3,7 +3,7 @@ version: 2.1
 executors:
   build:
     docker:
-      - image: circleci/golang:1.13
+      - image: circleci/golang:1.14
   deploy:
     docker:
       - image: circleci/node:8.10

Makefile

@@ -6,6 +6,7 @@ BUILD_HASH_SHORT = $(shell git rev-parse --short HEAD)
 LDFLAGS += -X "github.com/mattermost/mattermost-marketplace/internal/api.buildTag=$(BUILD_TAG)"
 LDFLAGS += -X "github.com/mattermost/mattermost-marketplace/internal/api.buildHash=$(BUILD_HASH)"
 LDFLAGS += -X "github.com/mattermost/mattermost-marketplace/internal/api.buildHashShort=$(BUILD_HASH_SHORT)"
+LDFLAGS += -X "main.upstreamURL=$(BUILD_UPSTREAM_URL)"
 SLS_STAGE ?= "dev"
 
 ## Checks the code style, tests, builds and bundles.
@@ -52,7 +53,7 @@ run: run-server
 
 ## Run the Plugin Marketplace
 .PHONY: run-server
-run-server:
+run-server: generate
 	go run -ldflags="$(LDFLAGS)" ./cmd/marketplace server
 
 ## Compile the server as a lambda function

README.md

@@ -38,6 +38,20 @@ Running all tests:
 $ make test
 ```
 
+### Proxying upstream
+
+The marketplace can be configured to proxy to an upstream marketplace, overlaying any locally defined plugins on top of the remote service. Invoke the server with the appropriate flag:
+
+```
+go run ./cmd/marketplace server --upstream https://api.integrations.mattermost.com
+```
+
+To compile this flag into the binary such as when building the lambda function, define the appropriate environment variable:
+```
+export BUILD_UPSTREAM_URL=https://api.integrations.mattermost.com
+make build-lambda
+```
+
 ### Updating plugins.json
 
 At the moment, the Marketplace simply points at the latest release of a fixed set of Mattermost plugins. In the future, this database will be fine-tuned to facilitate tracking multiple versions for the appropriate Mattermost server version. To update `plugins.json`, ensure you have [jq](https://stedolan.github.io/jq/) and [sponge](https://linux.die.net/man/1/sponge) installed and run:

cmd/lambda/main.go

@@ -13,6 +13,11 @@ import (
 	"github.com/mattermost/mattermost-marketplace/internal/store"
 )
 
+var (
+	// upstreamURL may be compiled into the binary by defining $BUILD_UPSTREAM_URL
+	upstreamURL = ""
+)
+
 var logger *logrus.Logger
 
 func main() {
@@ -22,7 +27,7 @@ func main() {
 	}
 }
 
-func newStatikStore(statikPath string, logger logrus.FieldLogger) (*store.Store, error) {
+func newStatikStore(statikPath string, logger logrus.FieldLogger) (*store.StaticStore, error) {
 	statikFS, err := fs.New()
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to open statik fileystem")
@@ -34,7 +39,7 @@ func newStatikStore(statikPath string, logger logrus.FieldLogger) (*store.Store,
 	}
 	defer database.Close()
 
-	statikStore, err := store.New(database, logger)
+	statikStore, err := store.NewStaticFromReader(database, logger)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to initialize store")
 	}
@@ -45,14 +50,26 @@ func newStatikStore(statikPath string, logger logrus.FieldLogger) (*store.Store,
 func listenAndServe() error {
 	logger = logrus.New()
 
+	var stores []store.Store
+
 	statikStore, err := newStatikStore("/plugins.json", logger)
 	if err != nil {
 		return err
 	}
+	stores = append(stores, statikStore)
+
+	if upstreamURL != "" {
+		upstreamStore, err := store.NewProxy(upstreamURL, logger)
+		if err != nil {
+			return errors.Wrap(err, "failed to initialize upstream store")
+		}
+
+		stores = append(stores, upstreamStore)
+	}
 
 	router := mux.NewRouter()
 	api.Register(router, &api.Context{
-		Store:  statikStore,
+		Store:  store.NewMerged(logger, stores...),
 		Logger: logger,
 	})
 

cmd/marketplace/server.go

@@ -18,13 +18,19 @@ import (
 	"github.com/mattermost/mattermost-marketplace/internal/store"
 )
 
-var instanceID string
+var (
+	instanceID string
+
+	// upstreamURL may be compiled into the binary by defining $BUILD_UPSTREAM_URL
+	upstreamURL string
+)
 
 func init() {
 	instanceID = model.NewId()
 
 	serverCmd.PersistentFlags().String("database", "plugins.json", "The read-only JSON file backing the server.")
 	serverCmd.PersistentFlags().String("listen", ":8085", "The interface and port on which to listen.")
+	serverCmd.PersistentFlags().String("upstream", upstreamURL, "An upstream marketplace server with which to merge results.")
 	serverCmd.PersistentFlags().Bool("debug", false, "Whether to output debug logs.")
 }
 
@@ -46,18 +52,33 @@ var serverCmd = &cobra.Command{
 		}
 		defer databaseFile.Close()
 
-		fileStore, err := store.New(databaseFile, logger)
+		var stores []store.Store
+
+		staticStore, err := store.NewStaticFromReader(databaseFile, logger)
 		if err != nil {
 			return errors.Wrap(err, "failed to initialize store")
 		}
+		stores = append(stores, staticStore)
+
+		upstreamURL, _ := command.Flags().GetString("upstream")
+		if upstreamURL != "" {
+			upstreamStore, err := store.NewProxy(upstreamURL, logger)
+			if err != nil {
+				return errors.Wrap(err, "failed to initialize upstream store")
+			}
+
+			logger.WithField("upstream", upstreamURL).Info("Proxying to upstream marketplace")
+
+			stores = append(stores, upstreamStore)
+		}
 
 		logger := logger.WithField("instance", instanceID)
 		logger.Info("Starting Plugin Marketplace")
 
 		router := mux.NewRouter()
 
 		api.Register(router, &api.Context{
-			Store:  fileStore,
+			Store:  store.NewMerged(logger, stores...),
 			Logger: logger,
 		})
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/mattermost/mattermost-marketplace
 
-go 1.12
+go 1.14
 
 require (
 	github.com/akrylysov/algnhsa v0.0.0-20190319020909-05b3d192e9a7

internal/api/plugins_test.go

@@ -23,7 +23,7 @@ func setupAPI(t *testing.T, plugins []*model.Plugin) (*api.Client, func()) {
 
 	data, err := json.Marshal(plugins)
 	require.NoError(t, err)
-	store, err := store.New(bytes.NewReader(data), logger)
+	store, err := store.NewStaticFromReader(bytes.NewReader(data), logger)
 	require.NoError(t, err)
 
 	router := mux.NewRouter()

internal/store/merged.go

@@ -0,0 +1,58 @@
+package store
+
+import (
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+
+	"github.com/mattermost/mattermost-marketplace/internal/model"
+)
+
+// Store describes the interface to the backing store.
+type Store interface {
+	GetPlugins(filter *model.PluginFilter) ([]*model.Plugin, error)
+}
+
+// Merged is a store that merges the results of multiple stores together, always preferring the
+// most recently updated plugin record when a conflict occurs.
+type Merged struct {
+	stores []Store
+	logger logrus.FieldLogger
+}
+
+// NewMerged creates a new instance of the merged store.
+func NewMerged(logger logrus.FieldLogger, stores ...Store) *Merged {
+	return &Merged{
+		stores: stores,
+		logger: logger,
+	}
+}
+
+// GetPlugins fetches the given page of plugins. The first page is 0.
+func (store *Merged) GetPlugins(pluginFilter *model.PluginFilter) ([]*model.Plugin, error) {
+	// Short-circuit if only one store is configured.
+	if len(store.stores) == 1 {
+		return store.stores[0].GetPlugins(pluginFilter)
+	}
+
+	plugins := []*model.Plugin{}
+	for i, store := range store.stores {
+		storePlugins, err := store.GetPlugins(&model.PluginFilter{
+			Page:          0,
+			PerPage:       model.AllPerPage,
+			Filter:        pluginFilter.Filter,
+			ServerVersion: pluginFilter.ServerVersion,
+		})
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to query store %d", i)
+		}
+
+		plugins = append(plugins, storePlugins...)
+	}
+
+	staticStore, err := NewStatic(plugins, store.logger)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to initialize static store")
+	}
+
+	return staticStore.GetPlugins(pluginFilter)
+}