Merge remote-tracking branch 'origin/release/1.13.0' into release/1.13.0

.gitignore

@@ -10,3 +10,6 @@
 
 #Mac
 .DS_Store
+
+#Test output files
+coverage.txt

Makefile

@@ -20,17 +20,25 @@ all: audit test build
 audit:
 	go list -m all | nancy sleuth
 
+.PHONY: lint
+lint:
+	exit
+
 .PHONY: build
 build:
 	@mkdir -p $(BUILD_ARCH)/$(BIN_DIR)
-	go build $(LDFLAGS) -o $(BUILD_ARCH)/$(BIN_DIR)/$(MAIN) cmd/dp-search-api/main.go
+	go build $(LDFLAGS) -o $(BUILD_ARCH)/$(BIN_DIR)/$(MAIN) main.go
 
 .PHONY: debug
 debug: build
-	HUMAN_LOG=1 go run $(LDFLAGS) -race cmd/$(MAIN)/main.go
+	HUMAN_LOG=1 go run $(LDFLAGS) -race main.go
 
 .PHONY: test
 test:
 	go test -cover -race ./...
 
+.PHONY: test-component
+test-component:
+	go test -cover -race -coverprofile="coverage.txt" -coverpkg=github.com/ONSdigital/$(MAIN)/... -component
+
 .PHONY: build debug test

README.md

@@ -6,14 +6,35 @@ A Go application microservice to provide query functionality on the ONS Website
 
 ### Getting started
 
-* Run `make debug`
+There are now 2 versions of ElasticSearch being used by this service:
+* 2.4.2 (the old/existing ElasticSearch)
+* 7.10.0 (Site Wide ElasticSearch)
+
+Version 2.4.2 is required by all endpoints except for the POST /search endpoint, which uses 7.10
+
+Set up dependencies locally as follows:
+
+* In dp-compose run `docker-compose up -d` to run both versions of ElasticSearch
+* NB. Version 2.4.2 will run on port 9200, version 7.10 will run on port 11200
+* If using endpoints that require version 2.4.2, there are no more dependencies to set up.
+* NB. What endpoints are available depends on what port the ELASTIC_SEARCH_URL points to. By default, it points to port 9200, so to point to 11200 (if required) run the following:
+* `export ELASTIC_SEARCH_URL="http://localhost:11200"`
+* If using the POST /search endpoint then authorisation for this requires running Vault and Zebedee as follows:
+* In any directory run `vault server -dev` as Zebedee has a dependency on Vault
+* In the zebedee directory run `./run.sh` to run Zebedee
+
+Then run `make debug`
 
 ### Dependencies
 
-Clone and set up the following project following the README instructions:
-- [dp-compose](https://github.com/ONSdigital/dp-compose)
+For endpoints that require the old/existing ElasticSearch (version 2.4.2):
+* Requires ElasticSearch running on port 9200
+* No further dependencies other than those defined in `go.mod`
 
-No further dependencies other than those defined in `go.mod`
+For endpoints that require the Site Wide ElasticSearch (version 7.10.0):
+* Requires ElasticSearch running on port 11200
+* Requires Zebedee running on port 8082
+* No further dependencies other than those defined in `go.mod`
 
 ### Configuration
 
@@ -25,9 +46,12 @@ environment variables, or with a link to a configuration guide.
 | AWS_REGION                  | eu-west-1               | The AWS region to use when signing requests with AWS SDK
 | AWS_SERVICE                 | "es"                    | The aws service that the AWS SDK signing mechanism needs to sign a request
 | BIND_ADDR                   | :23900                  | The host and port to bind to
-| ELASTIC_URL	              | "http://localhost:9200" | Http url of the ElasticSearch server
-| SIGN_ELASTICSEARCH_REQUESTS | false                   | Boolean flag to identify whether elasticsearch requests via elastic API need to be signed if elasticsearch cluster is running in aws
+| ELASTIC_SEARCH_URL	      | "http://localhost:9200" | Http url of the ElasticSearch server. For Site Wide ElasticSearch this needs to be set to "http://localhost:11200".
 | GRACEFUL_SHUTDOWN_TIMEOUT   | 5s                      | The graceful shutdown timeout in seconds (`time.Duration` format)
+| SIGN_ELASTICSEARCH_REQUESTS | false                   | Boolean flag to identify whether elasticsearch requests via elastic API need to be signed if elasticsearch cluster is running in aws
+| HEALTHCHECK_CRITICAL_TIMEOUT| 90s                     | Time to wait until an unhealthy dependent propagates its state to make this app unhealthy (`time.Duration` format)
+| HEALTHCHECK_INTERVAL        | 30s                     | Time between self-healthchecks (`time.Duration` format)
+| ZEBEDEE_URL                 | http://localhost:8082   | The URL to Zebedee (for authorisation)
 
 ### Contributing
 

api/api.go

@@ -1,31 +1,38 @@
 package api
 
-//go:generate moq -out mocks.go -pkg api . ElasticSearcher QueryBuilder ResponseTransformer
+//go:generate moq -out mocks.go -pkg api . ElasticSearcher QueryBuilder ResponseTransformer AuthHandler
 
 import (
 	"context"
+	"net/http"
 
-	"github.com/ONSdigital/dp-search-api/config"
-	"github.com/ONSdigital/dp-search-api/service"
-
-	"github.com/ONSdigital/go-ns/server"
-	"github.com/ONSdigital/log.go/v2/log"
+	"github.com/ONSdigital/dp-authorisation/auth"
+	dpelastic "github.com/ONSdigital/dp-elasticsearch/v2/elasticsearch"
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 )
 
-var httpServer *server.Server
+var (
+	update = auth.Permissions{Update: true}
+)
 
 //SearchAPI provides an API around elasticseach
 type SearchAPI struct {
-	Router        *mux.Router
-	QueryBuilder  QueryBuilder
-	ElasticSearch ElasticSearcher
-	Transformer   ResponseTransformer
-	ServiceList   *service.ExternalServiceList
+	Router             *mux.Router
+	QueryBuilder       QueryBuilder
+	dpESClient         *dpelastic.Client
+	deprecatedESClient ElasticSearcher
+	Transformer        ResponseTransformer
+	permissions        AuthHandler
+}
+
+// AuthHandler provides authorisation checks on requests
+type AuthHandler interface {
+	Require(required auth.Permissions, handler http.HandlerFunc) http.HandlerFunc
 }
 
-// ElasticSearcher provides client methods for the elasticsearch package
+// ElasticSearcher provides client methods for the elasticsearch package - now deprecated, due to be replaced
+// with the methods in dp-elasticsearch
 type ElasticSearcher interface {
 	Search(ctx context.Context, index string, docType string, request []byte) ([]byte, error)
 	MultiSearch(ctx context.Context, index string, docType string, request []byte) ([]byte, error)
@@ -42,72 +49,33 @@ type ResponseTransformer interface {
 	TransformSearchResponse(ctx context.Context, responseData []byte, query string, highlight bool) ([]byte, error)
 }
 
-// CreateAndInitialise initiates a new Search API
-func CreateAndInitialise(cfg *config.Config, queryBuilder QueryBuilder, elasticSearchClient ElasticSearcher, transformer ResponseTransformer, hc service.HealthChecker, errorChan chan error) error {
-
-	if elasticSearchClient == nil {
-		return errors.New("CreateAndInitialise called without a valid elasticsearch client")
-	}
-
-	if queryBuilder == nil {
-		return errors.New("CreateAndInitialise called without a valid query builder")
-	}
-	router := mux.NewRouter()
-
+// NewSearchAPI returns a new Search API struct after registering the routes
+func NewSearchAPI(router *mux.Router, dpESClient *dpelastic.Client, deprecatedESClient ElasticSearcher, queryBuilder QueryBuilder, transformer ResponseTransformer, permissions AuthHandler) (*SearchAPI, error) {
 	errData := SetupData()
 	if errData != nil {
-		return errors.Wrap(errData, "Failed to setup data templates")
+		return nil, errors.Wrap(errData, "Failed to setup data templates")
 	}
 
 	errTimeseries := SetupTimeseries()
 	if errTimeseries != nil {
-		return errors.Wrap(errTimeseries, "Failed to setup timeseries templates")
+		return nil, errors.Wrap(errTimeseries, "Failed to setup timeseries templates")
 	}
 
-	ctx := context.Background()
-	router.StrictSlash(true).Path("/health").HandlerFunc(hc.Handler)
-	hc.Start(ctx)
-
-	api := NewSearchAPI(router, elasticSearchClient, queryBuilder, transformer)
-
-	httpServer = server.New(cfg.BindAddr, api.Router)
-
-	// Disable this here to allow service to manage graceful shutdown of the entire app.
-	httpServer.HandleOSSignals = false
-
-	go func() {
-		ctx := context.Background()
-		log.Info(ctx, "search api starting")
-		if err := httpServer.ListenAndServe(); err != nil {
-			log.Error(ctx, "search api http server returned error", err)
-			errorChan <- err
-		}
-	}()
-
-	return nil
-}
-
-// NewSearchAPI returns a new Search API struct after registering the routes
-func NewSearchAPI(router *mux.Router, elasticSearch ElasticSearcher, queryBuilder QueryBuilder, transformer ResponseTransformer) *SearchAPI {
-
 	api := &SearchAPI{
-		Router:        router,
-		QueryBuilder:  queryBuilder,
-		ElasticSearch: elasticSearch,
-		Transformer:   transformer,
+		Router:             router,
+		QueryBuilder:       queryBuilder,
+		dpESClient:         dpESClient,
+		deprecatedESClient: deprecatedESClient,
+		Transformer:        transformer,
+		permissions:        permissions,
 	}
 
-	router.HandleFunc("/search", SearchHandlerFunc(queryBuilder, api.ElasticSearch, api.Transformer)).Methods("GET")
-	router.HandleFunc("/timeseries/{cdid}", TimeseriesLookupHandlerFunc(api.ElasticSearch)).Methods("GET")
-	router.HandleFunc("/data", DataLookupHandlerFunc(api.ElasticSearch)).Methods("GET")
-	return api
-}
+	router.HandleFunc("/search", SearchHandlerFunc(queryBuilder, api.deprecatedESClient, api.Transformer)).Methods("GET")
+	router.HandleFunc("/timeseries/{cdid}", TimeseriesLookupHandlerFunc(api.deprecatedESClient)).Methods("GET")
+	router.HandleFunc("/data", DataLookupHandlerFunc(api.deprecatedESClient)).Methods("GET")
 
-// Close represents the graceful shutting down of the http server
-func Close(ctx context.Context) error {
-	if err := httpServer.Shutdown(ctx); err != nil {
-		return err
-	}
-	log.Info(ctx, "graceful shutdown of http server complete")
-	return nil
+	createSearchIndexHandler := permissions.Require(update, CreateSearchIndexHandlerFunc(api.dpESClient))
+	router.HandleFunc("/search", createSearchIndexHandler).Methods("POST")
+
+	return api, nil
 }