modify schema, reindex script and dependencies to support dimensions and population types

.gitignore

@@ -15,3 +15,5 @@
 coverage.txt
 
 /main
+
+.go/

Dockerfile.local

@@ -0,0 +1,14 @@
+FROM golang:1.19-bullseye AS base
+
+ENV GOCACHE=/go/.go/cache GOPATH=/go/.go/path TZ=Europe/London
+
+# Install github.com/cespare/reflex
+RUN GOBIN=/bin go install github.com/cespare/[email protected]
+RUN PATH=$PATH:/bin
+
+# Clean cache, as we want all modules in the container to be under /go/.go/path
+RUN go clean -modcache
+
+# Map between the working directories of dev and live
+RUN ln -s /go /dp-search-data-importer
+WORKDIR /dp-search-data-importer

Makefile

@@ -46,6 +46,10 @@ build:
 debug: 
 	HUMAN_LOG=1 go run $(LDFLAGS) -race main.go
 
+.PHONY: debug-run
+debug-run:
+	HUMAN_LOG=1 DEBUG=1 go run -tags 'debug' -race $(LDFLAGS) main.go
+
 .PHONY: test
 test:
 	go test -cover -race ./...

README.md

@@ -96,7 +96,16 @@ NB. The Dataset API requires a mongo database named 'datasets', which must conta
 
 The Dataset API also requires this environment variable to be set to true: DISABLE_GRAPH_DB_DEPENDENCY
 
-Please make sure your elasticsearch server is running locally on localhost:11200 and version of the server is 7.10, which is the current supported version.
+Please make sure your elasticsearch server is running locally on localhost:11200 and version of the server is 7.10, which is the current supported version. You may use `dp-compose/v2/stacks/search` stack for this.
+
+If you want to run the reindex script locally but loading data from an environment (e.g. `sandbox`), you may run `dp ssh` with port forwarding for dataset-api and zebedee (please check the services IPs and ports in `https://consul.dp.aws.onsdigital.uk/ui/eu/services`) For example:
+
+```sh
+dp ssh sandbox publishing 2 -p 22000:10.30.138.234:26020
+dp ssh sandbox publishing 1 -p 8082:10.30.138.93:25108
+```
+
+If you do this the auth token will need to be a valid token accepted in the environment you are using.
 
 ###### Steps
 

api/api.go

@@ -19,10 +19,8 @@ var (
 // SearchAPI provides an API around elasticseach
 type SearchAPI struct {
 	Router             *mux.Router
-	QueryBuilder       QueryBuilder
 	dpESClient         DpElasticSearcher
 	deprecatedESClient ElasticSearcher
-	Transformer        ResponseTransformer
 	permissions        AuthHandler
 }
 
@@ -52,8 +50,8 @@ type QueryParamValidator interface {
 
 // QueryBuilder provides methods for the search package
 type QueryBuilder interface {
-	BuildSearchQuery(ctx context.Context, q, contentTypes, sort string, topics []string, limit, offset int, esVersion710 bool) ([]byte, error)
-	BuildCountQuery(ctx context.Context, query string) ([]byte, error)
+	BuildSearchQuery(ctx context.Context, req *query.SearchRequest, esVersion710 bool) ([]byte, error)
+	BuildCountQuery(ctx context.Context, req *query.CountRequest) ([]byte, error)
 }
 
 // ReleaseQueryBuilder provides an interface to build a search query for the Release content type
@@ -72,23 +70,55 @@ type ReleaseResponseTransformer interface {
 }
 
 // NewSearchAPI returns a new Search API struct after registering the routes
-func NewSearchAPI(router *mux.Router, dpESClient DpElasticSearcher, deprecatedESClient ElasticSearcher, queryBuilder QueryBuilder, transformer ResponseTransformer, permissions AuthHandler) (*SearchAPI, error) {
-	api := &SearchAPI{
+func NewSearchAPI(router *mux.Router, dpESClient DpElasticSearcher, deprecatedESClient ElasticSearcher, permissions AuthHandler) *SearchAPI {
+	return &SearchAPI{
 		Router:             router,
-		QueryBuilder:       queryBuilder,
 		dpESClient:         dpESClient,
 		deprecatedESClient: deprecatedESClient,
-		Transformer:        transformer,
 		permissions:        permissions,
 	}
+}
 
-	router.HandleFunc("/search", SearchHandlerFunc(queryBuilder, api.dpESClient, api.Transformer)).Methods("GET")
-	createSearchIndexHandler := permissions.Require(update, api.CreateSearchIndexHandlerFunc)
-	router.HandleFunc("/search", createSearchIndexHandler).Methods("POST")
-	return api, nil
+// RegisterGetSearch registers the handler for GET /search endpoint
+// with the provided validator and query builder
+// as well as the API's elasticsearch client and response transformer
+func (a *SearchAPI) RegisterGetSearch(validator QueryParamValidator, builder QueryBuilder, transformer ResponseTransformer) *SearchAPI {
+	a.Router.HandleFunc(
+		"/search",
+		SearchHandlerFunc(
+			validator,
+			builder,
+			a.dpESClient,
+			transformer,
+		),
+	).Methods(http.MethodGet)
+	return a
+}
+
+// RegisterPostSearch registers the handler for POST /search endpoint
+// enforcing required update permissions
+func (a *SearchAPI) RegisterPostSearch() *SearchAPI {
+	a.Router.HandleFunc(
+		"/search",
+		a.permissions.Require(
+			update,
+			a.CreateSearchIndexHandlerFunc,
+		),
+	).Methods(http.MethodPost)
+	return a
 }
 
-func (a *SearchAPI) AddSearchReleaseAPI(validator QueryParamValidator, builder ReleaseQueryBuilder, searcher DpElasticSearcher, transformer ReleaseResponseTransformer) *SearchAPI {
-	a.Router.HandleFunc("/search/releases", SearchReleasesHandlerFunc(validator, builder, searcher, transformer)).Methods("GET")
+// RegisterGetSearchRelease registers the handler for GET /search/releases endpoint
+// with the provided validator, query builder, searcher and validator
+func (a *SearchAPI) RegisterGetSearchReleases(validator QueryParamValidator, builder ReleaseQueryBuilder, transformer ReleaseResponseTransformer) *SearchAPI {
+	a.Router.HandleFunc(
+		"/search/releases",
+		SearchReleasesHandlerFunc(
+			validator,
+			builder,
+			a.dpESClient,
+			transformer,
+		),
+	).Methods(http.MethodGet)
 	return a
 }