Merge branch 'release/1.18.0'

LICENSE.md

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2016-2021 Office for National Statistics
+Copyright (c) 2016-2022 Office for National Statistics
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 

Makefile

@@ -25,13 +25,26 @@ lint:
 	go install github.com/golangci/golangci-lint/cmd/[email protected]
 	golangci-lint run ./...
 
+.PHONY: fmt
+fmt:
+	go fmt ./...
+
+.PHONY: local
+local:
+	export ELASTIC_SEARCH_URL=https://localhost:9200; \
+	export AWS_TLS_INSECURE_SKIP_VERIFY=true; \
+	export SIGN_ELASTICSEARCH_REQUESTS=true; \
+	export AWS_PROFILE=development; \
+	export AWS_FILENAME=$(HOME)/.aws/credentials; \
+	HUMAN_LOG=1 go run $(LDFLAGS) -race main.go
+
 .PHONY: build
 build:
 	@mkdir -p $(BUILD_ARCH)/$(BIN_DIR)
 	go build $(LDFLAGS) -o $(BUILD_ARCH)/$(BIN_DIR)/$(MAIN) main.go
 
 .PHONY: debug
-debug: build
+debug: 
 	HUMAN_LOG=1 go run $(LDFLAGS) -race main.go
 
 .PHONY: test
@@ -43,3 +56,12 @@ test-component:
 	go test -cover -race -coverprofile="coverage.txt" -coverpkg=github.com/ONSdigital/$(MAIN)/... -component
 
 .PHONY: build debug test
+
+.PHONY: build-reindex
+build-reindex:
+	@mkdir -p $(BUILD)
+	GOOS=linux GOARCH=amd64 go build -tags=aws -ldflags "-w -s" -o $(BUILD)/reindex cmd/reindex/main.go cmd/reindex/aws.go
+
+.PHONY: reindex
+reindex:
+	HUMAN_LOG=1 go run -ldflags "-w -s" cmd/reindex/main.go cmd/reindex/local.go

README.md

@@ -43,22 +43,90 @@ environment variables, or with a link to a configuration guide.
 
 | Environment variable | Default | Description
 | -------------------- | ------- | -----------
-| AWS_REGION                  | eu-west-1               | The AWS region to use when signing requests with AWS SDK
-| AWS_SERVICE                 | "es"                    | The aws service that the AWS SDK signing mechanism needs to sign a request
-| BIND_ADDR                   | :23900                  | The host and port to bind to
-| ELASTIC_SEARCH_URL	      | "http://localhost:9200" | Http url of the ElasticSearch server. For Site Wide ElasticSearch this needs to be set to "http://localhost:11200".
-| GRACEFUL_SHUTDOWN_TIMEOUT   | 5s                      | The graceful shutdown timeout in seconds (`time.Duration` format)
-| SIGN_ELASTICSEARCH_REQUESTS | false                   | Boolean flag to identify whether elasticsearch requests via elastic API need to be signed if elasticsearch cluster is running in aws
-| HEALTHCHECK_CRITICAL_TIMEOUT| 90s                     | Time to wait until an unhealthy dependent propagates its state to make this app unhealthy (`time.Duration` format)
-| HEALTHCHECK_INTERVAL        | 30s                     | Time between self-healthchecks (`time.Duration` format)
-| ZEBEDEE_URL                 | http://localhost:8082   | The URL to Zebedee (for authorisation)
+| AWS_FILENAME                 | ""                       | The AWS file location for finding credentials to sign AWS http requests
+| AWS_PROFILE                  | ""                       | The AWS profile to use from credentials file to sign AWS http requests
+| AWS_REGION                   | eu-west-1                | The AWS region to use when signing requests with AWS SDK
+| AWS_SERVICE                  | "es"                     | The AWS service that the AWS SDK signing mechanism needs to sign a request
+| AWS_TLS_INSECURE_SKIP_VERIFY | false                    | This should never be set to true, as it disables SSL certificate verification. Used only for development
+| BIND_ADDR                    | :23900                   | The host and port to bind to
+| ELASTIC_SEARCH_URL	       | "http://localhost:9200"  | Http url of the ElasticSearch server. For Site Wide ElasticSearch this needs to be set to "http://localhost:11200".
+| GRACEFUL_SHUTDOWN_TIMEOUT    | 5s                       | The graceful shutdown timeout in seconds (`time.Duration` format)
+| SIGN_ELASTICSEARCH_REQUESTS  | false                    | Boolean flag to identify whether elasticsearch requests via elastic API need to be signed if elasticsearch cluster is running in aws
+| HEALTHCHECK_CRITICAL_TIMEOUT | 90s                      | Time to wait until an unhealthy dependent propagates its state to make this app unhealthy (`time.Duration` format)
+| HEALTHCHECK_INTERVAL         | 30s                      | Time between self-healthchecks (`time.Duration` format)
+| ZEBEDEE_URL                  | http://localhost:8082    | The URL to Zebedee (for authorisation)
+
+
+### Connecting to AWS Elasticsearch cluster (dev only)
+
+#### Prerequisites
+
+- You will need an user account for the aws account you are trying to connect to
+- You will need to be given a policy to allow read and write access to the AWS Elasticsearch cluster
+
+#### Connecting
+
+To connect to managed Elasticsearch cluster in AWS, you will want to port forward 9200 to the domain endpoint. Using the dp tool, one can do this like so:
+
+```
+  dp ssh develop <ip of aws box> -p 9200:<elasticsearch cluster domain endpoint e.g. "<unique identifier>" + "eu-west-1.es.amazonaws.com:443"
+```
+
+Once connected, run the following make target:
+
+```
+  make local
+```
+
+### Running Bulk Indexer
+
+#### Locally
+
+Build the bulk indexer by running the following command
+```
+  make reindex
+```
+Then run the executable  
+```
+  ./reindex
+```
+Please make sure your elasticsearch server is running locally on localhost:9200 and version of the server is 7.10, which is the current supported version.
+
+#### Remote Environment
+
+###### Prerequisites
+
+Before attempting the following steps please make sure you have dp tool setup locally.For more info on setting up the dp tool: https://github.com/ONSdigital/dp-cli#build-and-run
+
+###### Steps
+
+Navigate to ```cmd/reindex/aws.go``` and update esurl to correct elastic search 7.10 url in environment and then build the bulk indexer by running the following command
+```
+  make build-reindex
+```
+Then copy to your environment build directory  by running the following command
+```
+dp scp <environment> publishing <box> ./build/reindex <location on publishing box>
+```
+For example
+```
+dp scp develop publishing 2 ./build/reindex .
+```
+Then ssh into the box as follows
+```
+dp ssh <environment> publishing <box>
+```
+Then run the executable
+```
+  ./build/reindex
+```
 
 ### Contributing
 
 See [CONTRIBUTING](CONTRIBUTING.md) for details.
 
 ### License
 
-Copyright © 2016-2021, Office for National Statistics (https://www.ons.gov.uk)
+Copyright © 2016-2022, Office for National Statistics (https://www.ons.gov.uk)
 
 Released under MIT license, see [LICENSE](LICENSE.md) for details.

api/api.go

@@ -1,13 +1,12 @@
 package api
 
-//go:generate moq -out mocks.go -pkg api . ElasticSearcher QueryBuilder ResponseTransformer AuthHandler
+//go:generate moq -out mocks.go -pkg api . ElasticSearcher DpElasticSearcher QueryBuilder ResponseTransformer AuthHandler
 
 import (
 	"context"
 	"net/http"
 
 	"github.com/ONSdigital/dp-authorisation/auth"
-	dpelastic "github.com/ONSdigital/dp-elasticsearch/v2/elasticsearch"
 	"github.com/gorilla/mux"
 	"github.com/pkg/errors"
 )
@@ -20,7 +19,7 @@ var (
 type SearchAPI struct {
 	Router             *mux.Router
 	QueryBuilder       QueryBuilder
-	dpESClient         *dpelastic.Client
+	dpESClient         DpElasticSearcher
 	deprecatedESClient ElasticSearcher
 	Transformer        ResponseTransformer
 	permissions        AuthHandler
@@ -38,6 +37,11 @@ type ElasticSearcher interface {
 	MultiSearch(ctx context.Context, index string, docType string, request []byte) ([]byte, error)
 }
 
+// DpElasticSearcher provides an interface for the dp-elasticsearch functionality
+type DpElasticSearcher interface {
+	CreateIndex(ctx context.Context, indexName string, indexSettings []byte) error
+}
+
 // QueryBuilder provides methods for the search package
 type QueryBuilder interface {
 	BuildSearchQuery(ctx context.Context, q, contentTypes, sort string, limit, offset int) ([]byte, error)
@@ -49,7 +53,7 @@ type ResponseTransformer interface {
 }
 
 // NewSearchAPI returns a new Search API struct after registering the routes
-func NewSearchAPI(router *mux.Router, dpESClient *dpelastic.Client, deprecatedESClient ElasticSearcher, queryBuilder QueryBuilder, transformer ResponseTransformer, permissions AuthHandler) (*SearchAPI, error) {
+func NewSearchAPI(router *mux.Router, dpESClient DpElasticSearcher, deprecatedESClient ElasticSearcher, queryBuilder QueryBuilder, transformer ResponseTransformer, permissions AuthHandler) (*SearchAPI, error) {
 	errData := SetupData()
 	if errData != nil {
 		return nil, errors.Wrap(errData, "Failed to setup data templates")
@@ -72,9 +76,7 @@ func NewSearchAPI(router *mux.Router, dpESClient *dpelastic.Client, deprecatedES
 	router.HandleFunc("/search", SearchHandlerFunc(queryBuilder, api.deprecatedESClient, api.Transformer)).Methods("GET")
 	router.HandleFunc("/timeseries/{cdid}", TimeseriesLookupHandlerFunc(api.deprecatedESClient)).Methods("GET")
 	router.HandleFunc("/data", DataLookupHandlerFunc(api.deprecatedESClient)).Methods("GET")
-
-	createSearchIndexHandler := permissions.Require(update, CreateSearchIndexHandlerFunc(api.dpESClient))
+	createSearchIndexHandler := permissions.Require(update, api.CreateSearchIndexHandlerFunc)
 	router.HandleFunc("/search", createSearchIndexHandler).Methods("POST")
-
 	return api, nil
 }