Merge branch 'develop' into release/1.8.0

README.md

@@ -21,11 +21,13 @@ An overview of the configuration options available, either as a table of
 environment variables, or with a link to a configuration guide.
 
 | Environment variable | Default | Description
-| -------------------------   | ----------------------- | -----------
+| -------------------- | ------- | -----------
+| AWS_REGION                  | eu-west-1               | The AWS region to use when signing requests with AWS SDK
+| AWS_SERVICE                 | "es"                    | The aws service that the AWS SDK signing mechanism needs to sign a request
 | BIND_ADDR                   | :23900                  | The host and port to bind to
 | ELASTIC_URL	              | "http://localhost:9200" | Http url of the ElasticSearch server
-| GRACEFUL_SHUTDOWN_TIMEOUT   | 5s                      | The graceful shutdown timeout in seconds (`time.Duration` format)
 | SIGN_ELASTICSEARCH_REQUESTS | false                   | Boolean flag to identify whether elasticsearch requests via elastic API need to be signed if elasticsearch cluster is running in aws
+| GRACEFUL_SHUTDOWN_TIMEOUT   | 5s                      | The graceful shutdown timeout in seconds (`time.Duration` format)
 
 ### Contributing
 

cmd/dp-search-api/main.go

@@ -7,6 +7,7 @@ import (
 	"syscall"
 
 	dphttp "github.com/ONSdigital/dp-net/http"
+	esauth "github.com/ONSdigital/dp-elasticsearch/v2/awsauth"
 	"github.com/ONSdigital/dp-search-api/api"
 	"github.com/ONSdigital/dp-search-api/config"
 	"github.com/ONSdigital/dp-search-api/elasticsearch"
@@ -50,7 +51,16 @@ func main() {
 		os.Exit(1)
 	}
 
-	elasticSearchClient := elasticsearch.New(cfg.ElasticSearchAPIURL, dphttp.NewClient(), cfg.SignElasticsearchRequests)
+	var esSigner *esauth.Signer
+	if cfg.SignElasticsearchRequests {
+		esSigner, err = esauth.NewAwsSigner("", "", cfg.AwsRegion, cfg.AwsService)
+		if err != nil {
+			log.Event(nil, "failed to create aws v4 signer", log.ERROR, log.Error(err))
+			os.Exit(1)
+		}
+	}
+
+	elasticSearchClient := elasticsearch.New(cfg.ElasticSearchAPIURL, dphttp.NewClient(), cfg.SignElasticsearchRequests, esSigner, cfg.AwsRegion, cfg.AwsService)
 	transformer := transformer.New()
 
 	if err := api.CreateAndInitialise(cfg.BindAddr, queryBuilder, elasticSearchClient, transformer, apiErrors); err != nil {

config/config.go

@@ -9,6 +9,8 @@ import (
 
 // Config is the search API handler config
 type Config struct {
+	AwsRegion                 string        `envconfig:"AWS_REGION"`
+	AwsService                string        `envconfig:"AWS_SERVICE"`
 	BindAddr                  string        `envconfig:"BIND_ADDR"`
 	ElasticSearchAPIURL       string        `envconfig:"ELASTIC_SEARCH_URL"`
 	GracefulShutdownTimeout   time.Duration `envconfig:"GRACEFUL_SHUTDOWN_TIMEOUT"`
@@ -24,6 +26,8 @@ func Get() (*Config, error) {
 	}
 
 	cfg = &Config{
+		AwsRegion:                  "eu-west-1",
+		AwsService:                 "es",
 		BindAddr:                  ":23900",
 		ElasticSearchAPIURL:       "http://localhost:9200",
 		GracefulShutdownTimeout:   5 * time.Second,

config/config_test.go

@@ -20,6 +20,8 @@ func TestSpec(t *testing.T) {
 			})
 
 			Convey("The values should be set to the expected defaults", func() {
+				So(cfg.AwsRegion, ShouldEqual, "eu-west-1")
+				So(cfg.AwsService, ShouldEqual, "es")
 				So(cfg.BindAddr, ShouldEqual, ":23900")
 				So(cfg.ElasticSearchAPIURL, ShouldEqual, "http://localhost:9200")
 				So(cfg.GracefulShutdownTimeout, ShouldEqual, 5*time.Second)

elasticsearch/client.go

@@ -6,22 +6,29 @@ import (
 	"io/ioutil"
 	"net/http"
 	"strings"
+	"time"
 
 	dphttp "github.com/ONSdigital/dp-net/http"
 	"github.com/pkg/errors"
-	awsauth "github.com/smartystreets/go-aws-auth"
+	esauth "github.com/ONSdigital/dp-elasticsearch/v2/awsauth"
 )
 
 // Client represents an instance of the elasticsearch client
 type Client struct {
+	awsRegion    string
+	awsSDKSigner *esauth.Signer
+	awsService   string
 	url          string
 	client       dphttp.Clienter
 	signRequests bool
 }
 
 // New creates a new elasticsearch client. Any trailing slashes from the URL are removed.
-func New(url string, client dphttp.Clienter, signRequests bool) *Client {
+func New(url string, client dphttp.Clienter, signRequests bool, awsSDKSigner *esauth.Signer, awsService string, awsRegion string) *Client {
 	return &Client{
+		awsSDKSigner: awsSDKSigner,
+		awsRegion:    awsRegion,
+		awsService:   awsService,
 		url:          strings.TrimRight(url, "/"),
 		client:       client,
 		signRequests: signRequests,
@@ -40,13 +47,17 @@ func (cli *Client) MultiSearch(ctx context.Context, index string, docType string
 
 // GetStatus makes status call for healthcheck purposes
 func (cli *Client) GetStatus(ctx context.Context) ([]byte, error) {
+
 	req, err := http.NewRequest("GET", cli.url+"/_cat/health", nil)
 	if err != nil {
 		return nil, err
 	}
 
 	if cli.signRequests {
-		awsauth.Sign(req)
+		reader := bytes.NewReader([]byte{})
+		if err = cli.awsSDKSigner.Sign(req, reader, time.Now()); err != nil {
+			return nil, err
+		}
 	}
 
 	resp, err := cli.client.Do(ctx, req)
@@ -71,7 +82,9 @@ func (cli *Client) post(ctx context.Context, index string, docType string, actio
 	}
 
 	if cli.signRequests {
-		awsauth.Sign(req)
+		if err = cli.awsSDKSigner.Sign(req, reader, time.Now()); err != nil {
+			return nil, err
+		}
 	}
 
 	resp, err := cli.client.Do(ctx, req)

elasticsearch/client_test.go

@@ -3,19 +3,20 @@ package elasticsearch
 import (
 	"context"
 	"errors"
+	esauth "github.com/ONSdigital/dp-elasticsearch/v2/awsauth"
+	dphttp "github.com/ONSdigital/dp-net/http"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
-	dphttp "github.com/ONSdigital/dp-net/http"
-
 	. "github.com/smartystreets/goconvey/convey"
 )
 
 func TestSearch(t *testing.T) {
 
 	Convey("When Search is called", t, func() {
+		// Define a mock struct to be used in your unit tests of myFunc.
 
 		Convey("Then a request with the search action should be posted", func() {
 			dphttpMock := &dphttp.ClienterMock{
@@ -24,7 +25,9 @@ func TestSearch(t *testing.T) {
 				},
 			}
 
-			client := New("http://localhost:999", dphttpMock, false)
+			var testSigner *esauth.Signer
+
+			client := New("http://localhost:999", dphttpMock, false, testSigner, "es", "eu-west-1")
 
 			res, err := client.Search(context.Background(), "index", "doctype", []byte("search request"))
 			So(err, ShouldBeNil)
@@ -46,7 +49,9 @@ func TestSearch(t *testing.T) {
 				},
 			}
 
-			client := New("http://localhost:999", dphttpMock, false)
+			var testSigner *esauth.Signer
+
+			client := New("http://localhost:999", dphttpMock, false, testSigner, "es", "eu-west-1")
 
 			_, err := client.Search(context.Background(), "index", "doctype", []byte("search request"))
 			So(err, ShouldNotBeNil)
@@ -76,7 +81,9 @@ func TestMultiSearch(t *testing.T) {
 				},
 			}
 
-			client := New("http://localhost:999", dphttpMock, false)
+			var testSigner *esauth.Signer
+
+			client := New("http://localhost:999", dphttpMock, false, testSigner, "es", "eu-west-1")
 
 			res, err := client.MultiSearch(context.Background(), "index", "doctype", []byte("multiSearch request"))
 			So(err, ShouldBeNil)
@@ -96,8 +103,9 @@ func TestMultiSearch(t *testing.T) {
 					return nil, errors.New("http error")
 				},
 			}
+			var testSigner *esauth.Signer
 
-			client := New("http://localhost:999", dphttpMock, false)
+			client := New("http://localhost:999", dphttpMock, false, testSigner, "es", "eu-west-1")
 
 			_, err := client.MultiSearch(context.Background(), "index", "doctype", []byte("search request"))
 			So(err, ShouldNotBeNil)
@@ -126,7 +134,9 @@ func TestGetStatus(t *testing.T) {
 				},
 			}
 
-			client := New("http://localhost:999", dphttpMock, false)
+			var testSigner *esauth.Signer
+
+			client := New("http://localhost:999", dphttpMock, false, testSigner, "es", "eu-west-1")
 
 			res, err := client.GetStatus(context.Background())
 			So(err, ShouldBeNil)
@@ -144,7 +154,9 @@ func TestGetStatus(t *testing.T) {
 				},
 			}
 
-			client := New("http://localhost:999", dphttpMock, false)
+			var testSigner *esauth.Signer
+
+			client := New("http://localhost:999", dphttpMock, false, testSigner, "es", "eu-west-1")
 
 			_, err := client.GetStatus(context.Background())
 			So(err, ShouldNotBeNil)
@@ -155,6 +167,23 @@ func TestGetStatus(t *testing.T) {
 			So(actualRequest.Method, ShouldResemble, "GET")
 
 		})
+
+		Convey("Then a signing error should be passed back", func() {
+			dphttpMock := &dphttp.ClienterMock{
+				DoFunc: func(ctx context.Context, req *http.Request) (*http.Response, error) {
+					return newResponse("moo"), nil
+				},
+			}
+
+			var testSigner *esauth.Signer
+
+			client := New("http://localhost:999", dphttpMock, true, testSigner, "es", "eu-west-1")
+
+			_, err := client.GetStatus(context.Background())
+			So(err, ShouldNotBeNil)
+			So(err.Error(), ShouldResemble, "v4 signer missing. Cannot sign request")
+
+		})
 	})
 }
 

go.mod

@@ -4,16 +4,16 @@ go 1.16
 
 require (
 	github.com/ONSdigital/dp-api-clients-go v1.34.4 // indirect
+	github.com/ONSdigital/dp-elasticsearch/v2 v2.1.1
 	github.com/ONSdigital/dp-net v1.0.12
 	github.com/ONSdigital/go-ns v0.0.0-20210410105122-6d6a140e952e
 	github.com/ONSdigital/log.go v1.0.1
+	github.com/aws/aws-sdk-go v1.38.65 // indirect
 	github.com/gorilla/mux v1.8.0
 	github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 // indirect
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/pkg/errors v0.9.1
-	github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9
 	github.com/smartystreets/goconvey v1.6.4
-	github.com/smartystreets/gunit v1.4.2 // indirect
 	github.com/tdewolff/minify v2.3.6+incompatible
 	github.com/tdewolff/parse v2.3.4+incompatible // indirect
 	github.com/tdewolff/test v1.0.6 // indirect

go.sum

@@ -2,6 +2,8 @@ github.com/ONSdigital/dp-api-clients-go v1.1.0/go.mod h1:9lqor0I7caCnRWr04gU/r7x
 github.com/ONSdigital/dp-api-clients-go v1.28.0/go.mod h1:iyJy6uRL4B6OYOJA0XMr5UHt6+Q8XmN9uwmURO+9Oj4=
 github.com/ONSdigital/dp-api-clients-go v1.34.4 h1:SHdhxHxflgoAesWpwrVR0OQz+pLKI4jMl58mZrzKyA4=
 github.com/ONSdigital/dp-api-clients-go v1.34.4/go.mod h1:kX+YKuoLYLfkeLHMvQKRRydZVxO7ZEYyYiwG2xhV51E=
+github.com/ONSdigital/dp-elasticsearch/v2 v2.1.1 h1:BoIW0jzmD0BOucas11qaWnQNgBxSqK2ht3Vbv26MzAw=
+github.com/ONSdigital/dp-elasticsearch/v2 v2.1.1/go.mod h1:f85VX6TzHkq3pM/ddUzufoFNTcx3FOi7VN+SQG0vMbg=
 github.com/ONSdigital/dp-frontend-models v1.1.0/go.mod h1:TT96P7Mi69N3Tc/jFNdbjiwG4GAaMjP26HLotFQ6BPw=
 github.com/ONSdigital/dp-healthcheck v0.0.0-20200131122546-9db6d3f0494e/go.mod h1:zighxZ/0m5u7zo0eAr8XFlA+Dz2ic7A1vna6YXvhCjQ=
 github.com/ONSdigital/dp-healthcheck v1.0.5/go.mod h1:2wbVAUHMl9+4tWhUlxYUuA1dnf2+NrwzC+So5f5BMLk=
@@ -23,6 +25,11 @@ github.com/ONSdigital/log.go v1.0.1-0.20200805084515-ee61165ea36a/go.mod h1:dDnQ
 github.com/ONSdigital/log.go v1.0.1-0.20200805145532-1f25087a0744/go.mod h1:y4E9MYC+cV9VfjRD0UBGj8PA7H3wABqQi87/ejrDhYc=
 github.com/ONSdigital/log.go v1.0.1 h1:SZ5wRZAwlt2jQUZ9AUzBB/PL+iG15KapfQpJUdA18/4=
 github.com/ONSdigital/log.go v1.0.1/go.mod h1:dIwSXuvFB5EsZG5x44JhsXZKMd80zlb0DZxmiAtpL4M=
+github.com/aws/aws-sdk-go v1.38.15/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/aws/aws-sdk-go v1.38.65 h1:umGu5gjIOKxzhi34T0DIA1TWupUDjV2aAW5vK6154Gg=
+github.com/aws/aws-sdk-go v1.38.65/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM=
 github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg=
 github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg=
@@ -41,6 +48,10 @@ github.com/gorilla/schema v1.1.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlI
 github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI=
 github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 h1:nqAlWFEdqI0ClbTDrhDvE/8LeQ4pftrqKUX9w5k0j3s=
 github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI=
+github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/justinas/alice v1.2.0 h1:+MHSA/vccVCF4Uq37S42jwlkvI2Xzl7zTPCN5BnZNVo=
@@ -56,14 +67,13 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex
 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
-github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9 h1:hp2CYQUINdZMHdvTdXtPOY2ainKl4IoMcpAXEf2xj3Q=
-github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
 github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
 github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
-github.com/smartystreets/gunit v1.4.2 h1:tyWYZffdPhQPfK5VsMQXfauwnJkqg7Tv5DLuQVYxq3Q=
-github.com/smartystreets/gunit v1.4.2/go.mod h1:ZjM1ozSIMJlAz/ay4SG8PeKF00ckUp+zMHZXV9/bvak=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/tdewolff/minify v2.3.6+incompatible h1:2hw5/9ZvxhWLvBUnHE06gElGYz+Jv9R4Eys0XUzItYo=
 github.com/tdewolff/minify v2.3.6+incompatible/go.mod h1:9Ov578KJUmAWpS6NeZwRZyT56Uf6o3Mcz9CEsg8USYs=
 github.com/tdewolff/parse v2.3.4+incompatible h1:x05/cnGwIMf4ceLuDMBOdQ1qGniMoxpP46ghf0Qzh38=
@@ -97,3 +107,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=