From 4943488768f1817f94e6f68297b082bf65586421 Mon Sep 17 00:00:00 2001 From: Noelle Date: Tue, 17 Aug 2021 13:45:56 +0100 Subject: [PATCH 01/11] revert to default highlighting --- templates/search/contentQuery.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/templates/search/contentQuery.tmpl b/templates/search/contentQuery.tmpl index f873b985..a5049f37 100644 --- a/templates/search/contentQuery.tmpl +++ b/templates/search/contentQuery.tmpl @@ -26,8 +26,8 @@ "excludes":["downloads.content","downloads*","pageData"]}, {{if .Highlight}} "highlight":{ - "pre_tags":[""], - "post_tags":[""], + "pre_tags":[""], + "post_tags":[""], "fields":{"terms":{"fragment_size":0,"number_of_fragments":0}, "description.title":{"fragment_size":0,"number_of_fragments":0}, "description.edition":{"fragment_size":0,"number_of_fragments":0}, From df5858a7f4b797f65631e13fc60e26120a52a571 Mon Sep 17 00:00:00 2001 From: Noelle Date: Wed, 18 Aug 2021 15:13:48 +0100 Subject: [PATCH 02/11] remove logic with start and end fields --- swagger.yaml | 7 ++ templates/search/departmentsQuery.tmpl | 4 +- transformer/transformer.go | 134 ++----------------------- 3 files changed, 17 insertions(+), 128 deletions(-) diff --git a/swagger.yaml b/swagger.yaml index 7bea9e33..4022cc9b 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -439,6 +439,13 @@ definitions: type: string _type: type: string + _highlight: + type: object + properties: + terms: + type: array + items: + type: string TimeseriesResponse: type: object diff --git a/templates/search/departmentsQuery.tmpl b/templates/search/departmentsQuery.tmpl index d25bdbfc..55a3fe0a 100644 --- a/templates/search/departmentsQuery.tmpl +++ b/templates/search/departmentsQuery.tmpl @@ -11,10 +11,10 @@ }, "highlight": { "pre_tags": [ - "" + "" ], "post_tags": [ - "" + "" ], "fields": { "terms": { diff --git a/transformer/transformer.go b/transformer/transformer.go index 460fe109..003e679d 100644 --- a/transformer/transformer.go +++ b/transformer/transformer.go @@ -3,14 +3,12 @@ package transformer import ( "context" "encoding/json" - "regexp" - "strings" - "github.com/pkg/errors" + "regexp" ) -const startHighlightTag string = "" -const endHighlightTag string = "" +const startHighlightTag string = "" +const endHighlightTag string = "" // Transformer represents an instance of the ResponseTransformer interface type Transformer struct{} @@ -31,10 +29,10 @@ type contentType struct { } type contentItem struct { - Description description `json:"description"` - Type string `json:"type"` - URI string `json:"uri"` - Matches *matches `json:"matches,omitempty"` + Description description `json:"description"` + Type string `json:"type"` + URI string `json:"uri"` + Matches *esHighlight `json:"matches,omitempty"` } type description struct { @@ -64,23 +62,6 @@ type contact struct { Email string `json:"email"` } -type matches struct { - Description struct { - Summary *[]matchDetails `json:"summary"` - Title *[]matchDetails `json:"title"` - Edition *[]matchDetails `json:"edition,omitempty"` - MetaDescription *[]matchDetails `json:"meta_description,omitempty"` - Keywords *[]matchDetails `json:"keywords,omitempty"` - DatasetID *[]matchDetails `json:"dataset_id,omitempty"` - } `json:"description"` -} - -type matchDetails struct { - Value string `json:"value,omitempty"` - Start int `json:"start"` - End int `json:"end"` -} - // Structs representing the raw elastic search response type esResponse struct { @@ -223,7 +204,7 @@ func buildContentItem(doc esResponseHit) contentItem { Description: buildDescription(doc), Type: doc.Source.Type, URI: doc.Source.URI, - Matches: buildMatches(doc.Highlight), + Matches: &doc.Highlight, } return ci @@ -253,105 +234,6 @@ func buildDescription(doc esResponseHit) description { } } -func buildMatches(hl esHighlight) *matches { - var matches matches - - if highlights := hl.DescriptionTitle; highlights != nil { - var titleMatches []matchDetails - for _, m := range *highlights { - foundMatchDetails, _ := findMatches(m) - titleMatches = append(titleMatches, foundMatchDetails...) - } - matches.Description.Title = &titleMatches - } - - if highlights := hl.DescriptionEdition; highlights != nil { - var editionMatches []matchDetails - for _, m := range *highlights { - foundMatchDetails, _ := findMatches(m) - editionMatches = append(editionMatches, foundMatchDetails...) - } - matches.Description.Edition = &editionMatches - } - - if highlights := hl.DescriptionSummary; highlights != nil { - var summaryMatches []matchDetails - for _, m := range *highlights { - foundMatchDetails, _ := findMatches(m) - summaryMatches = append(summaryMatches, foundMatchDetails...) - } - matches.Description.Summary = &summaryMatches - } - - if highlights := hl.DescriptionMeta; highlights != nil { - var summaryMatches []matchDetails - for _, m := range *highlights { - foundMatchDetails, _ := findMatches(m) - summaryMatches = append(summaryMatches, foundMatchDetails...) - } - matches.Description.MetaDescription = &summaryMatches - } - - if highlights := hl.DescriptionKeywords; highlights != nil { - var keywordsMatches []matchDetails - for _, m := range *highlights { - foundMatchDetails, value := findMatches(m) - for _, md := range foundMatchDetails { - md.Value = value - keywordsMatches = append(keywordsMatches, md) - } - } - matches.Description.Keywords = &keywordsMatches - } - - if highlights := hl.DescriptionDatasetID; highlights != nil { - var datasetIDMatches []matchDetails - for _, m := range *highlights { - foundMatchDetails, _ := findMatches(m) - datasetIDMatches = append(datasetIDMatches, foundMatchDetails...) - } - matches.Description.DatasetID = &datasetIDMatches - } - - return &matches -} - -// Find matches finds all the matching marked-up phrases and returns a slice of their start and end points in the string -// NB. The start and end values are the number of bytes, not characters, so be aware when the input contains higher-order -// UTF-8 characters. -func findMatches(s string) ([]matchDetails, string) { - - md := make([]matchDetails, 0, 2) - fs := s - - if start := strings.Index(s, startHighlightTag); start >= 0 { - left := s[0:start] - right := s[start+len(startHighlightTag) : len(s)] - if end := strings.Index(right, endHighlightTag); end >= 0 { - mid := right[0:end] - remain := right[end+len(endHighlightTag) : len(right)] - - md = append(md, matchDetails{ - Start: start + 1, - End: start + end, - }) - - remainMatches, remain := findMatches(remain) - for _, rm := range remainMatches { - rm.Start += len(left) + len(mid) - rm.End += len(left) + len(mid) - md = append(md, rm) - } - - right = mid + remain - } - - fs = left + right - } - - return md, fs -} - func buildContentTypes(bucket esBucket) contentType { return contentType{ Type: bucket.Key, From 1097ae2a18f64645e1b7d6de2a9b970ffcf463e9 Mon Sep 17 00:00:00 2001 From: Noelle Date: Wed, 18 Aug 2021 17:30:39 +0100 Subject: [PATCH 03/11] update swagger spec, make tests pass next --- swagger.yaml | 53 +----- transformer/testdata/search_example.json | 10 +- transformer/testdata/search_expected.json | 26 +-- transformer/transformer.go | 2 - transformer/transformer_test.go | 219 ---------------------- 5 files changed, 13 insertions(+), 297 deletions(-) diff --git a/swagger.yaml b/swagger.yaml index 4022cc9b..185f27a4 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -304,35 +304,9 @@ definitions: - summary - title matches: - type: object - properties: - description: - type: object - properties: - dataset_id: - type: array - items: - $ref: "#/definitions/MatchDetails" - edition: - type: array - items: - $ref: "#/definitions/MatchDetails" - keywords: - type: array - items: - $ref: "#/definitions/MatchDetails" - meta_description: - type: array - items: - $ref: "#/definitions/MatchDetails" - summary: - type: array - items: - $ref: "#/definitions/MatchDetails" - title: - type: array - items: - $ref: "#/definitions/MatchDetails" + type: array + items: + type: string type: type: string uri: @@ -363,26 +337,7 @@ definitions: terms: type: array items: - $ref: "#/definitions/MatchDetails" - - MatchDetails: - description: "A pair of integers to define the start and end of substring in the member that matched the search terms. The first character of the string is index 1." - type: object - properties: - value: - type: string - description: "For matches in keys that are part of an array, this will contain the value containing a match." - start: - type: integer - description: "An integer to define the start byte of a substring in the member that matched. The first character of the string is index 1." - example: 3 - end: - type: integer - description: "An integer to define the end byte of a substring in the member that matched." - example: 8 - required: - - start - - end + type: string Health: type: object diff --git a/transformer/testdata/search_example.json b/transformer/testdata/search_example.json index 206c2ddc..f9608cf8 100644 --- a/transformer/testdata/search_example.json +++ b/transformer/testdata/search_example.json @@ -47,12 +47,12 @@ }, "highlight": { "description.title": [ - "House Price Statistics for Small Areas in England and Wales" + "House Price Statistics for Small Areas in England and Wales" ], "description.keywords": [ - "regional house prices", - "area with cheapest houses", - "area with most expensive houses" + "regional house prices", + "area with cheapest >houses", + "area with most expensive houses" ] }, "sort": [ @@ -79,7 +79,7 @@ }, "highlight": { "description.title": [ - "Housing" + "Housing" ] }, "sort": [ diff --git a/transformer/testdata/search_expected.json b/transformer/testdata/search_expected.json index 549d82a2..52b47f39 100644 --- a/transformer/testdata/search_expected.json +++ b/transformer/testdata/search_expected.json @@ -40,25 +40,17 @@ "summary": null, "title": [ { - "start": 1, - "end": 5 } ], "keywords": [ { - "value": "regional house prices", - "start": 10, - "end": 14 + "value": "regional house prices" }, { - "value": "area with cheapest houses", - "start": 20, - "end": 25 + "value": "area with cheapest houses" }, { - "value": "area with most expensive houses", - "start": 26, - "end": 31 + "value": "area with most expensive houses" } ] } @@ -76,17 +68,7 @@ }, "type": "product_page", "uri": "/peoplepopulationandcommunity/housing", - "matches": { - "description": { - "summary": null, - "title": [ - { - "start": 1, - "end": 7 - } - ] - } - } + "matches": [] } ], "suggestions": [ diff --git a/transformer/transformer.go b/transformer/transformer.go index 003e679d..6eeec5f1 100644 --- a/transformer/transformer.go +++ b/transformer/transformer.go @@ -7,8 +7,6 @@ import ( "regexp" ) -const startHighlightTag string = "" -const endHighlightTag string = "" // Transformer represents an instance of the ResponseTransformer interface type Transformer struct{} diff --git a/transformer/transformer_test.go b/transformer/transformer_test.go index b8a5cab8..73f95a6a 100644 --- a/transformer/transformer_test.go +++ b/transformer/transformer_test.go @@ -76,225 +76,6 @@ func TestTransform(t *testing.T) { }) } -func TestBuildMatches(t *testing.T) { - Convey("Build matches for Title translates successfully", t, func() { - hl := esHighlight{ - DescriptionTitle: &[]string{"value and double value"}, - } - - matches := buildMatches(hl) - So(matches, ShouldNotBeNil) - So(matches.Description, ShouldNotBeNil) - - So(matches.Description.Title, ShouldNotBeNil) - titleDetails := *matches.Description.Title - So(titleDetails, ShouldNotBeEmpty) - So(len(titleDetails), ShouldEqual, 2) - So(titleDetails[0].Value, ShouldBeBlank) - So(titleDetails[0].Start, ShouldEqual, 1) - So(titleDetails[0].End, ShouldEqual, 5) - So(titleDetails[1].Value, ShouldBeBlank) - So(titleDetails[1].Start, ShouldEqual, 18) - So(titleDetails[1].End, ShouldEqual, 22) - }) - - Convey("Build matches for Edition translates successfully", t, func() { - hl := esHighlight{ - DescriptionEdition: &[]string{"value"}, - } - - matches := buildMatches(hl) - So(matches, ShouldNotBeNil) - So(matches.Description, ShouldNotBeNil) - - So(matches.Description.Edition, ShouldNotBeNil) - editionDetails := *matches.Description.Edition - So(editionDetails, ShouldNotBeEmpty) - So(len(editionDetails), ShouldEqual, 1) - So(editionDetails[0].Value, ShouldBeBlank) - So(editionDetails[0].Start, ShouldEqual, 1) - So(editionDetails[0].End, ShouldEqual, 5) - - }) - - Convey("Build matches for Summary translates successfully", t, func() { - hl := esHighlight{ - DescriptionSummary: &[]string{"single value"}, - } - - matches := buildMatches(hl) - So(matches, ShouldNotBeNil) - So(matches.Description, ShouldNotBeNil) - - So(matches.Description.Summary, ShouldNotBeNil) - summaryDetails := *matches.Description.Summary - So(summaryDetails, ShouldNotBeEmpty) - So(len(summaryDetails), ShouldEqual, 1) - So(summaryDetails[0].Value, ShouldBeBlank) - So(summaryDetails[0].Start, ShouldEqual, 8) - So(summaryDetails[0].End, ShouldEqual, 12) - }) - - Convey("Build matches for MetaDescription translates successfully", t, func() { - hl := esHighlight{ - DescriptionMeta: &[]string{"a value here but not value here"}, - } - - matches := buildMatches(hl) - So(matches, ShouldNotBeNil) - So(matches.Description, ShouldNotBeNil) - - So(matches.Description.MetaDescription, ShouldNotBeNil) - metaDetails := *matches.Description.MetaDescription - So(metaDetails, ShouldNotBeEmpty) - So(len(metaDetails), ShouldEqual, 1) - So(metaDetails[0].Value, ShouldBeBlank) - So(metaDetails[0].Start, ShouldEqual, 38) - So(metaDetails[0].End, ShouldEqual, 42) - }) - - Convey("Build matches for Keywords translates successfully", t, func() { - hl := esHighlight{ - DescriptionKeywords: &[]string{"one value", "value another value third value", "value again"}, - } - - matches := buildMatches(hl) - So(matches, ShouldNotBeNil) - So(matches.Description, ShouldNotBeNil) - - So(matches.Description.Keywords, ShouldNotBeNil) - keywordsDetails := *matches.Description.Keywords - So(keywordsDetails, ShouldNotBeEmpty) - So(len(keywordsDetails), ShouldEqual, 5) - So(keywordsDetails[0].Value, ShouldResemble, "one value") - So(keywordsDetails[0].Start, ShouldEqual, 5) - So(keywordsDetails[0].End, ShouldEqual, 9) - So(keywordsDetails[1].Value, ShouldResemble, "value another value third value") - So(keywordsDetails[1].Start, ShouldEqual, 1) - So(keywordsDetails[1].End, ShouldEqual, 5) - So(keywordsDetails[2].Value, ShouldResemble, "value another value third value") - So(keywordsDetails[2].Start, ShouldEqual, 15) - So(keywordsDetails[2].End, ShouldEqual, 19) - So(keywordsDetails[3].Value, ShouldResemble, "value another value third value") - So(keywordsDetails[3].Start, ShouldEqual, 27) - So(keywordsDetails[3].End, ShouldEqual, 31) - So(keywordsDetails[4].Value, ShouldResemble, "value again") - So(keywordsDetails[4].Start, ShouldEqual, 1) - So(keywordsDetails[4].End, ShouldEqual, 5) - }) - - Convey("Build matches for DatasetID translates successfully", t, func() { - hl := esHighlight{ - DescriptionDatasetID: &[]string{" space before value and after "}, - } - - matches := buildMatches(hl) - So(matches, ShouldNotBeNil) - So(matches.Description, ShouldNotBeNil) - - So(matches.Description.DatasetID, ShouldNotBeNil) - dataSetDetails := *matches.Description.DatasetID - So(dataSetDetails, ShouldNotBeEmpty) - So(len(dataSetDetails), ShouldEqual, 1) - So(dataSetDetails[0].Value, ShouldBeBlank) - So(dataSetDetails[0].Start, ShouldEqual, 21) - So(dataSetDetails[0].End, ShouldEqual, 25) - }) - - Convey("Build matches for all items translates successfully", t, func() { - hl := esHighlight{ - DescriptionTitle: &[]string{"value and double value"}, - DescriptionEdition: &[]string{"value"}, - DescriptionSummary: &[]string{"single value"}, - DescriptionMeta: &[]string{"a value here but not value here"}, - DescriptionKeywords: &[]string{"one value", "value another value third value", "value again"}, - DescriptionDatasetID: &[]string{" space before value and after "}, - } - - matches := buildMatches(hl) - So(matches, ShouldNotBeNil) - So(matches.Description, ShouldNotBeNil) - - So(matches.Description.Title, ShouldNotBeNil) - So(matches.Description.Title, ShouldNotBeEmpty) - - So(matches.Description.Edition, ShouldNotBeNil) - So(matches.Description.Edition, ShouldNotBeEmpty) - - So(matches.Description.Summary, ShouldNotBeNil) - So(matches.Description.Summary, ShouldNotBeEmpty) - - So(matches.Description.MetaDescription, ShouldNotBeNil) - So(matches.Description.MetaDescription, ShouldNotBeEmpty) - - So(matches.Description.Keywords, ShouldNotBeNil) - So(matches.Description.Keywords, ShouldNotBeEmpty) - - So(matches.Description.DatasetID, ShouldNotBeNil) - So(matches.Description.DatasetID, ShouldNotBeEmpty) - }) -} - -func TestFindMatches(t *testing.T) { - Convey("Find matches successfully", t, func() { - - Convey("No value returns empty array", func() { - s := "nothing here" - md, fs := findMatches(s) - So(md, ShouldBeEmpty) - So(len(md), ShouldEqual, 0) - So(fs, ShouldResemble, "nothing here") - }) - - Convey("Single value finds successfully", func() { - s := "single value" - md, fs := findMatches(s) - So(md, ShouldNotBeEmpty) - So(len(md), ShouldEqual, 1) - So(fs, ShouldResemble, "single value") - So(md[0].Start, ShouldEqual, 8) - So(md[0].End, ShouldEqual, 12) - }) - - Convey("Double value finds successfully", func() { - s := "value and double value" - md, fs := findMatches(s) - So(fs, ShouldResemble, "value and double value") - So(md, ShouldNotBeEmpty) - So(len(md), ShouldEqual, 2) - So(md[0].Start, ShouldEqual, 1) - So(md[0].End, ShouldEqual, 5) - So(md[1].Start, ShouldEqual, 18) - So(md[1].End, ShouldEqual, 22) - }) - - Convey("Tripple values find successfully", func() { - s := "value and double value and tripple–value" - md, fs := findMatches(s) - So(fs, ShouldResemble, "value and double value and tripple–value") - So(md, ShouldNotBeEmpty) - So(len(md), ShouldEqual, 3) - So(md[0].Start, ShouldEqual, 1) - So(md[0].End, ShouldEqual, 5) - So(md[1].Start, ShouldEqual, 18) - So(md[1].End, ShouldEqual, 22) - So(md[2].Start, ShouldEqual, 38) // 38 bytes. After UTF8 character, actual characters = 36 - So(md[2].End, ShouldEqual, 42) // 42 bytes. After UTF8 character, actual characters = 40 - }) - - Convey("UTF8 characters counted as bytes", func() { - s := "€–value" - md, fs := findMatches(s) - So(fs, ShouldResemble, "€–value") - So(md, ShouldNotBeEmpty) - So(len(md), ShouldEqual, 1) - So(md[0].Start, ShouldEqual, 7) // Should be 5 if chars - So(md[0].End, ShouldEqual, 11) // Should be 11 if chars - }) - - }) -} - func TestBuildAdditionalSuggestionsList(t *testing.T) { Convey("buildAdditionalSuggestionList successfully", t, func() { From a83f8512bc1463d0c09216b7000b7b44dd33c970 Mon Sep 17 00:00:00 2001 From: Noelle Date: Thu, 19 Aug 2021 15:09:50 +0100 Subject: [PATCH 04/11] make test pass --- swagger.yaml | 29 +++++++++++++++-- transformer/testdata/search_example.json | 2 +- transformer/testdata/search_expected.json | 39 ++++++++++++----------- 3 files changed, 48 insertions(+), 22 deletions(-) diff --git a/swagger.yaml b/swagger.yaml index 185f27a4..dc84c1f4 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -304,9 +304,32 @@ definitions: - summary - title matches: - type: array - items: - type: string + type: object + properties: + description_title: + type: array + items: + type: string + description_edition: + type: array + items: + type: string + description_summary: + type: array + items: + type: string + description_meta: + type: array + items: + type: string + description_keywords: + type: array + items: + type: string + description_dataset_id: + type: array + items: + type: string type: type: string uri: diff --git a/transformer/testdata/search_example.json b/transformer/testdata/search_example.json index f9608cf8..1ad1c843 100644 --- a/transformer/testdata/search_example.json +++ b/transformer/testdata/search_example.json @@ -51,7 +51,7 @@ ], "description.keywords": [ "regional house prices", - "area with cheapest >houses", + "area with cheapest houses", "area with most expensive houses" ] }, diff --git a/transformer/testdata/search_expected.json b/transformer/testdata/search_expected.json index 52b47f39..bef172f4 100644 --- a/transformer/testdata/search_expected.json +++ b/transformer/testdata/search_expected.json @@ -36,24 +36,18 @@ "unit": "" }, "matches": { - "description": { - "summary": null, - "title": [ - { - } + "description.title": [ + "House Price Statistics for Small Areas in England and Wales" ], - "keywords": [ - { - "value": "regional house prices" - }, - { - "value": "area with cheapest houses" - }, - { - "value": "area with most expensive houses" - } - ] - } + "description.edition": null, + "description.summary": null, + "description.metaDescription": null, + "description.keywords": [ + "regional house prices", + "area with cheapest houses", + "area with most expensive houses" + ], + "description.datasetId": null }, "type": "article", "uri": "/peoplepopulationandcommunity/housing/articles/housepricestatisticsforsmallareasinenglandandwales/2015-02-17" @@ -68,7 +62,16 @@ }, "type": "product_page", "uri": "/peoplepopulationandcommunity/housing", - "matches": [] + "matches": { + "description.title": [ + "Housing" + ], + "description.edition": null, + "description.summary": null, + "description.metaDescription": null, + "description.keywords": null, + "description.datasetId": null + } } ], "suggestions": [ From c8e95f4c7e0bad0199427a74b26f0aa485f6fc20 Mon Sep 17 00:00:00 2001 From: Noelle Date: Thu, 19 Aug 2021 15:12:18 +0100 Subject: [PATCH 05/11] update swagger spec --- swagger.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/swagger.yaml b/swagger.yaml index dc84c1f4..e1b88b4b 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -306,27 +306,27 @@ definitions: matches: type: object properties: - description_title: + description.title: type: array items: type: string - description_edition: + description.edition: type: array items: type: string - description_summary: + description.summary: type: array items: type: string - description_meta: + description.metaDesription: type: array items: type: string - description_keywords: + description.keywords: type: array items: type: string - description_dataset_id: + description.datasetId: type: array items: type: string From 78f747bb3e8f7ba524384bf791103ece01d1b3b0 Mon Sep 17 00:00:00 2001 From: Noelle Date: Fri, 20 Aug 2021 10:22:22 +0100 Subject: [PATCH 06/11] remove unnecessary code --- swagger.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/swagger.yaml b/swagger.yaml index e1b88b4b..4c7d1592 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -417,13 +417,6 @@ definitions: type: string _type: type: string - _highlight: - type: object - properties: - terms: - type: array - items: - type: string TimeseriesResponse: type: object From 4191a24f4bb56f232540ceb27e0a7dbdf423928e Mon Sep 17 00:00:00 2001 From: Noelle Date: Tue, 24 Aug 2021 09:34:39 +0100 Subject: [PATCH 07/11] add class to match ONS styling --- templates/search/contentQuery.tmpl | 2 +- templates/search/departmentsQuery.tmpl | 2 +- transformer/testdata/search_example.json | 10 +++++----- transformer/testdata/search_expected.json | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/templates/search/contentQuery.tmpl b/templates/search/contentQuery.tmpl index a5049f37..81dfdbd6 100644 --- a/templates/search/contentQuery.tmpl +++ b/templates/search/contentQuery.tmpl @@ -26,7 +26,7 @@ "excludes":["downloads.content","downloads*","pageData"]}, {{if .Highlight}} "highlight":{ - "pre_tags":[""], + "pre_tags":[""], "post_tags":[""], "fields":{"terms":{"fragment_size":0,"number_of_fragments":0}, "description.title":{"fragment_size":0,"number_of_fragments":0}, diff --git a/templates/search/departmentsQuery.tmpl b/templates/search/departmentsQuery.tmpl index 55a3fe0a..f9c67b8e 100644 --- a/templates/search/departmentsQuery.tmpl +++ b/templates/search/departmentsQuery.tmpl @@ -11,7 +11,7 @@ }, "highlight": { "pre_tags": [ - "" + "" ], "post_tags": [ "" diff --git a/transformer/testdata/search_example.json b/transformer/testdata/search_example.json index 1ad1c843..0dbe4457 100644 --- a/transformer/testdata/search_example.json +++ b/transformer/testdata/search_example.json @@ -47,12 +47,12 @@ }, "highlight": { "description.title": [ - "House Price Statistics for Small Areas in England and Wales" + "House Price Statistics for Small Areas in England and Wales" ], "description.keywords": [ - "regional house prices", - "area with cheapest houses", - "area with most expensive houses" + "regional house prices", + "area with cheapest houses", + "area with most expensive houses" ] }, "sort": [ @@ -79,7 +79,7 @@ }, "highlight": { "description.title": [ - "Housing" + "Housing" ] }, "sort": [ diff --git a/transformer/testdata/search_expected.json b/transformer/testdata/search_expected.json index bef172f4..7fde02d2 100644 --- a/transformer/testdata/search_expected.json +++ b/transformer/testdata/search_expected.json @@ -37,15 +37,15 @@ }, "matches": { "description.title": [ - "House Price Statistics for Small Areas in England and Wales" + "House Price Statistics for Small Areas in England and Wales" ], "description.edition": null, "description.summary": null, "description.metaDescription": null, "description.keywords": [ - "regional house prices", - "area with cheapest houses", - "area with most expensive houses" + "regional house prices", + "area with cheapest houses", + "area with most expensive houses" ], "description.datasetId": null }, @@ -64,7 +64,7 @@ "uri": "/peoplepopulationandcommunity/housing", "matches": { "description.title": [ - "Housing" + "Housing" ], "description.edition": null, "description.summary": null, From 6c763b2147b55e08fcdddb15ef4a3252fce94975 Mon Sep 17 00:00:00 2001 From: Noelle Date: Mon, 6 Sep 2021 15:11:05 +0100 Subject: [PATCH 08/11] added highlighting to the transformer --- api/api.go | 4 +- api/mocks.go | 143 +++++++++--------- api/search.go | 3 +- api/search_test.go | 2 +- transformer/testdata/search_example.json | 14 +- .../testdata/search_expected_highlighted.json | 58 +++++++ ...pected.json => search_expected_plain.json} | 28 +--- transformer/transformer.go | 73 ++++++--- transformer/transformer_test.go | 35 +++-- 9 files changed, 224 insertions(+), 136 deletions(-) create mode 100644 transformer/testdata/search_expected_highlighted.json rename transformer/testdata/{search_expected.json => search_expected_plain.json} (58%) diff --git a/api/api.go b/api/api.go index 733028b5..340c0cb9 100644 --- a/api/api.go +++ b/api/api.go @@ -39,7 +39,7 @@ type QueryBuilder interface { // ResponseTransformer provides methods for the transform package type ResponseTransformer interface { - TransformSearchResponse(ctx context.Context, responseData []byte, query string) ([]byte, error) + TransformSearchResponse(ctx context.Context, responseData []byte, query string, highlight bool) ([]byte, error) } // CreateAndInitialise initiates a new Search API @@ -110,5 +110,3 @@ func Close(ctx context.Context) error { log.Event(ctx, "graceful shutdown of http server complete", log.INFO) return nil } - - diff --git a/api/mocks.go b/api/mocks.go index 11a7db60..4101e250 100644 --- a/api/mocks.go +++ b/api/mocks.go @@ -8,37 +8,31 @@ import ( "sync" ) -var ( - lockElasticSearcherMockGetStatus sync.RWMutex - lockElasticSearcherMockMultiSearch sync.RWMutex - lockElasticSearcherMockSearch sync.RWMutex -) - // Ensure, that ElasticSearcherMock does implement ElasticSearcher. // If this is not the case, regenerate this file with moq. var _ ElasticSearcher = &ElasticSearcherMock{} // ElasticSearcherMock is a mock implementation of ElasticSearcher. // -// func TestSomethingThatUsesElasticSearcher(t *testing.T) { +// func TestSomethingThatUsesElasticSearcher(t *testing.T) { // -// // make and configure a mocked ElasticSearcher -// mockedElasticSearcher := &ElasticSearcherMock{ -// GetStatusFunc: func(ctx context.Context) ([]byte, error) { -// panic("mock out the GetStatus method") -// }, -// MultiSearchFunc: func(ctx context.Context, index string, docType string, request []byte) ([]byte, error) { -// panic("mock out the MultiSearch method") -// }, -// SearchFunc: func(ctx context.Context, index string, docType string, request []byte) ([]byte, error) { -// panic("mock out the Search method") -// }, -// } +// // make and configure a mocked ElasticSearcher +// mockedElasticSearcher := &ElasticSearcherMock{ +// GetStatusFunc: func(ctx context.Context) ([]byte, error) { +// panic("mock out the GetStatus method") +// }, +// MultiSearchFunc: func(ctx context.Context, index string, docType string, request []byte) ([]byte, error) { +// panic("mock out the MultiSearch method") +// }, +// SearchFunc: func(ctx context.Context, index string, docType string, request []byte) ([]byte, error) { +// panic("mock out the Search method") +// }, +// } // -// // use mockedElasticSearcher in code that requires ElasticSearcher -// // and then make assertions. +// // use mockedElasticSearcher in code that requires ElasticSearcher +// // and then make assertions. // -// } +// } type ElasticSearcherMock struct { // GetStatusFunc mocks the GetStatus method. GetStatusFunc func(ctx context.Context) ([]byte, error) @@ -79,6 +73,9 @@ type ElasticSearcherMock struct { Request []byte } } + lockGetStatus sync.RWMutex + lockMultiSearch sync.RWMutex + lockSearch sync.RWMutex } // GetStatus calls GetStatusFunc. @@ -91,9 +88,9 @@ func (mock *ElasticSearcherMock) GetStatus(ctx context.Context) ([]byte, error) }{ Ctx: ctx, } - lockElasticSearcherMockGetStatus.Lock() + mock.lockGetStatus.Lock() mock.calls.GetStatus = append(mock.calls.GetStatus, callInfo) - lockElasticSearcherMockGetStatus.Unlock() + mock.lockGetStatus.Unlock() return mock.GetStatusFunc(ctx) } @@ -106,9 +103,9 @@ func (mock *ElasticSearcherMock) GetStatusCalls() []struct { var calls []struct { Ctx context.Context } - lockElasticSearcherMockGetStatus.RLock() + mock.lockGetStatus.RLock() calls = mock.calls.GetStatus - lockElasticSearcherMockGetStatus.RUnlock() + mock.lockGetStatus.RUnlock() return calls } @@ -128,9 +125,9 @@ func (mock *ElasticSearcherMock) MultiSearch(ctx context.Context, index string, DocType: docType, Request: request, } - lockElasticSearcherMockMultiSearch.Lock() + mock.lockMultiSearch.Lock() mock.calls.MultiSearch = append(mock.calls.MultiSearch, callInfo) - lockElasticSearcherMockMultiSearch.Unlock() + mock.lockMultiSearch.Unlock() return mock.MultiSearchFunc(ctx, index, docType, request) } @@ -149,9 +146,9 @@ func (mock *ElasticSearcherMock) MultiSearchCalls() []struct { DocType string Request []byte } - lockElasticSearcherMockMultiSearch.RLock() + mock.lockMultiSearch.RLock() calls = mock.calls.MultiSearch - lockElasticSearcherMockMultiSearch.RUnlock() + mock.lockMultiSearch.RUnlock() return calls } @@ -171,9 +168,9 @@ func (mock *ElasticSearcherMock) Search(ctx context.Context, index string, docTy DocType: docType, Request: request, } - lockElasticSearcherMockSearch.Lock() + mock.lockSearch.Lock() mock.calls.Search = append(mock.calls.Search, callInfo) - lockElasticSearcherMockSearch.Unlock() + mock.lockSearch.Unlock() return mock.SearchFunc(ctx, index, docType, request) } @@ -192,35 +189,31 @@ func (mock *ElasticSearcherMock) SearchCalls() []struct { DocType string Request []byte } - lockElasticSearcherMockSearch.RLock() + mock.lockSearch.RLock() calls = mock.calls.Search - lockElasticSearcherMockSearch.RUnlock() + mock.lockSearch.RUnlock() return calls } -var ( - lockQueryBuilderMockBuildSearchQuery sync.RWMutex -) - // Ensure, that QueryBuilderMock does implement QueryBuilder. // If this is not the case, regenerate this file with moq. var _ QueryBuilder = &QueryBuilderMock{} // QueryBuilderMock is a mock implementation of QueryBuilder. // -// func TestSomethingThatUsesQueryBuilder(t *testing.T) { +// func TestSomethingThatUsesQueryBuilder(t *testing.T) { // -// // make and configure a mocked QueryBuilder -// mockedQueryBuilder := &QueryBuilderMock{ -// BuildSearchQueryFunc: func(ctx context.Context, q string, contentTypes string, sort string, limit int, offset int) ([]byte, error) { -// panic("mock out the BuildSearchQuery method") -// }, -// } +// // make and configure a mocked QueryBuilder +// mockedQueryBuilder := &QueryBuilderMock{ +// BuildSearchQueryFunc: func(ctx context.Context, q string, contentTypes string, sort string, limit int, offset int) ([]byte, error) { +// panic("mock out the BuildSearchQuery method") +// }, +// } // -// // use mockedQueryBuilder in code that requires QueryBuilder -// // and then make assertions. +// // use mockedQueryBuilder in code that requires QueryBuilder +// // and then make assertions. // -// } +// } type QueryBuilderMock struct { // BuildSearchQueryFunc mocks the BuildSearchQuery method. BuildSearchQueryFunc func(ctx context.Context, q string, contentTypes string, sort string, limit int, offset int) ([]byte, error) @@ -243,6 +236,7 @@ type QueryBuilderMock struct { Offset int } } + lockBuildSearchQuery sync.RWMutex } // BuildSearchQuery calls BuildSearchQueryFunc. @@ -265,9 +259,9 @@ func (mock *QueryBuilderMock) BuildSearchQuery(ctx context.Context, q string, co Limit: limit, Offset: offset, } - lockQueryBuilderMockBuildSearchQuery.Lock() + mock.lockBuildSearchQuery.Lock() mock.calls.BuildSearchQuery = append(mock.calls.BuildSearchQuery, callInfo) - lockQueryBuilderMockBuildSearchQuery.Unlock() + mock.lockBuildSearchQuery.Unlock() return mock.BuildSearchQueryFunc(ctx, q, contentTypes, sort, limit, offset) } @@ -290,38 +284,34 @@ func (mock *QueryBuilderMock) BuildSearchQueryCalls() []struct { Limit int Offset int } - lockQueryBuilderMockBuildSearchQuery.RLock() + mock.lockBuildSearchQuery.RLock() calls = mock.calls.BuildSearchQuery - lockQueryBuilderMockBuildSearchQuery.RUnlock() + mock.lockBuildSearchQuery.RUnlock() return calls } -var ( - lockResponseTransformerMockTransformSearchResponse sync.RWMutex -) - // Ensure, that ResponseTransformerMock does implement ResponseTransformer. // If this is not the case, regenerate this file with moq. var _ ResponseTransformer = &ResponseTransformerMock{} // ResponseTransformerMock is a mock implementation of ResponseTransformer. // -// func TestSomethingThatUsesResponseTransformer(t *testing.T) { +// func TestSomethingThatUsesResponseTransformer(t *testing.T) { // -// // make and configure a mocked ResponseTransformer -// mockedResponseTransformer := &ResponseTransformerMock{ -// TransformSearchResponseFunc: func(ctx context.Context, responseData []byte, query string) ([]byte, error) { -// panic("mock out the TransformSearchResponse method") -// }, -// } +// // make and configure a mocked ResponseTransformer +// mockedResponseTransformer := &ResponseTransformerMock{ +// TransformSearchResponseFunc: func(ctx context.Context, responseData []byte, query string, highlight bool) ([]byte, error) { +// panic("mock out the TransformSearchResponse method") +// }, +// } // -// // use mockedResponseTransformer in code that requires ResponseTransformer -// // and then make assertions. +// // use mockedResponseTransformer in code that requires ResponseTransformer +// // and then make assertions. // -// } +// } type ResponseTransformerMock struct { // TransformSearchResponseFunc mocks the TransformSearchResponse method. - TransformSearchResponseFunc func(ctx context.Context, responseData []byte, query string) ([]byte, error) + TransformSearchResponseFunc func(ctx context.Context, responseData []byte, query string, highlight bool) ([]byte, error) // calls tracks calls to the methods. calls struct { @@ -333,12 +323,15 @@ type ResponseTransformerMock struct { ResponseData []byte // Query is the query argument value. Query string + // Highlight is the highlight argument value. + Highlight bool } } + lockTransformSearchResponse sync.RWMutex } // TransformSearchResponse calls TransformSearchResponseFunc. -func (mock *ResponseTransformerMock) TransformSearchResponse(ctx context.Context, responseData []byte, query string) ([]byte, error) { +func (mock *ResponseTransformerMock) TransformSearchResponse(ctx context.Context, responseData []byte, query string, highlight bool) ([]byte, error) { if mock.TransformSearchResponseFunc == nil { panic("ResponseTransformerMock.TransformSearchResponseFunc: method is nil but ResponseTransformer.TransformSearchResponse was just called") } @@ -346,15 +339,17 @@ func (mock *ResponseTransformerMock) TransformSearchResponse(ctx context.Context Ctx context.Context ResponseData []byte Query string + Highlight bool }{ Ctx: ctx, ResponseData: responseData, Query: query, + Highlight: highlight, } - lockResponseTransformerMockTransformSearchResponse.Lock() + mock.lockTransformSearchResponse.Lock() mock.calls.TransformSearchResponse = append(mock.calls.TransformSearchResponse, callInfo) - lockResponseTransformerMockTransformSearchResponse.Unlock() - return mock.TransformSearchResponseFunc(ctx, responseData, query) + mock.lockTransformSearchResponse.Unlock() + return mock.TransformSearchResponseFunc(ctx, responseData, query, highlight) } // TransformSearchResponseCalls gets all the calls that were made to TransformSearchResponse. @@ -364,14 +359,16 @@ func (mock *ResponseTransformerMock) TransformSearchResponseCalls() []struct { Ctx context.Context ResponseData []byte Query string + Highlight bool } { var calls []struct { Ctx context.Context ResponseData []byte Query string + Highlight bool } - lockResponseTransformerMockTransformSearchResponse.RLock() + mock.lockTransformSearchResponse.RLock() calls = mock.calls.TransformSearchResponse - lockResponseTransformerMockTransformSearchResponse.RUnlock() + mock.lockTransformSearchResponse.RUnlock() return calls } diff --git a/api/search.go b/api/search.go index 00722eff..7a3ad7d3 100644 --- a/api/search.go +++ b/api/search.go @@ -111,7 +111,8 @@ func SearchHandlerFunc(queryBuilder QueryBuilder, elasticSearchClient ElasticSea } if !paramGetBool(params, "raw", false) { - responseData, err = transformer.TransformSearchResponse(ctx, responseData, q) + //TODO - determine whether to return highlighted response or not + responseData, err = transformer.TransformSearchResponse(ctx, responseData, q, true) if err != nil { log.Event(ctx, "transformation of response data failed", log.Error(err), log.ERROR) http.Error(w, "Failed to transform search result", http.StatusInternalServerError) diff --git a/api/search_test.go b/api/search_test.go index d8ee2293..372627f0 100644 --- a/api/search_test.go +++ b/api/search_test.go @@ -276,7 +276,7 @@ func newQueryBuilderMock(query []byte, err error) *QueryBuilderMock { func newResponseTransformerMock(response []byte, err error) *ResponseTransformerMock { return &ResponseTransformerMock{ - TransformSearchResponseFunc: func(ctx context.Context, responseData []byte, query string) ([]byte, error) { + TransformSearchResponseFunc: func(ctx context.Context, responseData []byte, query string, highlight bool) ([]byte, error) { return response, err }, } diff --git a/transformer/testdata/search_example.json b/transformer/testdata/search_example.json index 0dbe4457..5f21d0be 100644 --- a/transformer/testdata/search_example.json +++ b/transformer/testdata/search_example.json @@ -36,7 +36,7 @@ "edition": "1995 to 2013", "source": "", "title": "House Price Statistics for Small Areas in England and Wales", - "metaDescription": "A brief overview of average (median) house prices using estimates from the sale prices of residential properties in England and Wales between 1995 and 2013. We look at the trends and features of average house prices paid over this period for 3 different geographies: local authorities, parliamentary constituencies and middle layer super output areas (MSOAs).", + "metaDescription": "A brief overview of average (median) house prices using estimates from the sale prices of residential properties in England and Wales between 1995 and 2013.", "nationalStatistic": false, "_abstract": "A brief overview of average (median) house prices using estimates from the sale prices of residential properties in England and Wales between 1995 and 2013. We look at the trends and features of average house prices paid over this period for 3 different geographies: local authorities, parliamentary constituencies and middle layer super output areas (MSOAs).", "preUnit": "" @@ -49,10 +49,22 @@ "description.title": [ "House Price Statistics for Small Areas in England and Wales" ], + "description.edition": [ + "1995 to 2013" + ], + "description.summary": [ + "regional house prices" + ], + "description.metaDescription": [ + "A brief overview of average (median) house prices using estimates from the sale prices of residential properties in England and Wales between 1995 and 2013." + ], "description.keywords": [ "regional house prices", "area with cheapest houses", "area with most expensive houses" + ], + "description.datasetId": [ + "/peoplepopulationandcommunity/housing/articles/housepricestatisticsforsmallareasinenglandandwales/2015-02-17" ] }, "sort": [ diff --git a/transformer/testdata/search_expected_highlighted.json b/transformer/testdata/search_expected_highlighted.json new file mode 100644 index 00000000..7dbd25cc --- /dev/null +++ b/transformer/testdata/search_expected_highlighted.json @@ -0,0 +1,58 @@ +{ + "count": 2, + "took": 96, + "content_types": [ + { + "type": "article", + "count": 1 + }, + { + "type": "product_page", + "count": 1 + } + ], + "items": [ + { + "description": { + "contact": { + "email": "somebody@ons.gov.uk", + "name": "Somebody's Name", + "telephone": "+44 (0)1329 123456" + }, + "dataset_id": "/peoplepopulationandcommunity/housing/articles/housepricestatisticsforsmallareasinenglandandwales/2015-02-17", + "edition": "1995 to 2013", + "keywords": [ + "regional house prices", + "property prices", + "area with cheapest houses", + "area with most expensive houses" + ], + "latest_release": true, + "meta_description": "A brief overview of average (median) house prices using estimates from the sale prices of residential properties in England and Wales between 1995 and 2013.", + "national_statistic": false, + "release_date": "2015-02-17T00:00:00.000Z", + "source": "", + "summary": "regional house prices", + "title": "House Price Statistics for Small Areas in England and Wales", + "unit": "" + }, + "type": "article", + "uri": "/peoplepopulationandcommunity/housing/articles/housepricestatisticsforsmallareasinenglandandwales/2015-02-17" + }, + { + "description": { + "summary": "Data about property prices. These are an informal indicator of the confidence people feel in the UK economy. We collect the data from a sample of mortgage completions, so we do not include property sold to cash buyers.", + "title": "Housing", + "unit": "", + "source": "", + "pre_unit": "" + }, + "type": "product_page", + "uri": "/peoplepopulationandcommunity/housing" + } + ], + "suggestions": [ + "home", + "tent" + ] +} \ No newline at end of file diff --git a/transformer/testdata/search_expected.json b/transformer/testdata/search_expected_plain.json similarity index 58% rename from transformer/testdata/search_expected.json rename to transformer/testdata/search_expected_plain.json index 7fde02d2..5b68da58 100644 --- a/transformer/testdata/search_expected.json +++ b/transformer/testdata/search_expected_plain.json @@ -27,7 +27,7 @@ "area with most expensive houses" ], "latest_release": true, - "meta_description": "A brief overview of average (median) house prices using estimates from the sale prices of residential properties in England and Wales between 1995 and 2013. We look at the trends and features of average house prices paid over this period for 3 different geographies: local authorities, parliamentary constituencies and middle layer super output areas (MSOAs).", + "meta_description": "A brief overview of average (median) house prices using estimates from the sale prices of residential properties in England and Wales between 1995 and 2013.", "national_statistic": false, "release_date": "2015-02-17T00:00:00.000Z", "source": "", @@ -35,20 +35,6 @@ "title": "House Price Statistics for Small Areas in England and Wales", "unit": "" }, - "matches": { - "description.title": [ - "House Price Statistics for Small Areas in England and Wales" - ], - "description.edition": null, - "description.summary": null, - "description.metaDescription": null, - "description.keywords": [ - "regional house prices", - "area with cheapest houses", - "area with most expensive houses" - ], - "description.datasetId": null - }, "type": "article", "uri": "/peoplepopulationandcommunity/housing/articles/housepricestatisticsforsmallareasinenglandandwales/2015-02-17" }, @@ -61,17 +47,7 @@ "pre_unit": "" }, "type": "product_page", - "uri": "/peoplepopulationandcommunity/housing", - "matches": { - "description.title": [ - "Housing" - ], - "description.edition": null, - "description.summary": null, - "description.metaDescription": null, - "description.keywords": null, - "description.datasetId": null - } + "uri": "/peoplepopulationandcommunity/housing" } ], "suggestions": [ diff --git a/transformer/transformer.go b/transformer/transformer.go index 6eeec5f1..819d8a88 100644 --- a/transformer/transformer.go +++ b/transformer/transformer.go @@ -5,11 +5,13 @@ import ( "encoding/json" "github.com/pkg/errors" "regexp" + "strings" ) - // Transformer represents an instance of the ResponseTransformer interface -type Transformer struct{} +type Transformer struct { + higlightReplacer strings.Replacer +} // Structs representing the transformed response type searchResponse struct { @@ -27,10 +29,9 @@ type contentType struct { } type contentItem struct { - Description description `json:"description"` - Type string `json:"type"` - URI string `json:"uri"` - Matches *esHighlight `json:"matches,omitempty"` + Description description `json:"description"` + Type string `json:"type"` + URI string `json:"uri"` } type description struct { @@ -142,11 +143,14 @@ type esSearchSuggestOptions struct { // New returns a new instance of Transformer func New() *Transformer { - return &Transformer{} + highlightReplacer := strings.NewReplacer("", "", "", "") + return &Transformer{ + higlightReplacer: *highlightReplacer, + } } // TransformSearchResponse transforms an elastic search response into a structure that matches the v1 api specification -func (t *Transformer) TransformSearchResponse(ctx context.Context, responseData []byte, query string) ([]byte, error) { +func (t *Transformer) TransformSearchResponse(ctx context.Context, responseData []byte, query string, highlight bool) ([]byte, error) { var source esResponse err := json.Unmarshal(responseData, &source) @@ -158,7 +162,7 @@ func (t *Transformer) TransformSearchResponse(ctx context.Context, responseData return nil, errors.New("Response to be transformed contained 0 items") } - sr := transform(&source) + sr := t.transform(&source, highlight) if sr.Count == 0 { as := buildAdditionalSuggestionList(query) @@ -172,7 +176,7 @@ func (t *Transformer) TransformSearchResponse(ctx context.Context, responseData return transformedData, nil } -func transform(source *esResponse) searchResponse { +func (t *Transformer) transform(source *esResponse, highlight bool) searchResponse { sr := searchResponse{ Count: source.Responses[0].Hits.Total, Items: []contentItem{}, @@ -181,7 +185,7 @@ func transform(source *esResponse) searchResponse { var took int = 0 for _, response := range source.Responses { for _, doc := range response.Hits.Hits { - sr.Items = append(sr.Items, buildContentItem(doc)) + sr.Items = append(sr.Items, t.buildContentItem(doc, highlight)) } for _, bucket := range response.Aggregations.DocCounts.Buckets { sr.ContentTypes = append(sr.ContentTypes, buildContentTypes(bucket)) @@ -197,34 +201,35 @@ func transform(source *esResponse) searchResponse { return sr } -func buildContentItem(doc esResponseHit) contentItem { +func (t *Transformer) buildContentItem(doc esResponseHit, highlight bool) contentItem { ci := contentItem{ - Description: buildDescription(doc), + Description: t.buildDescription(doc, highlight), Type: doc.Source.Type, URI: doc.Source.URI, - Matches: &doc.Highlight, } return ci } -func buildDescription(doc esResponseHit) description { +func (t *Transformer) buildDescription(doc esResponseHit, highlight bool) description { sd := doc.Source.Description + hl := doc.Highlight + return description{ - Summary: sd.Summary, + Summary: t.overlaySingleItem(hl.DescriptionSummary, sd.Summary, highlight), NextRelease: sd.NextRelease, Unit: sd.Unit, PreUnit: sd.PreUnit, - Keywords: sd.Keywords, + Keywords: t.overlayItemList(hl.DescriptionKeywords, sd.Keywords, highlight), ReleaseDate: sd.ReleaseDate, - Edition: sd.Edition, + Edition: t.overlaySingleItem(hl.DescriptionEdition, sd.Edition, highlight), LatestRelease: sd.LatestRelease, Language: sd.Language, Contact: sd.Contact, - DatasetID: sd.DatasetID, + DatasetID: t.overlaySingleItem(hl.DescriptionDatasetID, sd.DatasetID, highlight), Source: sd.Source, - Title: sd.Title, - MetaDescription: sd.MetaDescription, + Title: t.overlaySingleItem(hl.DescriptionTitle, sd.Title, highlight), + MetaDescription: t.overlaySingleItem(hl.DescriptionMeta, sd.MetaDescription, highlight), NationalStatistic: sd.NationalStatistic, Headline1: sd.Headline1, Headline2: sd.Headline2, @@ -232,6 +237,32 @@ func buildDescription(doc esResponseHit) description { } } +func (t *Transformer) overlaySingleItem(hl *[]string, def string, highlight bool) string { + overlaid := def + if highlight && hl != nil && len(*hl) > 0 { + overlaid = (*hl)[0] + } + return overlaid +} + +func (t *Transformer) overlayItemList(hlList *[]string, defaultList *[]string, highlight bool) *[]string { + if defaultList == nil { + return nil + } + overlaid := *defaultList + if highlight && hlList != nil { + for _, hl := range *hlList { + unformatted := t.higlightReplacer.Replace(hl) + for i, defItem := range overlaid { + if defItem == unformatted { + overlaid[i] = hl + } + } + } + } + return &overlaid +} + func buildContentTypes(bucket esBucket) contentType { return contentType{ Type: bucket.Key, diff --git a/transformer/transformer_test.go b/transformer/transformer_test.go index 73f95a6a..bf02bacd 100644 --- a/transformer/transformer_test.go +++ b/transformer/transformer_test.go @@ -11,6 +11,7 @@ import ( func TestTransform(t *testing.T) { Convey("Transforms unmarshalled search responses successfully", t, func() { + transformer := New() Convey("Zero suggestions creates empty array", func() { es := esResponse{ Responses: []esResponseItem{esResponseItem{ @@ -21,7 +22,7 @@ func TestTransform(t *testing.T) { }, }}, } - sr := transform(&es) + sr := transformer.transform(&es, false) So(sr.Suggestions, ShouldBeEmpty) }) @@ -37,7 +38,7 @@ func TestTransform(t *testing.T) { }, }}, } - sr := transform(&es) + sr := transformer.transform(&es, true) So(sr.Suggestions, ShouldNotBeEmpty) So(len(sr.Suggestions), ShouldEqual, 1) So(sr.Suggestions[0], ShouldResemble, "option1") @@ -66,7 +67,7 @@ func TestTransform(t *testing.T) { }, }}, } - sr := transform(&es) + sr := transformer.transform(&es, true) So(sr.Suggestions, ShouldNotBeEmpty) So(len(sr.Suggestions), ShouldEqual, 3) So(sr.Suggestions[0], ShouldResemble, "option1") @@ -101,37 +102,51 @@ func TestBuildAdditionalSuggestionsList(t *testing.T) { func TestTransformSearchResponse(t *testing.T) { Convey("With a transformer initialised", t, func() { ctx := context.Background() - t := New() + transformer := New() So(t, ShouldNotBeNil) Convey("Throws error on invalid JSON", func() { sampleResponse := []byte(`{"invalid":"json"`) - _, err := t.TransformSearchResponse(ctx, sampleResponse, "test-query") + _, err := transformer.TransformSearchResponse(ctx, sampleResponse, "test-query", true) So(err, ShouldNotBeNil) So(err.Error(), ShouldResemble, "Failed to decode elastic search response: unexpected end of JSON input") }) Convey("Handles missing responses", func() { sampleResponse := []byte(`{}`) - _, err := t.TransformSearchResponse(ctx, sampleResponse, "test-query") + _, err := transformer.TransformSearchResponse(ctx, sampleResponse, "test-query", true) So(err, ShouldNotBeNil) So(err.Error(), ShouldResemble, "Response to be transformed contained 0 items") }) - Convey("Converts an example response", func() { + Convey("Converts an example response with highlighting", func() { sampleResponse, err := ioutil.ReadFile("testdata/search_example.json") So(err, ShouldBeNil) - expected, err := ioutil.ReadFile("testdata/search_expected.json") + expected, err := ioutil.ReadFile("testdata/search_expected_highlighted.json") So(err, ShouldBeNil) - actual, err := t.TransformSearchResponse(ctx, sampleResponse, "test-query") + actual, err := transformer.TransformSearchResponse(ctx, sampleResponse, "test-query", true) So(err, ShouldBeNil) So(actual, ShouldNotBeEmpty) var exp, act searchResponse So(json.Unmarshal(expected, &exp), ShouldBeNil) So(json.Unmarshal(actual, &act), ShouldBeNil) So(act, ShouldResemble, exp) + }) + + Convey("Converts an example response without highlighting", func() { + sampleResponse, err := ioutil.ReadFile("testdata/search_example.json") + So(err, ShouldBeNil) + expected, err := ioutil.ReadFile("testdata/search_expected_plain.json") + So(err, ShouldBeNil) + actual, err := transformer.TransformSearchResponse(ctx, sampleResponse, "test-query", false) + So(err, ShouldBeNil) + So(actual, ShouldNotBeEmpty) + var exp, act searchResponse + So(json.Unmarshal(expected, &exp), ShouldBeNil) + So(json.Unmarshal(actual, &act), ShouldBeNil) + So(act, ShouldResemble, exp) }) Convey("Calls buildAdditionalSuggestionsList if zero search results", func() { @@ -140,7 +155,7 @@ func TestTransformSearchResponse(t *testing.T) { expected, err := ioutil.ReadFile("testdata/zero_search_expected.json") So(err, ShouldBeNil) - actual, err := t.TransformSearchResponse(ctx, sampleResponse, "test query \"with quote marks\"") + actual, err := transformer.TransformSearchResponse(ctx, sampleResponse, "test query \"with quote marks\"", false) So(err, ShouldBeNil) So(actual, ShouldNotBeEmpty) var exp, act searchResponse From 8ad091e34c796f910d2a78412db0b457acc50e7f Mon Sep 17 00:00:00 2001 From: Noelle Date: Mon, 6 Sep 2021 15:45:37 +0100 Subject: [PATCH 09/11] updated tests, swagger spec and api to accept highlighting param --- api/search.go | 5 +++-- api/search_test.go | 51 ++++++++++++++++++++++++++++++++++++++++++++++ swagger.yaml | 33 ++++++------------------------ 3 files changed, 60 insertions(+), 29 deletions(-) diff --git a/api/search.go b/api/search.go index 7a3ad7d3..377125f3 100644 --- a/api/search.go +++ b/api/search.go @@ -50,6 +50,8 @@ func SearchHandlerFunc(queryBuilder QueryBuilder, elasticSearchClient ElasticSea q := params.Get("q") sort := paramGet(params, "sort", "relevance") + highlight := paramGetBool(params, "highlight", true) + limitParam := paramGet(params, "limit", "10") limit, err := strconv.Atoi(limitParam) if err != nil { @@ -111,8 +113,7 @@ func SearchHandlerFunc(queryBuilder QueryBuilder, elasticSearchClient ElasticSea } if !paramGetBool(params, "raw", false) { - //TODO - determine whether to return highlighted response or not - responseData, err = transformer.TransformSearchResponse(ctx, responseData, q, true) + responseData, err = transformer.TransformSearchResponse(ctx, responseData, q, highlight) if err != nil { log.Event(ctx, "transformation of response data failed", log.Error(err), log.ERROR) http.Error(w, "Failed to transform search result", http.StatusInternalServerError) diff --git a/api/search_test.go b/api/search_test.go index 372627f0..5f9ec4cc 100644 --- a/api/search_test.go +++ b/api/search_test.go @@ -216,6 +216,57 @@ func TestSearchHandlerFunc(t *testing.T) { actualRequest := string(esMock.MultiSearchCalls()[0].Request) So(actualRequest, ShouldResemble, validQueryDoc) So(trMock.TransformSearchResponseCalls(), ShouldHaveLength, 1) + So(trMock.TransformSearchResponseCalls()[0].Highlight, ShouldBeTrue) + actualResponse := string(trMock.TransformSearchResponseCalls()[0].ResponseData) + So(actualResponse, ShouldResemble, validESResponse) + }) + + Convey("Should return OK for valid search result with highlight = true", t, func() { + qbMock := newQueryBuilderMock([]byte(validQueryDoc), nil) + esMock := newElasticSearcherMock([]byte(validESResponse), nil) + trMock := newResponseTransformerMock([]byte(validTransformedResponse), nil) + + searchHandler := SearchHandlerFunc(qbMock, esMock, trMock) + + req := httptest.NewRequest("GET", "http://localhost:8080/search?q="+validQueryParam+"&highlight=true", nil) + resp := httptest.NewRecorder() + + searchHandler.ServeHTTP(resp, req) + + So(resp.Code, ShouldEqual, http.StatusOK) + So(resp.Body.String(), ShouldResemble, validTransformedResponse) + So(qbMock.BuildSearchQueryCalls(), ShouldHaveLength, 1) + So(qbMock.BuildSearchQueryCalls()[0].Q, ShouldResemble, validQueryParam) + So(esMock.MultiSearchCalls(), ShouldHaveLength, 1) + actualRequest := string(esMock.MultiSearchCalls()[0].Request) + So(actualRequest, ShouldResemble, validQueryDoc) + So(trMock.TransformSearchResponseCalls(), ShouldHaveLength, 1) + So(trMock.TransformSearchResponseCalls()[0].Highlight, ShouldBeTrue) + actualResponse := string(trMock.TransformSearchResponseCalls()[0].ResponseData) + So(actualResponse, ShouldResemble, validESResponse) + }) + + Convey("Should return OK for valid search result with highlight = false", t, func() { + qbMock := newQueryBuilderMock([]byte(validQueryDoc), nil) + esMock := newElasticSearcherMock([]byte(validESResponse), nil) + trMock := newResponseTransformerMock([]byte(validTransformedResponse), nil) + + searchHandler := SearchHandlerFunc(qbMock, esMock, trMock) + + req := httptest.NewRequest("GET", "http://localhost:8080/search?q="+validQueryParam+"&highlight=false", nil) + resp := httptest.NewRecorder() + + searchHandler.ServeHTTP(resp, req) + + So(resp.Code, ShouldEqual, http.StatusOK) + So(resp.Body.String(), ShouldResemble, validTransformedResponse) + So(qbMock.BuildSearchQueryCalls(), ShouldHaveLength, 1) + So(qbMock.BuildSearchQueryCalls()[0].Q, ShouldResemble, validQueryParam) + So(esMock.MultiSearchCalls(), ShouldHaveLength, 1) + actualRequest := string(esMock.MultiSearchCalls()[0].Request) + So(actualRequest, ShouldResemble, validQueryDoc) + So(trMock.TransformSearchResponseCalls(), ShouldHaveLength, 1) + So(trMock.TransformSearchResponseCalls()[0].Highlight, ShouldBeFalse) actualResponse := string(trMock.TransformSearchResponseCalls()[0].ResponseData) So(actualResponse, ShouldResemble, validESResponse) }) diff --git a/swagger.yaml b/swagger.yaml index 4c7d1592..62fd73e2 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -108,6 +108,12 @@ paths: type: string collectionFormat: csv required: false + - in: query + name: highlight + description: "Determines whether to return HTML highlighted fields." + type: boolean + required: false + default: true - in: query name: sort description: "The order to return the results." @@ -303,33 +309,6 @@ definitions: required: - summary - title - matches: - type: object - properties: - description.title: - type: array - items: - type: string - description.edition: - type: array - items: - type: string - description.summary: - type: array - items: - type: string - description.metaDesription: - type: array - items: - type: string - description.keywords: - type: array - items: - type: string - description.datasetId: - type: array - items: - type: string type: type: string uri: From 81c40b4626969c33811fac4f5d141946fe208cb7 Mon Sep 17 00:00:00 2001 From: Steve Mynott Date: Wed, 8 Sep 2021 09:58:20 +0100 Subject: [PATCH 10/11] use v2 logging --- api/api.go | 9 +++++---- api/data.go | 12 ++++++------ api/health.go | 4 ++-- api/search.go | 29 +++++++++++++++-------------- api/timeseries.go | 9 +++++---- cmd/dp-search-api/main.go | 33 +++++++++++++++++---------------- go.mod | 3 +-- go.sum | 13 ++++++++++--- 8 files changed, 61 insertions(+), 51 deletions(-) diff --git a/api/api.go b/api/api.go index 340c0cb9..eb22c473 100644 --- a/api/api.go +++ b/api/api.go @@ -9,7 +9,7 @@ import ( "github.com/ONSdigital/dp-search-api/service" "github.com/ONSdigital/go-ns/server" - "github.com/ONSdigital/log.go/log" + "github.com/ONSdigital/log.go/v2/log" "github.com/gorilla/mux" "github.com/pkg/errors" ) @@ -76,9 +76,10 @@ func CreateAndInitialise(cfg *config.Config, queryBuilder QueryBuilder, elasticS httpServer.HandleOSSignals = false go func() { - log.Event(nil, "search api starting", log.INFO) + ctx := context.Background() + log.Info(ctx, "search api starting") if err := httpServer.ListenAndServe(); err != nil { - log.Event(nil, "search api http server returned error", log.Error(err), log.ERROR) + log.Error(ctx, "search api http server returned error", err) errorChan <- err } }() @@ -107,6 +108,6 @@ func Close(ctx context.Context) error { if err := httpServer.Shutdown(ctx); err != nil { return err } - log.Event(ctx, "graceful shutdown of http server complete", log.INFO) + log.Info(ctx, "graceful shutdown of http server complete") return nil } diff --git a/api/data.go b/api/data.go index 095b008e..1f645e07 100644 --- a/api/data.go +++ b/api/data.go @@ -7,7 +7,7 @@ import ( "net/http" "github.com/ONSdigital/dp-search-api/query" - "github.com/ONSdigital/log.go/log" + "github.com/ONSdigital/log.go/v2/log" ) type dataLookupRequest struct { @@ -41,35 +41,35 @@ func DataLookupHandlerFunc(elasticSearchClient ElasticSearcher) http.HandlerFunc err := dataTemplates.Execute(&doc, reqParams) if err != nil { - log.Event(ctx, "creation of search from template failed", log.Data{"Params": reqParams}, log.Error(err), log.ERROR) + log.Error(ctx, "creation of search from template failed", err, log.Data{"Params": reqParams}) http.Error(w, "Failed to create query", http.StatusInternalServerError) return } formattedQuery, err := query.FormatMultiQuery(doc.Bytes()) if err != nil { - log.Event(ctx, "formating of data query for elasticsearch failed", log.Error(err), log.ERROR) + log.Error(ctx, "formating of data query for elasticsearch failed", err) http.Error(w, "Failed to create query", http.StatusInternalServerError) return } responseString, err := elasticSearchClient.Search(ctx, "", "", formattedQuery) if err != nil { - log.Event(ctx, "elasticsearch data query failed", log.Error(err), log.ERROR) + log.Error(ctx, "elasticsearch data query failed", err) http.Error(w, "Failed to run data query", http.StatusInternalServerError) return } responseData := dataLookupResponse{Responses: make([]interface{}, 1)} if err := json.Unmarshal([]byte(responseString), &responseData.Responses[0]); err != nil { - log.Event(ctx, "failed to unmarshal response from elasticsearch for data query", log.Error(err), log.ERROR) + log.Error(ctx, "failed to unmarshal response from elasticsearch for data query", err) http.Error(w, "Failed to process data query", http.StatusInternalServerError) return } dataWithResponse, err := json.Marshal(responseData) if err != nil { - log.Event(ctx, "Failed to marshal response data for data query", log.Error(err), log.ERROR) + log.Error(ctx, "Failed to marshal response data for data query", err) http.Error(w, "Failed to encode data query response", http.StatusInternalServerError) return } diff --git a/api/health.go b/api/health.go index 094571a7..5a2149f8 100644 --- a/api/health.go +++ b/api/health.go @@ -6,7 +6,7 @@ import ( "net/http" "strings" - "github.com/ONSdigital/log.go/log" + "github.com/ONSdigital/log.go/v2/log" ) type healthMessage struct { @@ -41,7 +41,7 @@ func HealthCheckHandlerCreator(elasticSearchClient ElasticSearcher) func(http.Re Status: "error", Error: healthIssue, }); err != nil { - log.Event(ctx, "elasticsearch healthcheck status json failed to parse", log.Error(err), log.ERROR) + log.Error(ctx, "elasticsearch healthcheck status json failed to parse", err) panic(err) } } diff --git a/api/search.go b/api/search.go index 377125f3..11404995 100644 --- a/api/search.go +++ b/api/search.go @@ -6,7 +6,8 @@ import ( "net/url" "strconv" - "github.com/ONSdigital/log.go/log" + "github.com/ONSdigital/log.go/v2/log" + "github.com/pkg/errors" ) const defaultContentTypes string = "bulletin," + @@ -55,18 +56,18 @@ func SearchHandlerFunc(queryBuilder QueryBuilder, elasticSearchClient ElasticSea limitParam := paramGet(params, "limit", "10") limit, err := strconv.Atoi(limitParam) if err != nil { - log.Event(ctx, "numeric search parameter provided with non numeric characters", log.Data{ + log.Warn(ctx, "numeric search parameter provided with non numeric characters", log.Data{ "param": "limit", "value": limitParam, - }, log.WARN) + }) http.Error(w, "Invalid limit parameter", http.StatusBadRequest) return } if limit < 0 { - log.Event(ctx, "numeric search parameter provided with negative value", log.Data{ + log.Warn(ctx, "numeric search parameter provided with negative value", log.Data{ "param": "limit", "value": limitParam, - }, log.WARN) + }) http.Error(w, "Invalid limit parameter", http.StatusBadRequest) return } @@ -74,18 +75,18 @@ func SearchHandlerFunc(queryBuilder QueryBuilder, elasticSearchClient ElasticSea offsetParam := paramGet(params, "offset", "0") offset, err := strconv.Atoi(offsetParam) if err != nil { - log.Event(ctx, "numeric search parameter provided with non numeric characters", log.Data{ + log.Warn(ctx, "numeric search parameter provided with non numeric characters", log.Data{ "param": "from", "value": offsetParam, - }, log.WARN) + }) http.Error(w, "Invalid offset parameter", http.StatusBadRequest) return } if offset < 0 { - log.Event(ctx, "numeric search parameter provided with negative value", log.Data{ + log.Warn(ctx, "numeric search parameter provided with negative value", log.Data{ "param": "from", "value": offsetParam, - }, log.WARN) + }) http.Error(w, "Invalid offset parameter", http.StatusBadRequest) return } @@ -94,20 +95,20 @@ func SearchHandlerFunc(queryBuilder QueryBuilder, elasticSearchClient ElasticSea formattedQuery, err := queryBuilder.BuildSearchQuery(ctx, q, typesParam, sort, limit, offset) if err != nil { - log.Event(ctx, "creation of search query failed", log.Data{"q": q, "sort": sort, "limit": limit, "offset": offset}, log.Error(err), log.ERROR) + log.Error(ctx, "creation of search query failed", err, log.Data{"q": q, "sort": sort, "limit": limit, "offset": offset}) http.Error(w, "Failed to create search query", http.StatusInternalServerError) return } responseData, err := elasticSearchClient.MultiSearch(ctx, "ons", "", formattedQuery) if err != nil { - log.Event(ctx, "elasticsearch query failed", log.Error(err), log.ERROR) + log.Error(ctx, "elasticsearch query failed", err) http.Error(w, "Failed to run search query", http.StatusInternalServerError) return } if !json.Valid([]byte(responseData)) { - log.Event(ctx, "elastic search returned invalid JSON for search query", log.ERROR) + log.Error(ctx, "elastic search returned invalid JSON for search query", errors.New("elastic search returned invalid JSON for search query")) http.Error(w, "Failed to process search query", http.StatusInternalServerError) return } @@ -115,7 +116,7 @@ func SearchHandlerFunc(queryBuilder QueryBuilder, elasticSearchClient ElasticSea if !paramGetBool(params, "raw", false) { responseData, err = transformer.TransformSearchResponse(ctx, responseData, q, highlight) if err != nil { - log.Event(ctx, "transformation of response data failed", log.Error(err), log.ERROR) + log.Error(ctx, "transformation of response data failed", err) http.Error(w, "Failed to transform search result", http.StatusInternalServerError) return } @@ -124,7 +125,7 @@ func SearchHandlerFunc(queryBuilder QueryBuilder, elasticSearchClient ElasticSea w.Header().Set("Content-Type", "application/json;charset=utf-8") _, err = w.Write(responseData) if err != nil { - log.Event(ctx, "writing response failed", log.Error(err), log.ERROR) + log.Error(ctx, "writing response failed", err) http.Error(w, "Failed to write http response", http.StatusInternalServerError) return } diff --git a/api/timeseries.go b/api/timeseries.go index f5805d1e..2c7a4211 100644 --- a/api/timeseries.go +++ b/api/timeseries.go @@ -7,8 +7,9 @@ import ( "strings" "text/template" - "github.com/ONSdigital/log.go/log" + "github.com/ONSdigital/log.go/v2/log" "github.com/gorilla/mux" + "github.com/pkg/errors" ) type timeseriesLookupRequest struct { @@ -35,20 +36,20 @@ func TimeseriesLookupHandlerFunc(elasticSearchClient ElasticSearcher) http.Handl var doc bytes.Buffer err := timeseriesTemplate.Execute(&doc, reqParams) if err != nil { - log.Event(ctx, "creation of timeseries query from template failed", log.Data{"Params": reqParams}, log.Error(err), log.ERROR) + log.Error(ctx, "creation of timeseries query from template failed", err, log.Data{"Params": reqParams}) http.Error(w, "Failed to create query", http.StatusInternalServerError) return } responseData, err := elasticSearchClient.Search(ctx, "ons", "timeseries", doc.Bytes()) if err != nil { - log.Event(ctx, "elasticsearch query failed", log.Error(err), log.ERROR) + log.Error(ctx, "elasticsearch query failed", err) http.Error(w, "Failed to run timeseries query", http.StatusInternalServerError) return } if !json.Valid([]byte(responseData)) { - log.Event(ctx, "elastic search returned invalid JSON for timeseries query", log.ERROR) + log.Error(ctx, "elastic search returned invalid JSON for timeseries query", errors.New("elastic search returned invalid JSON for timeseries query")) http.Error(w, "Failed to process timeseries query", http.StatusInternalServerError) return } diff --git a/cmd/dp-search-api/main.go b/cmd/dp-search-api/main.go index afda543a..bd7ce521 100644 --- a/cmd/dp-search-api/main.go +++ b/cmd/dp-search-api/main.go @@ -2,11 +2,12 @@ package main import ( "context" - "github.com/ONSdigital/dp-search-api/service" "os" "os/signal" "syscall" + "github.com/ONSdigital/dp-search-api/service" + esauth "github.com/ONSdigital/dp-elasticsearch/v2/awsauth" elastic "github.com/ONSdigital/dp-elasticsearch/v2/elasticsearch" dphttp "github.com/ONSdigital/dp-net/http" @@ -15,7 +16,7 @@ import ( "github.com/ONSdigital/dp-search-api/elasticsearch" "github.com/ONSdigital/dp-search-api/query" "github.com/ONSdigital/dp-search-api/transformer" - "github.com/ONSdigital/log.go/log" + "github.com/ONSdigital/log.go/v2/log" ) const serviceName = "dp-search-api" @@ -32,24 +33,25 @@ var ( func main() { log.Namespace = serviceName + ctx := context.Background() cfg, err := config.Get() if err != nil { - log.Event(nil, "error retrieving config", log.Error(err), log.FATAL) + log.Fatal(ctx, "error retrieving config", err) os.Exit(1) } // sensitive fields are omitted from config.String(). - log.Event(nil, "config on startup", log.Data{"config": cfg}, log.INFO) + log.Info(ctx, "config on startup", log.Data{"config": cfg}) signals := make(chan os.Signal, 1) signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) apiErrors := make(chan error, 1) - log.Event(nil, "initialising query builder", log.INFO) + log.Info(ctx, "initialising query builder") queryBuilder, err := query.NewQueryBuilder() if err != nil { - log.Event(nil, "error initialising query builder", log.Error(err), log.FATAL) + log.Fatal(ctx, "error initialising query builder", err) os.Exit(1) } @@ -57,7 +59,7 @@ func main() { if cfg.SignElasticsearchRequests { esSigner, err = esauth.NewAwsSigner("", "", cfg.AwsRegion, cfg.AwsService) if err != nil { - log.Event(nil, "failed to create aws v4 signer", log.ERROR, log.Error(err)) + log.Error(ctx, "failed to create aws v4 signer", err) os.Exit(1) } } @@ -68,40 +70,39 @@ func main() { svcList := service.NewServiceList(&service.Init{}) // Get HealthCheck - ctx := context.Background() hc, err := registerCheckers(ctx, cfg, elasticHTTPClient, esSigner, svcList) if err != nil { - log.Event(nil, "could not register healthcheck", log.FATAL, log.Error(err)) + log.Fatal(ctx, "could not register healthcheck", err) os.Exit(1) } if err := api.CreateAndInitialise(cfg, queryBuilder, elasticSearchClient, transformer, hc, apiErrors); err != nil { - log.Event(nil, "error initialising API", log.Error(err), log.FATAL) + log.Fatal(ctx, "error initialising API", err) os.Exit(1) } gracefulShutdown := func() { - log.Event(nil, "commencing graceful shutdown", log.Data{"graceful_shutdown_timeout": cfg.GracefulShutdownTimeout}, log.INFO) + log.Info(ctx, "commencing graceful shutdown", log.Data{"graceful_shutdown_timeout": cfg.GracefulShutdownTimeout}) ctx, cancel := context.WithTimeout(context.Background(), cfg.GracefulShutdownTimeout) // stop any incoming requests before closing any outbound connections if err := api.Close(ctx); err != nil { - log.Event(ctx, "error closing API", log.Error(err), log.ERROR) + log.Error(ctx, "error closing API", err) } hc.Stop() - log.Event(ctx, "shutdown complete", log.INFO) + log.Info(ctx, "shutdown complete") cancel() } // blocks until a fatal error occurs select { case err := <-apiErrors: - log.Event(nil, "search api error received", log.Error(err), log.FATAL) + log.Fatal(ctx, "search api error received", err) case <-signals: - log.Event(nil, "os signal received", log.INFO) + log.Info(ctx, "os signal received") gracefulShutdown() } @@ -123,7 +124,7 @@ func registerCheckers(ctx context.Context, elasticClient := elastic.NewClientWithHTTPClientAndAwsSigner(cfg.ElasticSearchAPIURL, esSigner, cfg.SignElasticsearchRequests, elasticHTTPClient) if err = hc.AddCheck("Elasticsearch", elasticClient.Checker); err != nil { - log.Event(ctx, "error creating elasticsearch health check", log.ERROR, log.Error(err)) + log.Error(ctx, "error creating elasticsearch health check", err) hasErrors = true } diff --git a/go.mod b/go.mod index 5a6e7f73..9d56c989 100644 --- a/go.mod +++ b/go.mod @@ -8,10 +8,9 @@ require ( github.com/ONSdigital/dp-healthcheck v1.0.5 github.com/ONSdigital/dp-net v1.0.12 github.com/ONSdigital/go-ns v0.0.0-20210410105122-6d6a140e952e - github.com/ONSdigital/log.go v1.0.1 + github.com/ONSdigital/log.go/v2 v2.0.9 github.com/aws/aws-sdk-go v1.38.65 // indirect github.com/gorilla/mux v1.8.0 - github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 // indirect github.com/kelseyhightower/envconfig v1.4.0 github.com/pkg/errors v0.9.1 github.com/smartystreets/goconvey v1.6.4 diff --git a/go.sum b/go.sum index fa6b61a4..575c84fd 100644 --- a/go.sum +++ b/go.sum @@ -26,6 +26,8 @@ github.com/ONSdigital/log.go v1.0.1-0.20200805084515-ee61165ea36a/go.mod h1:dDnQ github.com/ONSdigital/log.go v1.0.1-0.20200805145532-1f25087a0744/go.mod h1:y4E9MYC+cV9VfjRD0UBGj8PA7H3wABqQi87/ejrDhYc= github.com/ONSdigital/log.go v1.0.1 h1:SZ5wRZAwlt2jQUZ9AUzBB/PL+iG15KapfQpJUdA18/4= github.com/ONSdigital/log.go v1.0.1/go.mod h1:dIwSXuvFB5EsZG5x44JhsXZKMd80zlb0DZxmiAtpL4M= +github.com/ONSdigital/log.go/v2 v2.0.9 h1:dMtuN89vCP21iRuOBAGInn7ZzxIEGajC3o5pjoicnsY= +github.com/ONSdigital/log.go/v2 v2.0.9/go.mod h1:VyTDkL82FtiAkaNFaT+bURBhLbP7NsIx4rkVbdpiuEg= github.com/aws/aws-sdk-go v1.38.15/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.38.65 h1:umGu5gjIOKxzhi34T0DIA1TWupUDjV2aAW5vK6154Gg= github.com/aws/aws-sdk-go v1.38.65/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -35,8 +37,9 @@ github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZi github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7VRjQhwPlR/v0glZXNYzBHdNUzf/Am2Nmg= github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/go-avro/avro v0.0.0-20171219232920-444163702c11/go.mod h1:kxj6THYP0dmFPk4Z+bijIAhJoGgeBfyOKXMduhvdJPA= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= @@ -59,11 +62,13 @@ github.com/justinas/alice v1.2.0 h1:+MHSA/vccVCF4Uq37S42jwlkvI2Xzl7zTPCN5BnZNVo= github.com/justinas/alice v1.2.0/go.mod h1:fN5HRH/reO/zrUflLfTN43t3vXvKzvZIENsNEe7i7qA= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= -github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.11 h1:FxPOTFNqGkuDUGi3H/qkUbQO4ZiBa2brKq5r0l8TGeM= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -98,6 +103,8 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= From c3d4a810e838f1a2f77e154e3eb24629e4fb63e3 Mon Sep 17 00:00:00 2001 From: Steve Mynott Date: Wed, 8 Sep 2021 09:59:56 +0100 Subject: [PATCH 11/11] upgrade modules dp-healthcheck v1.1.0 dp-net v1.2.0 go-ns v0.0.0-20210831102424-ebdecc20fe9e --- go.mod | 7 +++---- go.sum | 46 ++++++++++++++++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index 9d56c989..1b22b206 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,10 @@ module github.com/ONSdigital/dp-search-api go 1.16 require ( - github.com/ONSdigital/dp-api-clients-go v1.34.4 // indirect github.com/ONSdigital/dp-elasticsearch/v2 v2.2.0 - github.com/ONSdigital/dp-healthcheck v1.0.5 - github.com/ONSdigital/dp-net v1.0.12 - github.com/ONSdigital/go-ns v0.0.0-20210410105122-6d6a140e952e + github.com/ONSdigital/dp-healthcheck v1.1.0 + github.com/ONSdigital/dp-net v1.2.0 + github.com/ONSdigital/go-ns v0.0.0-20210831102424-ebdecc20fe9e github.com/ONSdigital/log.go/v2 v2.0.9 github.com/aws/aws-sdk-go v1.38.65 // indirect github.com/gorilla/mux v1.8.0 diff --git a/go.sum b/go.sum index 575c84fd..8addda6f 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,37 @@ github.com/ONSdigital/dp-api-clients-go v1.1.0/go.mod h1:9lqor0I7caCnRWr04gU/r7x5dqxgoODob8L48q+cE4E= github.com/ONSdigital/dp-api-clients-go v1.28.0/go.mod h1:iyJy6uRL4B6OYOJA0XMr5UHt6+Q8XmN9uwmURO+9Oj4= -github.com/ONSdigital/dp-api-clients-go v1.34.4 h1:SHdhxHxflgoAesWpwrVR0OQz+pLKI4jMl58mZrzKyA4= -github.com/ONSdigital/dp-api-clients-go v1.34.4/go.mod h1:kX+YKuoLYLfkeLHMvQKRRydZVxO7ZEYyYiwG2xhV51E= +github.com/ONSdigital/dp-api-clients-go v1.34.3/go.mod h1:kX+YKuoLYLfkeLHMvQKRRydZVxO7ZEYyYiwG2xhV51E= +github.com/ONSdigital/dp-api-clients-go v1.41.1 h1:xkeT6dCTFSAoBpZxgiJUiuqgcfjCX+c52CIiZo1Y2iU= +github.com/ONSdigital/dp-api-clients-go v1.41.1/go.mod h1:Ga1+ANjviu21NFJI9wp5NctJIdB4TJLDGbpQFl2V8Wc= github.com/ONSdigital/dp-elasticsearch/v2 v2.2.0 h1:dnHq+0WZtKXIb0tFK//lc/eG8TXLNMwt5k+UnCiwLfM= github.com/ONSdigital/dp-elasticsearch/v2 v2.2.0/go.mod h1:f85VX6TzHkq3pM/ddUzufoFNTcx3FOi7VN+SQG0vMbg= github.com/ONSdigital/dp-frontend-models v1.1.0/go.mod h1:TT96P7Mi69N3Tc/jFNdbjiwG4GAaMjP26HLotFQ6BPw= github.com/ONSdigital/dp-healthcheck v0.0.0-20200131122546-9db6d3f0494e/go.mod h1:zighxZ/0m5u7zo0eAr8XFlA+Dz2ic7A1vna6YXvhCjQ= -github.com/ONSdigital/dp-healthcheck v1.0.5 h1:DXnohGIqXaLLeYGdaGOhgkZjAbWMNoLAjQ3EgZeMT3M= github.com/ONSdigital/dp-healthcheck v1.0.5/go.mod h1:2wbVAUHMl9+4tWhUlxYUuA1dnf2+NrwzC+So5f5BMLk= +github.com/ONSdigital/dp-healthcheck v1.1.0 h1:fKOf8MMe8l4EW28ljX0wNZ5oZTgx/slAs7lyEx4eq2c= +github.com/ONSdigital/dp-healthcheck v1.1.0/go.mod h1:vZwyjMJiCHjp/sJ2R1ZEqzZT0rJ0+uHVGwxqdP4J5vg= github.com/ONSdigital/dp-mocking v0.0.0-20190905163309-fee2702ad1b9/go.mod h1:BcIRgitUju//qgNePRBmNjATarTtynAgc0yV29VpLEk= github.com/ONSdigital/dp-net v1.0.5-0.20200805082802-e518bc287596/go.mod h1:wDVhk2pYosQ1q6PXxuFIRYhYk2XX5+1CeRRnXpSczPY= github.com/ONSdigital/dp-net v1.0.5-0.20200805145012-9227a11caddb/go.mod h1:MrSZwDUvp8u1VJEqa+36Gwq4E7/DdceW+BDCvGes6Cs= github.com/ONSdigital/dp-net v1.0.5-0.20200805150805-cac050646ab5/go.mod h1:de3LB9tedE0tObBwa12dUOt5rvTW4qQkF5rXtt4b6CE= github.com/ONSdigital/dp-net v1.0.7/go.mod h1:1QFzx32FwPKD2lgZI6MtcsUXritsBdJihlzIWDrQ/gc= -github.com/ONSdigital/dp-net v1.0.12 h1:Vd06ia1FXKR9uyhzWykQ52b1LTp4N0VOLnrF7KOeP78= github.com/ONSdigital/dp-net v1.0.12/go.mod h1:2lvIKOlD4T3BjWQwjHhBUO2UNWDk82u/+mHRn0R3C9A= +github.com/ONSdigital/dp-net v1.2.0 h1:gP9pBt/J8gktYeKsb7hq6uOC2xx1tfvTorUBNXp6pX0= +github.com/ONSdigital/dp-net v1.2.0/go.mod h1:NinlaqcsPbIR+X7j5PXCl3UI5G2zCL041SDF6WIiiO4= github.com/ONSdigital/dp-rchttp v0.0.0-20190919143000-bb5699e6fd59/go.mod h1:KkW68U3FPuivW4ogi9L8CPKNj9ZxGko4qcUY7KoAAkQ= github.com/ONSdigital/dp-rchttp v0.0.0-20200114090501-463a529590e8/go.mod h1:821jZtK0oBsV8hjIkNr8vhAWuv0FxJBPJuAHa2B70Gk= github.com/ONSdigital/go-ns v0.0.0-20191104121206-f144c4ec2e58/go.mod h1:iWos35il+NjbvDEqwtB736pyHru0MPFE/LqcwkV1wDc= -github.com/ONSdigital/go-ns v0.0.0-20210410105122-6d6a140e952e h1:64qsVTEixfFUuPcl4SOOGU3WRv136aEKRqIwXNIrttE= -github.com/ONSdigital/go-ns v0.0.0-20210410105122-6d6a140e952e/go.mod h1:kb2qyEcVuTkuz9yxOKTXroMrEniTsR2u9rwAqxgQ8MQ= +github.com/ONSdigital/go-ns v0.0.0-20210831102424-ebdecc20fe9e h1:tXdCJg2SUi2vLPA22bgVmeAY+sp3pKtUsAboBrOkO5Y= +github.com/ONSdigital/go-ns v0.0.0-20210831102424-ebdecc20fe9e/go.mod h1:BCx4ULp5nT3dT7Mft5iMrp9439JG9lqIlc0JOPmsHTg= github.com/ONSdigital/log.go v0.0.0-20191127134126-2a610b254f20/go.mod h1:BD7D8FWP1fzwUWsrCopEG72jl9cchCaVNIGSz6YvL+Y= github.com/ONSdigital/log.go v1.0.0/go.mod h1:UnGu9Q14gNC+kz0DOkdnLYGoqugCvnokHBRBxFRpVoQ= github.com/ONSdigital/log.go v1.0.1-0.20200805084515-ee61165ea36a/go.mod h1:dDnQATFXCBOknvj6ZQuKfmDhbOWf3e8mtV+dPEfWJqs= github.com/ONSdigital/log.go v1.0.1-0.20200805145532-1f25087a0744/go.mod h1:y4E9MYC+cV9VfjRD0UBGj8PA7H3wABqQi87/ejrDhYc= -github.com/ONSdigital/log.go v1.0.1 h1:SZ5wRZAwlt2jQUZ9AUzBB/PL+iG15KapfQpJUdA18/4= github.com/ONSdigital/log.go v1.0.1/go.mod h1:dIwSXuvFB5EsZG5x44JhsXZKMd80zlb0DZxmiAtpL4M= +github.com/ONSdigital/log.go v1.1.0 h1:XFE8U5lPeiXyujgUtbh+pKCotiICeIGFEAauNk9c24A= +github.com/ONSdigital/log.go v1.1.0/go.mod h1:0hOVuYR3bDUI30VRo48d5KHfJIoe+spuPXqgt6UF78o= +github.com/ONSdigital/log.go/v2 v2.0.0/go.mod h1:PR7vXrv9dZKUc7SI/0toxBbStk84snmybBnWpe+xY2o= +github.com/ONSdigital/log.go/v2 v2.0.5/go.mod h1:PR7vXrv9dZKUc7SI/0toxBbStk84snmybBnWpe+xY2o= github.com/ONSdigital/log.go/v2 v2.0.9 h1:dMtuN89vCP21iRuOBAGInn7ZzxIEGajC3o5pjoicnsY= github.com/ONSdigital/log.go/v2 v2.0.9/go.mod h1:VyTDkL82FtiAkaNFaT+bURBhLbP7NsIx4rkVbdpiuEg= github.com/aws/aws-sdk-go v1.38.15/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -38,13 +44,17 @@ github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9 h1:wWke/RUCl7V github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmAp6Sws4L7+Q/OokbWDAK1ibXYhB3PXFP1kol5hPg= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/go-avro/avro v0.0.0-20171219232920-444163702c11/go.mod h1:kxj6THYP0dmFPk4Z+bijIAhJoGgeBfyOKXMduhvdJPA= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe h1:rcf1P0fm+1l0EjG16p06mYLj9gW9X36KgdHJ/88hS4g= +github.com/gopherjs/gopherjs v0.0.0-20210202160940-bed99a852dfe/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -67,16 +77,18 @@ github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= -github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= +github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= +github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -90,14 +102,16 @@ github.com/unrolled/render v1.0.2/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0ob golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -107,14 +121,22 @@ golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210414055047-fe65e336abe0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e h1:WUoyKPm6nCo1BnNUvPGnFG3T5DUVem42yDJZZ4CNxMA= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=