Skip to content

Commit

Permalink
Merge branch 'release/1.21.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
@RMPall
RMPall committed May 3, 2022
2 parents 250ff30 + 97f14f0 commit 3bce1b9
Show file tree
Hide file tree
Showing 70 changed files with 152 additions and 551 deletions.
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,5 @@

#Test output files
coverage.txt

/main
3 changes: 1 addition & 2 deletions Dockerfile.concourse
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ FROM onsdigital/dp-concourse-tools-ubuntu

WORKDIR /app/

COPY templates templates
COPY dp-search-api .

CMD ./dp-search-api
CMD ./dp-search-api
1 change: 0 additions & 1 deletion ci/scripts/build.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,5 @@ export GOPATH=$cwd/go

pushd dp-search-api
make build && mv build/$(go env GOOS)-$(go env GOARCH)/* $cwd/build
cp -r templates $cwd/build
cp Dockerfile.concourse $cwd/build
popd
13 changes: 10 additions & 3 deletions cmd/reindex/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"log"
"net/url"
"strings"
"sync"
"time"
Expand Down Expand Up @@ -224,8 +225,14 @@ func transformZebedeeDoc(extractedChan chan Document, transformedChan chan<- Doc

func transformMetadataDoc(metadataChan chan dataset.Metadata, transformedChan chan<- Document, wg *sync.WaitGroup) {
for metadata := range metadataChan {
uri := metadata.DatasetLinks.LatestVersion.URL
parsedURI, err := url.Parse(uri)
if err != nil {
log.Fatalf("error occured while parsing url: %v", err)
}
cmdData := extractorModels.CMDData{
UID: metadata.DatasetDetails.ID,
URI: parsedURI.Path,
VersionDetails: extractorModels.VersionDetails{
ReleaseDate: metadata.Version.ReleaseDate,
},
Expand All @@ -249,7 +256,7 @@ func transformMetadataDoc(metadataChan chan dataset.Metadata, transformedChan ch

transformedDoc := Document{
ID: exporterEventData.UID,
URI: metadata.URI,
URI: parsedURI.Path,
Body: body,
}
transformedChan <- transformedDoc
Expand Down Expand Up @@ -441,8 +448,8 @@ func retrieveLatestMetadata(ctx context.Context, datasetClient clients.DatasetAP
for i := 0; i < maxConcurrentExtractions; i++ {
wg.Add(1)
go func() {
for metadata := range editionMetadata {
metadata, err := datasetClient.GetVersionMetadata(ctx, "", serviceAuthToken, "", metadata.id, metadata.editionID, metadata.version)
for edMetadata := range editionMetadata {
metadata, err := datasetClient.GetVersionMetadata(ctx, "", serviceAuthToken, "", edMetadata.id, edMetadata.editionID, edMetadata.version)
if err != nil {
continue
}
Expand Down
28 changes: 6 additions & 22 deletions go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,12 @@ require (
github.com/ONSdigital/dp-api-clients-go/v2 v2.103.0
github.com/ONSdigital/dp-authorisation v0.2.0
github.com/ONSdigital/dp-component-test v0.6.3
github.com/ONSdigital/dp-elasticsearch/v3 v3.0.0-alpha.2
github.com/ONSdigital/dp-elasticsearch/v3 v3.0.0-alpha.3
github.com/ONSdigital/dp-healthcheck v1.2.3
github.com/ONSdigital/dp-net/v2 v2.0.0
github.com/ONSdigital/dp-search-data-extractor v0.9.0
github.com/ONSdigital/dp-search-data-importer v0.5.0
github.com/ONSdigital/log.go/v2 v2.1.0
github.com/ONSdigital/dp-search-data-extractor v0.12.0
github.com/ONSdigital/dp-search-data-importer v0.7.0
github.com/ONSdigital/log.go/v2 v2.2.0
github.com/cucumber/godog v0.12.2
github.com/google/go-cmp v0.5.6
github.com/gorilla/mux v1.8.0
Expand All @@ -26,47 +26,31 @@ require (

require (
github.com/ONSdigital/dp-api-clients-go v1.43.0 // indirect
github.com/ONSdigital/dp-elasticsearch/v2 v2.3.0 // indirect
github.com/ONSdigital/dp-kafka/v2 v2.4.3 // indirect
github.com/ONSdigital/dp-mongodb-in-memory v1.1.0 // indirect
github.com/ONSdigital/dp-net v1.4.0 // indirect
github.com/ONSdigital/dp-net v1.4.1 // indirect
github.com/ONSdigital/dp-rchttp v1.0.0 // indirect
github.com/ONSdigital/go-ns v0.0.0-20200205115900-a11716f93bad // indirect
github.com/Shopify/sarama v1.29.1 // indirect
github.com/aws/aws-sdk-go v1.42.47 // indirect
github.com/cucumber/gherkin-go/v19 v19.0.3 // indirect
github.com/cucumber/messages-go/v16 v16.0.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/eapache/go-resiliency v1.2.0 // indirect
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
github.com/eapache/queue v1.1.0 // indirect
github.com/elastic/go-elasticsearch/v7 v7.10.0 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/go-avro/avro v0.0.0-20171219232920-444163702c11 // indirect
github.com/go-stack/stack v1.8.0 // indirect
github.com/gofrs/uuid v4.0.0+incompatible // indirect
github.com/golang/snappy v0.0.3 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gopherjs/gopherjs v0.0.0-20220104163920-15ed2e8cf2bd // indirect
github.com/hashicorp/go-immutable-radix v1.3.0 // indirect
github.com/hashicorp/go-memdb v1.3.0 // indirect
github.com/hashicorp/go-uuid v1.0.2 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.2 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/jtolds/gls v4.20.0+incompatible // indirect
github.com/justinas/alice v1.2.0 // indirect
github.com/klauspost/compress v1.13.6 // indirect
github.com/kr/text v0.2.0 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
github.com/mattn/go-isatty v0.0.14 // indirect
github.com/pierrec/lz4 v2.6.1+incompatible // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/smartystreets/assertions v1.2.1 // indirect
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
Expand Down
Loading

0 comments on commit 3bce1b9

Please sign in to comment.