From c5d20ec834284717eb326dc53c05b8b3447192d8 Mon Sep 17 00:00:00 2001 From: Rafa Date: Thu, 23 Mar 2023 13:13:31 +0000 Subject: [PATCH 1/4] Add version etag --- dataset/data.go | 1 + dataset/dataset.go | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/dataset/data.go b/dataset/data.go index 63f5ef14..90945ff3 100644 --- a/dataset/data.go +++ b/dataset/data.go @@ -119,6 +119,7 @@ type Version struct { UsageNotes *[]UsageNote `json:"usage_notes,omitempty"` IsBasedOn *IsBasedOn `json:"is_based_on,omitempty"` LowestGeography string `json:"lowest_geography,omitempty"` + ETag string `json:"e_tag,omitempty"` } type UpdateInstance struct { diff --git a/dataset/dataset.go b/dataset/dataset.go index c42db9ac..099fe979 100644 --- a/dataset/dataset.go +++ b/dataset/dataset.go @@ -566,7 +566,7 @@ func (c *Client) GetVersionsBatchProcess(ctx context.Context, userAuthToken, ser } // GetVersion gets a specific version for an edition from the dataset api -func (c *Client) GetVersion(ctx context.Context, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, version string) (m Version, err error) { +func (c *Client) GetVersion(ctx context.Context, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, version string) (v Version, err error) { uri := fmt.Sprintf("%s/datasets/%s/editions/%s/versions/%s", c.hcCli.URL, datasetID, edition, version) resp, err := c.doGetWithAuthHeadersAndWithDownloadToken(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, uri) @@ -585,7 +585,9 @@ func (c *Client) GetVersion(ctx context.Context, userAuthToken, serviceAuthToken return } - err = json.Unmarshal(b, &m) + err = json.Unmarshal(b, &v) + v.ETag, _ = headers.GetResponseETag(resp) + return } From 7c0bc15b6f726c030cbbea5261b6407de3eebd1e Mon Sep 17 00:00:00 2001 From: Rafa Date: Thu, 23 Mar 2023 14:11:41 +0000 Subject: [PATCH 2/4] Add unit test for GetVersion And refactor to check all headers --- dataset/dataset_test.go | 356 ++++++++++++++++++++++++++++++++++------ 1 file changed, 302 insertions(+), 54 deletions(-) diff --git a/dataset/dataset_test.go b/dataset/dataset_test.go index 1fa9629b..6f41ea85 100644 --- a/dataset/dataset_test.go +++ b/dataset/dataset_test.go @@ -8,6 +8,7 @@ import ( "io/ioutil" "net/http" "net/url" + "strconv" "testing" "time" @@ -21,12 +22,13 @@ import ( ) const ( - userAuthToken = "iamatoken" - serviceAuthToken = "iamaservicetoken" - collectionID = "iamacollectionID" - testHost = "http://localhost:8080" - testIfMatch = "testIfMatch" - testETag = "testETag" + userAuthToken = "iamatoken" + serviceAuthToken = "iamaservicetoken" + downloadServiceAuthToken = "downloadToken" + collectionID = "iamacollectionID" + testHost = "http://localhost:8080" + testIfMatch = "testIfMatch" + testETag = "testETag" ) var ( @@ -34,12 +36,25 @@ var ( initialState = health.CreateCheckState(service) ) -var checkRequestBase = func(httpClient *dphttp.ClienterMock, expectedMethod string, expectedUri, expectedIfMatch string) { +type expectedHeaders struct { + FlorenceToken string + ServiceToken string + CollectionId string + IfMatch string + DownloadServiceToken string +} + +var checkRequestBase = func(httpClient *dphttp.ClienterMock, expectedMethod, expectedUri string, expectedHeaders expectedHeaders) { So(len(httpClient.DoCalls()), ShouldEqual, 1) So(httpClient.DoCalls()[0].Req.URL.RequestURI(), ShouldResemble, expectedUri) So(httpClient.DoCalls()[0].Req.Method, ShouldEqual, expectedMethod) - So(httpClient.DoCalls()[0].Req.Header.Get(dprequest.AuthHeaderKey), ShouldEqual, "Bearer "+serviceAuthToken) - So(httpClient.DoCalls()[0].Req.Header.Get("If-Match"), ShouldEqual, expectedIfMatch) + if expectedHeaders.ServiceToken != "" { + So(httpClient.DoCalls()[0].Req.Header.Get(dprequest.AuthHeaderKey), ShouldEqual, "Bearer "+expectedHeaders.ServiceToken) + } + So(httpClient.DoCalls()[0].Req.Header.Get("If-Match"), ShouldEqual, expectedHeaders.IfMatch) + So(httpClient.DoCalls()[0].Req.Header.Get("Collection-Id"), ShouldEqual, expectedHeaders.CollectionId) + So(httpClient.DoCalls()[0].Req.Header.Get("X-Florence-Token"), ShouldEqual, expectedHeaders.FlorenceToken) + So(httpClient.DoCalls()[0].Req.Header.Get("X-Download-Service-Token"), ShouldEqual, expectedHeaders.DownloadServiceToken) } // getRequestPatchBody returns the patch request body sent with the provided httpClient in iteration callIndex @@ -221,11 +236,76 @@ func TestClient_HealthChecker(t *testing.T) { }) } +func TestClient_GetVersion(t *testing.T) { + ctx := context.Background() + + Convey("Given dataset api has a version", t, func() { + + datasetId := "dataset-id" + edition := "2023" + versionString := "1" + versionNumber, _ := strconv.Atoi(versionString) + etag := "version-etag" + + version := Version{ + ID: "version-id", + CollectionID: collectionID, + Edition: edition, + Version: versionNumber, + Dimensions: []VersionDimension{ + { + Name: "geography", + ID: "city", + Label: "City", + }, + { + Name: "siblings", + ID: "number_of_siblings_3", + Label: "Number Of Siblings (3 Mappings)", + }, + }, + ReleaseDate: "today", + LowestGeography: "lowest", + State: "published", + } + httpClient := createHTTPClientMock(MockedHTTPResponse{http.StatusOK, version, map[string]string{"Etag": etag}}) + datasetClient := newDatasetClient(httpClient) + + Convey("when GetVersion is called", func() { + So(version.ETag, ShouldBeEmpty) // Ensure etag is coming from the header + + got, err := datasetClient.GetVersion(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetId, edition, versionString) + + Convey("Then no error is returned", func() { + So(err, ShouldBeNil) + }) + Convey("And it returns the version", func() { + version.ETag = etag // Check the etag has been set + So(got, ShouldResemble, version) + }) + Convey("And the relevant api call has been made", func() { + expectedUrl := fmt.Sprintf("/datasets/%s/editions/%s/versions/%s", datasetId, edition, versionString) + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + DownloadServiceToken: downloadServiceAuthToken, + } + checkRequestBase(httpClient, http.MethodGet, expectedUrl, expectedHeaders) + }) + }) + }) +} + func TestClient_PutVersion(t *testing.T) { checkResponse := func(httpClient *dphttp.ClienterMock, expectedVersion Version) { - - checkRequestBase(httpClient, http.MethodPut, "/datasets/123/editions/2017/versions/1", "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodPut, "/datasets/123/editions/2017/versions/1", expectedHeaders) actualBody, _ := ioutil.ReadAll(httpClient.DoCalls()[0].Req.Body) @@ -412,7 +492,12 @@ func TestClient_GetDatasets(t *testing.T) { Convey("and dphttpclient.Do is called 1 time with the expected URI", func() { expectedURI := fmt.Sprintf("/datasets?offset=%d&limit=%d", offset, limit) - checkRequestBase(httpClient, http.MethodGet, expectedURI, "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, expectedURI, expectedHeaders) }) }) @@ -423,7 +508,12 @@ func TestClient_GetDatasets(t *testing.T) { Convey("and dphttpclient.Do is called 1 time with the expected URI", func() { expectedURI := fmt.Sprintf("/datasets?offset=%d&limit=%d&is_based_on=%s", offset, limit, isBasedOn) - checkRequestBase(httpClient, http.MethodGet, expectedURI, "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, expectedURI, expectedHeaders) }) }) @@ -468,7 +558,12 @@ func TestClient_GetDatasets(t *testing.T) { Convey("and dphttpclient.Do is called 1 time with the expected URI", func() { expectedURI := fmt.Sprintf("/datasets") - checkRequestBase(httpClient, http.MethodGet, expectedURI, "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, expectedURI, expectedHeaders) }) }) }) @@ -670,14 +765,14 @@ func TestClient_GetVersionsInBatches(t *testing.T) { } Convey("then GetDatasetsInBatches succeeds and returns the accumulated items from all the batches", func() { - datasets, err := datasetClient.GetVersionsInBatches(ctx, userAuthToken, serviceAuthToken, "", collectionID, datasetID, edition, batchSize, maxWorkers) + datasets, err := datasetClient.GetVersionsInBatches(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, batchSize, maxWorkers) So(err, ShouldBeNil) So(datasets, ShouldResemble, expectedDatasets) }) Convey("then GetDatasetsBatchProcess calls the batchProcessor function twice, with the expected batches", func() { - err := datasetClient.GetVersionsBatchProcess(ctx, userAuthToken, serviceAuthToken, "", collectionID, datasetID, edition, testProcess, batchSize, maxWorkers) + err := datasetClient.GetVersionsBatchProcess(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, testProcess, batchSize, maxWorkers) So(err, ShouldBeNil) So(processedBatches, ShouldResemble, []VersionsList{versionsResponse1, versionsResponse2}) So(httpClient.DoCalls(), ShouldHaveLength, 2) @@ -700,13 +795,13 @@ func TestClient_GetVersionsInBatches(t *testing.T) { } Convey("then GetOptionsInBatches fails with the expected error and the process is aborted", func() { - _, err := datasetClient.GetVersionsInBatches(ctx, userAuthToken, serviceAuthToken, "", collectionID, datasetID, edition, batchSize, maxWorkers) + _, err := datasetClient.GetVersionsInBatches(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, batchSize, maxWorkers) So(err.(*ErrInvalidDatasetAPIResponse).actualCode, ShouldEqual, http.StatusBadRequest) So(err.(*ErrInvalidDatasetAPIResponse).uri, ShouldResemble, "http://localhost:8080/datasets/test-dataset/editions/test-edition/versions?offset=0&limit=1") }) Convey("then GetDatasetsBatchProcess fails with the expected error and doesn't call the batchProcessor", func() { - err := datasetClient.GetVersionsBatchProcess(ctx, userAuthToken, serviceAuthToken, "", collectionID, datasetID, edition, testProcess, batchSize, maxWorkers) + err := datasetClient.GetVersionsBatchProcess(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, testProcess, batchSize, maxWorkers) So(err.(*ErrInvalidDatasetAPIResponse).actualCode, ShouldEqual, http.StatusBadRequest) So(err.(*ErrInvalidDatasetAPIResponse).uri, ShouldResemble, "http://localhost:8080/datasets/test-dataset/editions/test-edition/versions?offset=0&limit=1") So(processedBatches, ShouldResemble, []VersionsList{}) @@ -727,13 +822,13 @@ func TestClient_GetVersionsInBatches(t *testing.T) { } Convey("then GetDatasetsInBatches fails with the expected error, corresponding to the second batch, and the process is aborted", func() { - _, err := datasetClient.GetVersionsInBatches(ctx, userAuthToken, serviceAuthToken, "", collectionID, datasetID, edition, batchSize, maxWorkers) + _, err := datasetClient.GetVersionsInBatches(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, batchSize, maxWorkers) So(err.(*ErrInvalidDatasetAPIResponse).actualCode, ShouldEqual, http.StatusBadRequest) So(err.(*ErrInvalidDatasetAPIResponse).uri, ShouldResemble, "http://localhost:8080/datasets/test-dataset/editions/test-edition/versions?offset=1&limit=1") }) Convey("then GetDatasetsBatchProcess fails with the expected error and calls the batchProcessor for the first batch only", func() { - err := datasetClient.GetVersionsBatchProcess(ctx, userAuthToken, serviceAuthToken, "", collectionID, datasetID, edition, testProcess, batchSize, maxWorkers) + err := datasetClient.GetVersionsBatchProcess(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, testProcess, batchSize, maxWorkers) So(err.(*ErrInvalidDatasetAPIResponse).actualCode, ShouldEqual, http.StatusBadRequest) So(err.(*ErrInvalidDatasetAPIResponse).uri, ShouldResemble, "http://localhost:8080/datasets/test-dataset/editions/test-edition/versions?offset=1&limit=1") So(processedBatches, ShouldResemble, []VersionsList{versionsResponse1}) @@ -757,7 +852,12 @@ func TestClient_GetDatasetCurrentAndNext(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time", func() { - checkRequestBase(httpClient, http.MethodGet, "/datasets/123", "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, "/datasets/123", expectedHeaders) }) }) }) @@ -774,7 +874,12 @@ func TestClient_GetDatasetCurrentAndNext(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time", func() { - checkRequestBase(httpClient, http.MethodGet, "/datasets/123", "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, "/datasets/123", expectedHeaders) }) }) }) @@ -796,7 +901,12 @@ func TestClient_GetFullEditionsDetails(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time", func() { - checkRequestBase(httpClient, http.MethodGet, "/datasets/123/editions", "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, "/datasets/123/editions", expectedHeaders) }) }) }) @@ -813,7 +923,12 @@ func TestClient_GetFullEditionsDetails(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time", func() { - checkRequestBase(httpClient, http.MethodGet, "/datasets/123/editions", "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, "/datasets/123/editions", expectedHeaders) }) }) }) @@ -839,7 +954,12 @@ func TestClient_GetFullEditionsDetails(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time", func() { - checkRequestBase(httpClient, http.MethodGet, "/datasets/123/editions", "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, "/datasets/123/editions", expectedHeaders) }) }) }) @@ -860,7 +980,12 @@ func TestClient_GetFullEditionsDetails(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with expected parameters", func() { - checkRequestBase(httpClient, http.MethodGet, "/datasets/123/editions", "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, "/datasets/123/editions", expectedHeaders) }) }) }) @@ -881,7 +1006,12 @@ func TestClient_GetFullEditionsDetails(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with expected parameters", func() { - checkRequestBase(httpClient, http.MethodGet, "/datasets/123/editions", "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, "/datasets/123/editions", expectedHeaders) }) }) }) @@ -907,7 +1037,13 @@ func TestClient_GetInstance(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, path and headers", func() { - checkRequestBase(httpClient, http.MethodGet, "/instances/123", testIfMatch) + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodGet, "/instances/123", expectedHeaders) }) }) }) @@ -929,7 +1065,13 @@ func TestClient_GetInstance(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, path and headers", func() { - checkRequestBase(httpClient, http.MethodGet, "/instances/123", testIfMatch) + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodGet, "/instances/123", expectedHeaders) }) }) }) @@ -960,7 +1102,13 @@ func TestClient_GetInstance(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, path and headers", func() { - checkRequestBase(httpClient, http.MethodGet, "/instances/123", testIfMatch) + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodGet, "/instances/123", expectedHeaders) }) }) }) @@ -985,7 +1133,11 @@ func TestClient_GetInstanceDimensionsBytes(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, path and headers", func() { - checkRequestBase(httpClient, http.MethodGet, "/instances/123/dimensions", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodGet, "/instances/123/dimensions", expectedHeaders) }) }) }) @@ -1014,7 +1166,11 @@ func TestClient_GetInstanceDimensionsBytes(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, path and headers", func() { - checkRequestBase(httpClient, http.MethodGet, "/instances/123/dimensions", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodGet, "/instances/123/dimensions", expectedHeaders) }) }) }) @@ -1060,7 +1216,10 @@ func TestClient_PostInstance(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, body and headers", func() { - checkRequestBase(httpClient, http.MethodPost, "/instances", "") + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + } + checkRequestBase(httpClient, http.MethodPost, "/instances", expectedHeaders) payload, err := ioutil.ReadAll(httpClient.DoCalls()[0].Req.Body) So(err, ShouldBeNil) So(payload, ShouldResemble, expectedPayload) @@ -1101,7 +1260,12 @@ func TestClient_GetInstances(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time", func() { - checkRequestBase(httpClient, http.MethodGet, "/instances", "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, "/instances", expectedHeaders) }) }) @@ -1116,7 +1280,12 @@ func TestClient_GetInstances(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected query parameters", func() { - checkRequestBase(httpClient, http.MethodGet, "/instances?id=123&version=999", "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, "/instances?id=123&version=999", expectedHeaders) }) }) }) @@ -1270,7 +1439,11 @@ func Test_PutInstanceImportTasks(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, path, headers and body", func() { - checkRequestBase(httpClient, http.MethodPut, "/instances/123/import_tasks", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPut, "/instances/123/import_tasks", expectedHeaders) payload, err := ioutil.ReadAll(httpClient.DoCalls()[0].Req.Body) So(err, ShouldBeNil) So(payload, ShouldResemble, expectedPayload) @@ -1310,7 +1483,11 @@ func TestClient_PostInstanceDimensions(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, path, headers and body", func() { - checkRequestBase(httpClient, http.MethodPost, "/instances/123/dimensions", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPost, "/instances/123/dimensions", expectedHeaders) payload, err := ioutil.ReadAll(httpClient.DoCalls()[0].Req.Body) So(err, ShouldBeNil) So(payload, ShouldResemble, expectedPayload) @@ -1332,7 +1509,11 @@ func TestClient_PostInstanceDimensions(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, path and headers", func() { - checkRequestBase(httpClient, http.MethodPost, "/instances/123/dimensions", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPost, "/instances/123/dimensions", expectedHeaders) payload, err := ioutil.ReadAll(httpClient.DoCalls()[0].Req.Body) So(err, ShouldBeNil) So(payload, ShouldResemble, expectedPayload) @@ -1366,7 +1547,11 @@ func TestClient_PutInstanceState(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, path, headers and body", func() { - checkRequestBase(httpClient, http.MethodPut, "/instances/123", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPut, "/instances/123", expectedHeaders) payload, err := ioutil.ReadAll(httpClient.DoCalls()[0].Req.Body) So(err, ShouldBeNil) So(payload, ShouldResemble, expectedPayload) @@ -1394,7 +1579,11 @@ func Test_UpdateInstanceWithNewInserts(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expectedmethod, path and headers", func() { - checkRequestBase(httpClient, http.MethodPut, "/instances/123/inserted_observations/999", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPut, "/instances/123/inserted_observations/999", expectedHeaders) }) }) }) @@ -1427,7 +1616,11 @@ func TestClient_PutInstanceData(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, path, headers and body", func() { - checkRequestBase(httpClient, http.MethodPut, "/instances/123", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPut, "/instances/123", expectedHeaders) payload, err := ioutil.ReadAll(httpClient.DoCalls()[0].Req.Body) So(err, ShouldBeNil) So(payload, ShouldResemble, expectedPayload) @@ -1449,7 +1642,11 @@ func TestClient_PutInstanceData(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with expected method, path, headers and body", func() { - checkRequestBase(httpClient, http.MethodPut, "/instances/123", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPut, "/instances/123", expectedHeaders) payload, err := ioutil.ReadAll(httpClient.DoCalls()[0].Req.Body) So(err, ShouldBeNil) So(payload, ShouldResemble, expectedPayload) @@ -1493,7 +1690,11 @@ func TestClient_GetInstanceDimensions(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected method, path and headers", func() { - checkRequestBase(httpClient, http.MethodGet, "/instances/123/dimensions", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodGet, "/instances/123/dimensions", expectedHeaders) }) }) }) @@ -1514,7 +1715,11 @@ func TestClient_GetInstanceDimensions(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with expected method, path and headers", func() { - checkRequestBase(httpClient, http.MethodGet, "/instances/123/dimensions", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodGet, "/instances/123/dimensions", expectedHeaders) }) }) }) @@ -1701,7 +1906,11 @@ func TestClient_PatchInstanceDimensionOption(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected patch body, method, path and headers", func() { - checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions/456/options/789", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions/456/options/789", expectedHeaders) expectedPatches := []dprequest.Patch{ {Op: dprequest.OpAdd.String(), Path: "/node_id", Value: testNodeID}, {Op: dprequest.OpAdd.String(), Path: "/order", Value: testOrder}, @@ -1719,7 +1928,11 @@ func TestClient_PatchInstanceDimensionOption(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected patch body, method, path and headers", func() { - checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions/456/options/789", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions/456/options/789", expectedHeaders) expectedPatches := []dprequest.Patch{ {Op: dprequest.OpAdd.String(), Path: "/node_id", Value: testNodeID}, } @@ -1736,7 +1949,11 @@ func TestClient_PatchInstanceDimensionOption(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected patch body, method, path and headers", func() { - checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions/456/options/789", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions/456/options/789", expectedHeaders) expectedPatches := []dprequest.Patch{ {Op: dprequest.OpAdd.String(), Path: "/order", Value: testOrder}, } @@ -1774,7 +1991,11 @@ func TestClient_PatchInstanceDimensionOption(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with expected parameters", func() { - checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions/456/options/789", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions/456/options/789", expectedHeaders) }) }) }) @@ -1824,7 +2045,12 @@ func TestClient_GetOptions(t *testing.T) { Convey("and dphttpclient.Do is called 1 time with the expected URI", func() { expectedURI := fmt.Sprintf("/datasets/%s/editions/%s/versions/%s/dimensions/%s/options?offset=%d&limit=%d", instanceID, edition, version, dimension, offset, limit) - checkRequestBase(httpClient, http.MethodGet, expectedURI, "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, expectedURI, expectedHeaders) }) }) @@ -1862,7 +2088,12 @@ func TestClient_GetOptions(t *testing.T) { Convey("and dphttpclient.Do is called 1 time with the expected URI, providing the list of IDs and no offset or limit", func() { expectedURI := fmt.Sprintf("/datasets/%s/editions/%s/versions/%s/dimensions/%s/options?id=testOption,somethingElse", instanceID, edition, version, dimension) - checkRequestBase(httpClient, http.MethodGet, expectedURI, "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, expectedURI, expectedHeaders) }) }) @@ -1899,7 +2130,12 @@ func TestClient_GetOptions(t *testing.T) { Convey("and dphttpclient.Do is called 1 time with the expected URI", func() { expectedURI := fmt.Sprintf("/datasets/%s/editions/%s/versions/%s/dimensions/%s/options", instanceID, edition, version, dimension) - checkRequestBase(httpClient, http.MethodGet, expectedURI, "") + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + } + checkRequestBase(httpClient, http.MethodGet, expectedURI, expectedHeaders) }) }) }) @@ -2086,7 +2322,11 @@ func TestClient_PatchInstanceDimensions(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected patch body, method, path and headers", func() { - checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions", expectedHeaders) expectedPatches := []dprequest.Patch{ {Op: dprequest.OpAdd.String(), Path: "/-", Value: optionUpserts}, } @@ -2103,7 +2343,11 @@ func TestClient_PatchInstanceDimensions(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with the expected patch body, method, path and headers", func() { - checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions", expectedHeaders) expectedPatches := []dprequest.Patch{ {Op: dprequest.OpAdd.String(), Path: "/dim1/options/op1/node_id", Value: "node1"}, {Op: dprequest.OpAdd.String(), Path: "/dim1/options/op1/order", Value: 5}, @@ -2181,7 +2425,11 @@ func TestClient_PatchInstanceDimensions(t *testing.T) { }) Convey("and dphttpclient.Do is called 1 time with expected parameters", func() { - checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions", testIfMatch) + expectedHeaders := expectedHeaders{ + ServiceToken: serviceAuthToken, + IfMatch: testIfMatch, + } + checkRequestBase(httpClient, http.MethodPatch, "/instances/123/dimensions", expectedHeaders) }) }) }) From 785475728f526cdbbb20715176abd427b177fa6f Mon Sep 17 00:00:00 2001 From: Rafa Date: Thu, 23 Mar 2023 15:04:15 +0000 Subject: [PATCH 3/4] New function to retrieve headers for Get Version --- dataset/data.go | 6 +++++- dataset/dataset.go | 15 +++++++++++++-- dataset/dataset_test.go | 28 +++++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/dataset/data.go b/dataset/data.go index 90945ff3..50710ac4 100644 --- a/dataset/data.go +++ b/dataset/data.go @@ -119,7 +119,6 @@ type Version struct { UsageNotes *[]UsageNote `json:"usage_notes,omitempty"` IsBasedOn *IsBasedOn `json:"is_based_on,omitempty"` LowestGeography string `json:"lowest_geography,omitempty"` - ETag string `json:"e_tag,omitempty"` } type UpdateInstance struct { @@ -469,6 +468,11 @@ type Temporal struct { Frequency string `json:"frequency"` } +// ResponseHedaers represents headers that are available in the HTTP response +type ResponseHeaders struct { + ETag string +} + // ToString builds a string of metadata information func (m Metadata) ToString() string { var b bytes.Buffer diff --git a/dataset/dataset.go b/dataset/dataset.go index 099fe979..4ec10976 100644 --- a/dataset/dataset.go +++ b/dataset/dataset.go @@ -567,9 +567,21 @@ func (c *Client) GetVersionsBatchProcess(ctx context.Context, userAuthToken, ser // GetVersion gets a specific version for an edition from the dataset api func (c *Client) GetVersion(ctx context.Context, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, version string) (v Version, err error) { + v, _, err = c.getVersion(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, version) + return +} + +// GetVersionWithHeaders gets a specific version for an edition from the dataset api and additional response headers +func (c *Client) GetVersionWithHeaders(ctx context.Context, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, version string) (v Version, h ResponseHeaders, err error) { + v, resp, err := c.getVersion(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, version) + h.ETag, _ = headers.GetResponseETag(resp) + return +} + +func (c *Client) getVersion(ctx context.Context, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetID, edition, version string) (v Version, resp *http.Response, err error) { uri := fmt.Sprintf("%s/datasets/%s/editions/%s/versions/%s", c.hcCli.URL, datasetID, edition, version) - resp, err := c.doGetWithAuthHeadersAndWithDownloadToken(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, uri) + resp, err = c.doGetWithAuthHeadersAndWithDownloadToken(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, uri) if err != nil { return } @@ -586,7 +598,6 @@ func (c *Client) GetVersion(ctx context.Context, userAuthToken, serviceAuthToken } err = json.Unmarshal(b, &v) - v.ETag, _ = headers.GetResponseETag(resp) return } diff --git a/dataset/dataset_test.go b/dataset/dataset_test.go index 6f41ea85..10d73110 100644 --- a/dataset/dataset_test.go +++ b/dataset/dataset_test.go @@ -272,15 +272,12 @@ func TestClient_GetVersion(t *testing.T) { datasetClient := newDatasetClient(httpClient) Convey("when GetVersion is called", func() { - So(version.ETag, ShouldBeEmpty) // Ensure etag is coming from the header - got, err := datasetClient.GetVersion(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetId, edition, versionString) Convey("Then no error is returned", func() { So(err, ShouldBeNil) }) Convey("And it returns the version", func() { - version.ETag = etag // Check the etag has been set So(got, ShouldResemble, version) }) Convey("And the relevant api call has been made", func() { @@ -294,6 +291,31 @@ func TestClient_GetVersion(t *testing.T) { checkRequestBase(httpClient, http.MethodGet, expectedUrl, expectedHeaders) }) }) + + Convey("when GetVersionWithHeaders is called", func() { + got, h, err := datasetClient.GetVersionWithHeaders(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetId, edition, versionString) + + Convey("Then no error is returned", func() { + So(err, ShouldBeNil) + }) + Convey("And it returns the version", func() { + So(got, ShouldResemble, version) + }) + Convey("And it returns the headers", func() { + So(h, ShouldNotBeNil) + So(h.ETag, ShouldEqual, etag) + }) + Convey("And the relevant api call has been made", func() { + expectedUrl := fmt.Sprintf("/datasets/%s/editions/%s/versions/%s", datasetId, edition, versionString) + expectedHeaders := expectedHeaders{ + FlorenceToken: userAuthToken, + ServiceToken: serviceAuthToken, + CollectionId: collectionID, + DownloadServiceToken: downloadServiceAuthToken, + } + checkRequestBase(httpClient, http.MethodGet, expectedUrl, expectedHeaders) + }) + }) }) } From 6486ee70dc8241225792b7a5273e6ce9937b182e Mon Sep 17 00:00:00 2001 From: Rafa Date: Fri, 24 Mar 2023 14:47:24 +0000 Subject: [PATCH 4/4] Simplify conveys --- dataset/dataset_test.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/dataset/dataset_test.go b/dataset/dataset_test.go index 10d73110..81d83c64 100644 --- a/dataset/dataset_test.go +++ b/dataset/dataset_test.go @@ -274,13 +274,10 @@ func TestClient_GetVersion(t *testing.T) { Convey("when GetVersion is called", func() { got, err := datasetClient.GetVersion(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetId, edition, versionString) - Convey("Then no error is returned", func() { + Convey("Then it returns the right values", func() { So(err, ShouldBeNil) - }) - Convey("And it returns the version", func() { So(got, ShouldResemble, version) - }) - Convey("And the relevant api call has been made", func() { + // And the relevant api call has been made expectedUrl := fmt.Sprintf("/datasets/%s/editions/%s/versions/%s", datasetId, edition, versionString) expectedHeaders := expectedHeaders{ FlorenceToken: userAuthToken, @@ -295,17 +292,12 @@ func TestClient_GetVersion(t *testing.T) { Convey("when GetVersionWithHeaders is called", func() { got, h, err := datasetClient.GetVersionWithHeaders(ctx, userAuthToken, serviceAuthToken, downloadServiceAuthToken, collectionID, datasetId, edition, versionString) - Convey("Then no error is returned", func() { + Convey("Then it returns the right values", func() { So(err, ShouldBeNil) - }) - Convey("And it returns the version", func() { So(got, ShouldResemble, version) - }) - Convey("And it returns the headers", func() { So(h, ShouldNotBeNil) So(h.ETag, ShouldEqual, etag) - }) - Convey("And the relevant api call has been made", func() { + // And the relevant api call has been made expectedUrl := fmt.Sprintf("/datasets/%s/editions/%s/versions/%s", datasetId, edition, versionString) expectedHeaders := expectedHeaders{ FlorenceToken: userAuthToken,