Merge branch 'develop' into release/1.25.0

ONSdigital · Dec 9, 2020 · e4147b8 · e4147b8
2 parents c69f998 + e555ffc
commit e4147b8
Show file tree

Hide file tree

Showing 29 changed files with 837 additions and 203 deletions.
diff --git a/api/api.go b/api/api.go
@@ -66,6 +66,8 @@ type DatasetAPI struct {
 	permissions               AuthHandler
 	instancePublishedChecker  *instance.PublishCheck
 	versionPublishedChecker   *PublishCheck
+	defaultLimit              int
+	defaultOffset             int
 }
 
 // Setup creates a new Dataset API instance and register the API routes based on the application configuration.
@@ -86,6 +88,8 @@ func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, d
 		permissions:               permissions,
 		versionPublishedChecker:   nil,
 		instancePublishedChecker:  nil,
+		defaultLimit:              cfg.MongoConfig.Limit,
+		defaultOffset:             cfg.MongoConfig.Offset,
 	}
 
 	if api.enablePrivateEndpoints {

diff --git a/api/dataset.go b/api/dataset.go
@@ -29,6 +29,8 @@ var (
 	// errors that should return a 400 status
 	datasetsBadRequest = map[error]bool{
 		errs.ErrAddUpdateDatasetBadRequest: true,
+		errs.ErrTypeMismatch:               true,
+		errs.ErrDatasetTypeInvalid:         true,
 	}
 
 	// errors that should return a 404 status
@@ -175,6 +177,19 @@ func (api *DatasetAPI) addDataset(w http.ResponseWriter, r *http.Request) {
 			return nil, errs.ErrAddUpdateDatasetBadRequest
 		}
 
+		dataType, err := models.ValidateDatasetType(ctx, dataset.Type)
+		if err != nil {
+			log.Event(ctx, "addDataset endpoint: error Invalid dataset type", log.ERROR, log.Error(err), logData)
+			return nil, err
+		}
+
+		datasetType, err := models.ValidateNomisURL(ctx, dataType.String(), dataset.NomisReferenceURL)
+		if err != nil {
+			log.Event(ctx, "addDataset endpoint: error dataset.Type mismatch", log.ERROR, log.Error(err), logData)
+			return nil, err
+		}
+
+		dataset.Type = datasetType
 		dataset.State = models.CreatedState
 		dataset.ID = datasetID
 
@@ -251,6 +266,14 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) {
 			return err
 		}
 
+		dataset.Type = currentDataset.Next.Type
+
+		_, err = models.ValidateNomisURL(ctx, dataset.Type, dataset.NomisReferenceURL)
+		if err != nil {
+			log.Event(ctx, "putDataset endpoint: error dataset.Type mismatch", log.ERROR, log.Error(err), data)
+			return err
+		}
+
 		if dataset.State == models.PublishedState {
 			if err := api.publishDataset(ctx, currentDataset, nil); err != nil {
 				log.Event(ctx, "putDataset endpoint: failed to update dataset document to published", log.ERROR, log.Error(err), data)

diff --git a/api/dataset_test.go b/api/dataset_test.go
@@ -33,10 +33,9 @@ const (
 )
 
 var (
-	datasetPayload = `{"contacts":[{"email":"[email protected]","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","periodicity":"yearly","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"}}`
-
-	urlBuilder = url.NewBuilder("localhost:20000")
-	mu         sync.Mutex
+	datasetPayload = `{"contacts":[{"email":"[email protected]","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"type":"nomis","nomis_reference_url":"https://www.nomis.co.uk"}`
+	urlBuilder     = url.NewBuilder("localhost:20000")
+	mu             sync.Mutex
 )
 
 func getAuthorisationHandlerMock() *mocks.AuthHandlerMock {
@@ -54,6 +53,8 @@ func GetAPIWithMocks(mockedDataStore store.Storer, mockedGeneratedDownloads Down
 	cfg.ServiceAuthToken = authToken
 	cfg.DatasetAPIURL = host
 	cfg.EnablePrivateEndpoints = true
+	cfg.MongoConfig.Limit = 0
+	cfg.MongoConfig.Offset = 0
 
 	return Setup(testContext, cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedGeneratedDownloads, datasetPermissions, permissions)
 }
@@ -397,6 +398,107 @@ func TestPostDatasetReturnsError(t *testing.T) {
 			So(err, ShouldEqual, io.EOF)
 		})
 	})
+
+	Convey("When the request has an filterable datatype and nomis url it should return type mismatch error", t, func() {
+		var b string
+		b = `{"contacts":[{"email":"[email protected]","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"type":"filterable","nomis_reference_url":"https://www.nomis.co.uk"}`
+
+		r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123123", bytes.NewBufferString(b))
+		So(err, ShouldBeNil)
+
+		w := httptest.NewRecorder()
+		mockedDataStore := &storetest.StorerMock{
+			GetDatasetFunc: func(string) (*models.DatasetUpdate, error) {
+				return nil, errs.ErrDatasetNotFound
+			},
+			UpsertDatasetFunc: func(string, *models.DatasetUpdate) error {
+				return nil
+			},
+		}
+
+		datasetPermissions := getAuthorisationHandlerMock()
+		permissions := getAuthorisationHandlerMock()
+		api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions)
+		api.Router.ServeHTTP(w, r)
+
+		So(w.Body.String(), ShouldResemble, "type mismatch\n")
+		So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1)
+		So(w.Code, ShouldEqual, http.StatusBadRequest)
+		So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 0)
+
+		Convey("then the request body has been drained", func() {
+			_, err = r.Body.Read(make([]byte, 1))
+			So(err, ShouldEqual, io.EOF)
+		})
+	})
+
+	Convey("When the request has an invalid datatype it should return invalid type errorq", t, func() {
+		var b string
+		b = `{"contacts":[{"email":"[email protected]","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"type":"nomis_filterable"}`
+
+		r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(b))
+		So(err, ShouldBeNil)
+
+		w := httptest.NewRecorder()
+		mockedDataStore := &storetest.StorerMock{
+			GetDatasetFunc: func(string) (*models.DatasetUpdate, error) {
+				return nil, errs.ErrDatasetNotFound
+			},
+			UpsertDatasetFunc: func(string, *models.DatasetUpdate) error {
+				return nil
+			},
+		}
+
+		datasetPermissions := getAuthorisationHandlerMock()
+		permissions := getAuthorisationHandlerMock()
+		api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions)
+		api.Router.ServeHTTP(w, r)
+
+		So(w.Code, ShouldEqual, http.StatusBadRequest)
+		So(w.Body.String(), ShouldResemble, "invalid dataset type\n")
+		So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1)
+		So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 0)
+
+		Convey("then the request body has been drained", func() {
+			_, err = r.Body.Read(make([]byte, 1))
+			So(err, ShouldEqual, io.EOF)
+		})
+	})
+
+	Convey("When the request body has an empty type field it should create a dataset with type defaulted to filterable", t, func() {
+		var b string
+		b = `{"contacts":[{"email":"[email protected]","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"type":""}`
+		res := `{"id":"123123","next":{"contacts":[{"email":"[email protected]","name":"John Cox","telephone":"01623 456789"}],"description":"census","id":"123123","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"},"editions":{"href":"http://localhost:22000/datasets/123123/editions"},"self":{"href":"http://localhost:22000/datasets/123123"}},"next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department"},"state":"created","theme":"population","title":"CensusEthnicity","type":"filterable"}}`
+		r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123123", bytes.NewBufferString(b))
+		So(err, ShouldBeNil)
+
+		w := httptest.NewRecorder()
+		mockedDataStore := &storetest.StorerMock{
+			GetDatasetFunc: func(string) (*models.DatasetUpdate, error) {
+				return nil, errs.ErrDatasetNotFound
+			},
+			UpsertDatasetFunc: func(string, *models.DatasetUpdate) error {
+				return nil
+			},
+		}
+
+		datasetPermissions := getAuthorisationHandlerMock()
+		permissions := getAuthorisationHandlerMock()
+		mockedDataStore.UpsertDataset("123123", &models.DatasetUpdate{Next: &models.Dataset{}})
+		api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions)
+		api.Router.ServeHTTP(w, r)
+
+		So(w.Code, ShouldEqual, http.StatusCreated)
+		So(w.Body.String(), ShouldContainSubstring, res)
+		So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1)
+		So(mockedDataStore.UpsertDatasetCalls(), ShouldHaveLength, 2)
+
+		Convey("then the request body has been drained", func() {
+			_, err = r.Body.Read(make([]byte, 1))
+			So(err, ShouldEqual, io.EOF)
+		})
+	})
+
 }
 
 func TestPutDatasetReturnsSuccessfully(t *testing.T) {
@@ -410,7 +512,7 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) {
 		w := httptest.NewRecorder()
 		mockedDataStore := &storetest.StorerMock{
 			GetDatasetFunc: func(string) (*models.DatasetUpdate, error) {
-				return &models.DatasetUpdate{Next: &models.Dataset{}}, nil
+				return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil
 			},
 			UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error {
 				return nil
@@ -439,6 +541,41 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) {
 			So(err, ShouldEqual, io.EOF)
 		})
 	})
+
+	Convey("When update dataset type has a value of filterable and stored dataset type is nomis return status ok", t, func() {
+		// Dataset type field cannot be updated and hence is ignored in any updates to the dataset
+
+		var b string
+		b = `{"contacts":[{"email":"[email protected]","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"type":"filterable","nomis_reference_url":"https://www.nomis.co.uk"}`
+
+		r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b))
+		So(err, ShouldBeNil)
+
+		w := httptest.NewRecorder()
+
+		mockedDataStore := &storetest.StorerMock{
+			GetDatasetFunc: func(string) (*models.DatasetUpdate, error) {
+				return &models.DatasetUpdate{Next: &models.Dataset{Type: "nomis"}}, nil
+			},
+			UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error {
+				return nil
+			},
+		}
+
+		datasetPermissions := getAuthorisationHandlerMock()
+		permissions := getAuthorisationHandlerMock()
+		api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions)
+
+		api.Router.ServeHTTP(w, r)
+		So(w.Code, ShouldEqual, http.StatusOK)
+		So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1)
+		So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 1)
+
+		Convey("then the request body has been drained", func() {
+			_, err = r.Body.Read(make([]byte, 1))
+			So(err, ShouldEqual, io.EOF)
+		})
+	})
 }
 
 func TestPutDatasetReturnsError(t *testing.T) {
@@ -556,6 +693,41 @@ func TestPutDatasetReturnsError(t *testing.T) {
 		})
 	})
 
+	Convey("When updating the dataset nomis_reference_url and the stored dataset type is not nomis return bad request", t, func() {
+		var b string
+		b = `{"contacts":[{"email":"[email protected]","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"},"nomis_reference_url":"https://www.nomis.co.uk"}`
+
+		r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b))
+		So(err, ShouldBeNil)
+
+		w := httptest.NewRecorder()
+
+		mockedDataStore := &storetest.StorerMock{
+			GetDatasetFunc: func(string) (*models.DatasetUpdate, error) {
+				return &models.DatasetUpdate{Next: &models.Dataset{Type: "filterable"}}, nil
+			},
+			UpdateDatasetFunc: func(context.Context, string, *models.Dataset, string) error {
+				return nil
+			},
+		}
+
+		datasetPermissions := getAuthorisationHandlerMock()
+		permissions := getAuthorisationHandlerMock()
+
+		api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions)
+
+		api.Router.ServeHTTP(w, r)
+		So(w.Code, ShouldEqual, http.StatusBadRequest)
+		So(w.Body.String(), ShouldResemble, errs.ErrTypeMismatch.Error()+"\n")
+		So(mockedDataStore.GetDatasetCalls(), ShouldHaveLength, 1)
+		So(mockedDataStore.UpdateDatasetCalls(), ShouldHaveLength, 0)
+
+		Convey("then the request body has been drained", func() {
+			_, err = r.Body.Read(make([]byte, 1))
+			So(err, ShouldEqual, io.EOF)
+		})
+	})
+
 	Convey("When the request is not authorised to update dataset return status unauthorised", t, func() {
 		var b string
 		b = "{\"edition\":\"2017\",\"state\":\"created\",\"license\":\"ONS\",\"release_date\":\"2017-04-04\",\"version\":\"1\"}"