diff --git a/Makefile b/Makefile index 6bda65d3..b4fae539 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,8 @@ build: go build -o $(BUILD_ARCH)/$(BIN_DIR)/dp-dataset-api main.go debug: HUMAN_LOG=1 go run main.go - +acceptance: build + MONGODB_DATABASE=test HUMAN_LOG=1 go run main.go test: go test -cover $(shell go list ./... | grep -v /vendor/) diff --git a/README.md b/README.md index 7e913907..3d404ac5 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,6 @@ See [CONTRIBUTING](CONTRIBUTING.md) for details. ### License -Copyright © 2016-2017, Office for National Statistics (https://www.ons.gov.uk) +Copyright © 2016-2018, Office for National Statistics (https://www.ons.gov.uk) Released under MIT license, see [LICENSE](LICENSE.md) for details diff --git a/api/api.go b/api/api.go index 8c94a1e7..9f67b9d3 100644 --- a/api/api.go +++ b/api/api.go @@ -70,6 +70,7 @@ func routes(host, secretKey string, router *mux.Router, dataStore store.DataStor api.router.HandleFunc("/healthcheck", api.healthCheck).Methods("GET") + versionPublishChecker := PublishCheck{Datastore: dataStore.Backend} api.router.HandleFunc("/datasets", api.getDatasets).Methods("GET") api.router.HandleFunc("/datasets/{id}", api.privateAuth.Check(api.addDataset)).Methods("POST") api.router.HandleFunc("/datasets/{id}", api.getDataset).Methods("GET") @@ -78,25 +79,30 @@ func routes(host, secretKey string, router *mux.Router, dataStore store.DataStor api.router.HandleFunc("/datasets/{id}/editions/{edition}", api.getEdition).Methods("GET") api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions", api.getVersions).Methods("GET") api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}", api.getVersion).Methods("GET") - api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}", api.privateAuth.Check(api.putVersion)).Methods("PUT") + api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}", api.privateAuth.Check(versionPublishChecker.Check(api.putVersion))).Methods("PUT") api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}/metadata", api.getMetadata).Methods("GET") api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}/dimensions", api.getDimensions).Methods("GET") api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}/dimensions/{dimension}/options", api.getDimensionOptions).Methods("GET") - instance := instance.Store{Host: api.host, Storer: api.dataStore.Backend} - api.router.HandleFunc("/instances", api.privateAuth.Check(instance.GetList)).Methods("GET") - api.router.HandleFunc("/instances", api.privateAuth.Check(instance.Add)).Methods("POST") - api.router.HandleFunc("/instances/{id}", api.privateAuth.Check(instance.Get)).Methods("GET") - api.router.HandleFunc("/instances/{id}", api.privateAuth.Check(instance.Update)).Methods("PUT") - api.router.HandleFunc("/instances/{id}/events", api.privateAuth.Check(instance.AddEvent)).Methods("POST") - api.router.HandleFunc("/instances/{id}/inserted_observations/{inserted_observations}", api.privateAuth.Check(instance.UpdateObservations)).Methods("PUT") - api.router.HandleFunc("/instances/{id}/import_tasks", api.privateAuth.Check(instance.UpdateImportTask)).Methods("PUT") + instanceAPI := instance.Store{Host: api.host, Storer: api.dataStore.Backend} + instancePublishChecker := instance.PublishCheck{Datastore: dataStore.Backend} + api.router.HandleFunc("/instances", api.privateAuth.Check(instanceAPI.GetList)).Methods("GET") + api.router.HandleFunc("/instances", api.privateAuth.Check(instanceAPI.Add)).Methods("POST") + api.router.HandleFunc("/instances/{id}", api.privateAuth.Check(instanceAPI.Get)).Methods("GET") + api.router.HandleFunc("/instances/{id}", api.privateAuth.Check(instancePublishChecker.Check(instanceAPI.Update))).Methods("PUT") + api.router.HandleFunc("/instances/{id}/dimensions/{dimension}", api.privateAuth.Check(instancePublishChecker.Check(instanceAPI.UpdateDimension))).Methods("PUT") + api.router.HandleFunc("/instances/{id}/events", api.privateAuth.Check(instanceAPI.AddEvent)).Methods("POST") + api.router.HandleFunc("/instances/{id}/inserted_observations/{inserted_observations}", + api.privateAuth.Check(instancePublishChecker.Check(instanceAPI.UpdateObservations))).Methods("PUT") + api.router.HandleFunc("/instances/{id}/import_tasks", api.privateAuth.Check(instancePublishChecker.Check(instanceAPI.UpdateImportTask))).Methods("PUT") dimension := dimension.Store{Storer: api.dataStore.Backend} api.router.HandleFunc("/instances/{id}/dimensions", dimension.GetNodes).Methods("GET") - api.router.HandleFunc("/instances/{id}/dimensions", api.privateAuth.Check(dimension.Add)).Methods("POST") + api.router.HandleFunc("/instances/{id}/dimensions", api.privateAuth.Check(instancePublishChecker.Check(dimension.Add))).Methods("POST") api.router.HandleFunc("/instances/{id}/dimensions/{dimension}/options", dimension.GetUnique).Methods("GET") - api.router.HandleFunc("/instances/{id}/dimensions/{dimension}/options/{value}/node_id/{node_id}", api.privateAuth.Check(dimension.AddNodeID)).Methods("PUT") + api.router.HandleFunc("/instances/{id}/dimensions/{dimension}/options/{value}/node_id/{node_id}", + api.privateAuth.Check(instancePublishChecker.Check(dimension.AddNodeID))).Methods("PUT") + return &api } diff --git a/api/dataset.go b/api/dataset.go index e7891582..0d4c07e8 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -8,6 +8,7 @@ import ( errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" + "github.com/ONSdigital/dp-dataset-api/store" "github.com/ONSdigital/go-ns/log" "github.com/gorilla/mux" "github.com/pkg/errors" @@ -310,30 +311,18 @@ func (api *DatasetAPI) addDataset(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() dataset.State = models.CreatedState + dataset.ID = datasetID - var accessRights string - if dataset.Links != nil { - if dataset.Links.AccessRights != nil { - if dataset.Links.AccessRights.HRef != "" { - accessRights = dataset.Links.AccessRights.HRef - } - } + if dataset.Links == nil { + dataset.Links = &models.DatasetLinks{} } - dataset.ID = datasetID - dataset.Links = &models.DatasetLinks{ - Editions: &models.LinkObject{ - HRef: fmt.Sprintf("%s/datasets/%s/editions", api.host, datasetID), - }, - Self: &models.LinkObject{ - HRef: fmt.Sprintf("%s/datasets/%s", api.host, datasetID), - }, - } - - if accessRights != "" { - dataset.Links.AccessRights = &models.LinkObject{ - HRef: accessRights, - } + dataset.Links.Editions = &models.LinkObject{ + HRef: fmt.Sprintf("%s/datasets/%s/editions", api.host, datasetID), + } + + dataset.Links.Self = &models.LinkObject{ + HRef: fmt.Sprintf("%s/datasets/%s", api.host, datasetID), } dataset.LastUpdated = time.Now() @@ -378,12 +367,27 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) { } defer r.Body.Close() - if err := api.dataStore.Backend.UpdateDataset(datasetID, dataset); err != nil { - log.ErrorC("failed to update dataset resource", err, log.Data{"dataset_id": datasetID}) + currentDataset, err := api.dataStore.Backend.GetDataset(datasetID) + if err != nil { + log.ErrorC("failed to find dataset", err, log.Data{"dataset_id": datasetID}) handleErrorType(datasetDocType, err, w) return } + if dataset.State == models.PublishedState { + if err := api.publishDataset(currentDataset, nil); err != nil { + log.ErrorC("failed to update dataset document to published", err, log.Data{"dataset_id": datasetID}) + handleErrorType(versionDocType, err, w) + return + } + } else { + if err := api.dataStore.Backend.UpdateDataset(datasetID, dataset, currentDataset.Next.State); err != nil { + log.ErrorC("failed to update dataset resource", err, log.Data{"dataset_id": datasetID}) + handleErrorType(datasetDocType, err, w) + return + } + } + setJSONContentType(w) w.WriteHeader(http.StatusOK) log.Debug("update dataset", log.Data{"dataset_id": datasetID}) @@ -396,14 +400,15 @@ func (api *DatasetAPI) putVersion(w http.ResponseWriter, r *http.Request) { version := vars["version"] versionDoc, err := models.CreateVersion(r.Body) + defer r.Body.Close() if err != nil { log.ErrorC("failed to model version resource based on request", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) http.Error(w, err.Error(), http.StatusBadRequest) return } - defer r.Body.Close() - if err = api.dataStore.Backend.CheckDatasetExists(datasetID, ""); err != nil { + currentDataset, err := api.dataStore.Backend.GetDataset(datasetID) + if err != nil { log.ErrorC("failed to find dataset", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) handleErrorType(versionDocType, err, w) return @@ -422,14 +427,6 @@ func (api *DatasetAPI) putVersion(w http.ResponseWriter, r *http.Request) { return } - // Check current state of version document - if currentVersion.State == models.PublishedState { - err = fmt.Errorf("unable to update document, already published") - log.Error(err, nil) - http.Error(w, err.Error(), http.StatusForbidden) - return - } - // Combine update version document to existing version document newVersion := createNewVersionDoc(currentVersion, versionDoc) log.Debug("combined current version document with update request", log.Data{"dataset_id": datasetID, "edition": edition, "version": version, "updated_version": newVersion}) @@ -454,7 +451,7 @@ func (api *DatasetAPI) putVersion(w http.ResponseWriter, r *http.Request) { } // Pass in newVersion variable to include relevant data needed for update on dataset API (e.g. links) - if err := api.publishDataset(datasetID, newVersion); err != nil { + if err := api.publishDataset(currentDataset, newVersion); err != nil { log.ErrorC("failed to update dataset document once version state changes to publish", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) handleErrorType(versionDocType, err, w) return @@ -598,37 +595,16 @@ func createNewVersionDoc(currentVersion *models.Version, version *models.Version return version } -func (api *DatasetAPI) publishDataset(id string, version *models.Version) error { - currentDataset, err := api.dataStore.Backend.GetDataset(id) - if err != nil { - log.ErrorC("unable to update dataset", err, log.Data{"dataset_id": id}) - return err - } - - var accessRights string +func (api *DatasetAPI) publishDataset(currentDataset *models.DatasetUpdate, version *models.Version) error { + if version != nil { + currentDataset.Next.CollectionID = version.CollectionID - if currentDataset.Next.Links != nil { - if currentDataset.Next.Links.AccessRights != nil { - accessRights = currentDataset.Next.Links.AccessRights.HRef - } - } - - currentDataset.Next.CollectionID = version.CollectionID - currentDataset.Next.Links = &models.DatasetLinks{ - AccessRights: &models.LinkObject{ - HRef: accessRights, - }, - Editions: &models.LinkObject{ - HRef: fmt.Sprintf("%s/datasets/%s/editions", api.host, version.Links.Dataset.ID), - }, - LatestVersion: &models.LinkObject{ + currentDataset.Next.Links.LatestVersion = &models.LinkObject{ ID: version.Links.Version.ID, HRef: version.Links.Version.HRef, - }, - Self: &models.LinkObject{ - HRef: fmt.Sprintf("%s/datasets/%s", api.host, version.Links.Dataset.ID), - }, + } } + currentDataset.Next.State = models.PublishedState currentDataset.Next.LastUpdated = time.Now() @@ -642,8 +618,8 @@ func (api *DatasetAPI) publishDataset(id string, version *models.Version) error Next: currentDataset.Next, } - if err := api.dataStore.Backend.UpsertDataset(id, newDataset); err != nil { - log.ErrorC("unable to update dataset", err, log.Data{"dataset_id": id}) + if err := api.dataStore.Backend.UpsertDataset(currentDataset.ID, newDataset); err != nil { + log.ErrorC("unable to update dataset", err, log.Data{"dataset_id": currentDataset.ID}) return err } @@ -701,10 +677,13 @@ func (api *DatasetAPI) getDimensions(w http.ResponseWriter, r *http.Request) { } func (api *DatasetAPI) createListOfDimensions(versionDoc *models.Version, dimensions []bson.M) ([]models.Dimension, error) { + // Get dimension description from the version document and add to hash map dimensionDescriptions := make(map[string]string) + dimensionLabels := make(map[string]string) for _, details := range versionDoc.Dimensions { dimensionDescriptions[details.Name] = details.Description + dimensionLabels[details.Name] = details.Label } var results []models.Dimension @@ -722,6 +701,7 @@ func (api *DatasetAPI) createListOfDimensions(versionDoc *models.Version, dimens // Add description to dimension from hash map dimension.Description = dimensionDescriptions[dimension.Name] + dimension.Label = dimensionLabels[dimension.Name] results = append(results, dimension) } @@ -748,7 +728,21 @@ func (api *DatasetAPI) getDimensionOptions(w http.ResponseWriter, r *http.Reques versionID := vars["version"] dimension := vars["dimension"] - results, err := api.dataStore.Backend.GetDimensionOptions(datasetID, editionID, versionID, dimension) + var state string + authenticated := true + if r.Header.Get(internalToken) != api.internalToken { + state = models.PublishedState + authenticated = false + } + + version, err := api.dataStore.Backend.GetVersion(datasetID, editionID, versionID, state) + if err != nil { + log.ErrorC("failed to get version", err, log.Data{"dataset_id": datasetID, "edition": editionID, "version": versionID, "authenticated": authenticated}) + handleErrorType(versionDocType, err, w) + return + } + + results, err := api.dataStore.Backend.GetDimensionOptions(version, dimension) if err != nil { log.ErrorC("failed to get a list of dimension options", err, log.Data{"dataset_id": datasetID, "edition": editionID, "version": versionID, "dimension": dimension}) handleErrorType(dimensionOptionDocType, err, w) @@ -896,6 +890,44 @@ func handleErrorType(docType string, err error, w http.ResponseWriter) { } } +// PublishCheck Checks if an version has been published +type PublishCheck struct { + Datastore store.Storer +} + +// Check wraps a HTTP handle. Checks that the state is not published +func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request)) http.HandlerFunc { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + vars := mux.Vars(r) + id := vars["id"] + edition := vars["edition"] + version := vars["version"] + + versionDoc, err := d.Datastore.GetVersion(id, edition, version, "") + if err != nil { + if err != errs.ErrVersionNotFound { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + // If document cannot be found do not handle error + handle(w, r) + return + } + + if versionDoc != nil { + if versionDoc.State == models.PublishedState { + err = errors.New("unable to update version as it has been published") + log.Error(err, log.Data{"version": versionDoc}) + http.Error(w, err.Error(), http.StatusForbidden) + return + } + } + + handle(w, r) + }) +} + func setJSONContentType(w http.ResponseWriter) { w.Header().Set("Content-Type", "application/json") } diff --git a/api/dataset_test.go b/api/dataset_test.go index beee92e3..c628525b 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -13,7 +13,6 @@ import ( "gopkg.in/mgo.v2/bson" - errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/mocks" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/store" @@ -23,6 +22,7 @@ import ( "github.com/ONSdigital/dp-dataset-api/url" "github.com/gorilla/mux" + errs "github.com/ONSdigital/dp-dataset-api/apierrors" . "github.com/smartystreets/goconvey/convey" ) @@ -212,7 +212,7 @@ func TestGetEditionsReturnsError(t *testing.T) { Convey("When the dataset does not exist return status bad request", t, func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123-456/editions", nil) - r.Header.Add("internal_token", "coffee") + r.Header.Add("internal-token", "coffee") w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(datasetID, state string) error { @@ -229,7 +229,7 @@ func TestGetEditionsReturnsError(t *testing.T) { Convey("When no editions exist against an existing dataset return status not found", t, func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123-456/editions", nil) - r.Header.Add("internal_token", "coffee") + r.Header.Add("internal-token", "coffee") w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(datasetID, state string) error { @@ -309,7 +309,7 @@ func TestGetEditionReturnsError(t *testing.T) { Convey("When the dataset does not exist return status bad request", t, func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123-456/editions/678", nil) - r.Header.Add("internal_token", "coffee") + r.Header.Add("internal-token", "coffee") w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(datasetID, state string) error { @@ -326,7 +326,7 @@ func TestGetEditionReturnsError(t *testing.T) { Convey("When edition does not exist for a dataset return status not found", t, func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123-456/editions/678", nil) - r.Header.Add("internal_token", "coffee") + r.Header.Add("internal-token", "coffee") w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(datasetID, state string) error { @@ -648,7 +648,7 @@ func TestPostDatasetsReturnsCreated(t *testing.T) { return nil }, } - mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{}) + mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{Next: &models.Dataset{}}) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}) api.router.ServeHTTP(w, r) @@ -762,7 +762,10 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) { So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - UpdateDatasetFunc: func(string, *models.Dataset) error { + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{}}, nil + }, + UpdateDatasetFunc: func(string, *models.Dataset, string) error { return nil }, } @@ -770,11 +773,12 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) { dataset := &models.Dataset{ Title: "CPI", } - mockedDataStore.UpdateDataset("123", dataset) + mockedDataStore.UpdateDataset("123", dataset, models.CreatedState) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}) api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 2) }) } @@ -790,7 +794,10 @@ func TestPutDatasetReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - UpdateDatasetFunc: func(string, *models.Dataset) error { + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{}}, nil + }, + UpdateDatasetFunc: func(string, *models.Dataset, string) error { return errBadRequest }, } @@ -798,7 +805,8 @@ func TestPutDatasetReturnsError(t *testing.T) { api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}) api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) - So(len(mockedDataStore.UpsertVersionCalls()), ShouldEqual, 0) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 0) }) Convey("When the api cannot connect to datastore return an internal server error", t, func() { @@ -810,7 +818,10 @@ func TestPutDatasetReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - UpdateDatasetFunc: func(string, *models.Dataset) error { + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{State: models.CreatedState}}, nil + }, + UpdateDatasetFunc: func(string, *models.Dataset, string) error { return errInternal }, } @@ -818,11 +829,12 @@ func TestPutDatasetReturnsError(t *testing.T) { dataset := &models.Dataset{ Title: "CPI", } - mockedDataStore.UpdateDataset("123", dataset) + mockedDataStore.UpdateDataset("123", dataset, models.CreatedState) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}) api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 2) }) @@ -835,20 +847,19 @@ func TestPutDatasetReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - UpdateDatasetFunc: func(string, *models.Dataset) error { + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound + }, + UpdateDatasetFunc: func(string, *models.Dataset, string) error { return errs.ErrDatasetNotFound }, } - dataset := &models.Dataset{ - Title: "CPI", - } - mockedDataStore.UpdateDataset("123", dataset) - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}) api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 2) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 0) }) Convey("When the request does not contain a valid internal token return status unauthorised", t, func() { @@ -859,7 +870,10 @@ func TestPutDatasetReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - UpdateDatasetFunc: func(string, *models.Dataset) error { + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Next: &models.Dataset{}}, nil + }, + UpdateDatasetFunc: func(string, *models.Dataset, string) error { return nil }, } @@ -867,6 +881,7 @@ func TestPutDatasetReturnsError(t *testing.T) { api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}) api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusUnauthorized) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 0) }) } @@ -888,8 +903,8 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return nil + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, CheckEditionExistsFunc: func(string, string, string) error { return nil @@ -928,9 +943,9 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 3) So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateEditionCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) @@ -953,8 +968,8 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return nil + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, CheckEditionExistsFunc: func(string, string, string) error { return nil @@ -979,9 +994,9 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 3) So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateDatasetWithAssociationCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateEditionCalls()), ShouldEqual, 0) @@ -1007,8 +1022,8 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return nil + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, CheckEditionExistsFunc: func(string, string, string) error { return nil @@ -1039,9 +1054,9 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { } So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetWithAssociationCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateEditionCalls()), ShouldEqual, 0) @@ -1064,9 +1079,6 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return nil - }, CheckEditionExistsFunc: func(string, string, string) error { return nil }, @@ -1105,8 +1117,8 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { return &models.DatasetUpdate{ ID: "123", - Next: &models.Dataset{}, - Current: &models.Dataset{}, + Next: &models.Dataset{Links: &models.DatasetLinks{}}, + Current: &models.Dataset{Links: &models.DatasetLinks{}}, }, nil }, UpsertDatasetFunc: func(string, *models.DatasetUpdate) error { @@ -1117,14 +1129,13 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { mockedDataStore.UpdateVersion("a1b2c3", &models.Version{}) mockedDataStore.UpdateEdition("123", "2017", &models.Version{State: "published"}) mockedDataStore.GetDataset("123") - mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{}) + mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{Next: &models.Dataset{}}) api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock) api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 3) So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateEditionCalls()), ShouldEqual, 2) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 2) @@ -1146,8 +1157,8 @@ func TestPutVersionGenerateDownloadsError(t *testing.T) { GetVersionFunc: func(datasetID string, editionID string, version string, state string) (*models.Version, error) { return &v, nil }, - CheckDatasetExistsFunc: func(ID string, state string) error { - return nil + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, CheckEditionExistsFunc: func(ID string, editionID string, state string) error { return nil @@ -1181,14 +1192,14 @@ func TestPutVersionGenerateDownloadsError(t *testing.T) { Convey("and the expected store calls are made with the expected parameters", func() { genCalls := mockDownloadGenerator.GenerateCalls() - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) - So(mockedDataStore.CheckDatasetExistsCalls()[0].ID, ShouldEqual, "123") + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(mockedDataStore.GetDatasetCalls()[0].ID, ShouldEqual, "123") So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(mockedDataStore.CheckEditionExistsCalls()[0].ID, ShouldEqual, "123") So(mockedDataStore.CheckEditionExistsCalls()[0].EditionID, ShouldEqual, "2017") - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) So(mockedDataStore.GetVersionCalls()[0].DatasetID, ShouldEqual, "123") So(mockedDataStore.GetVersionCalls()[0].EditionID, ShouldEqual, "2017") So(mockedDataStore.GetVersionCalls()[0].Version, ShouldEqual, "1") @@ -1215,8 +1226,8 @@ func TestPutEmptyVersion(t *testing.T) { GetVersionFunc: func(datasetID string, editionID string, version string, state string) (*models.Version, error) { return &v, nil }, - CheckDatasetExistsFunc: func(ID string, state string) error { - return nil + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, CheckEditionExistsFunc: func(ID string, editionID string, state string) error { return nil @@ -1239,6 +1250,7 @@ func TestPutEmptyVersion(t *testing.T) { }) Convey("and the updated version is as expected", func() { + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 1) So(mockedDataStore.UpdateVersionCalls()[0].Version.Downloads, ShouldBeNil) }) @@ -1251,8 +1263,8 @@ func TestPutEmptyVersion(t *testing.T) { v.Downloads = xlsDownload return &v, nil }, - CheckDatasetExistsFunc: func(ID string, state string) error { - return nil + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, CheckEditionExistsFunc: func(ID string, editionID string, state string) error { return nil @@ -1282,16 +1294,15 @@ func TestPutEmptyVersion(t *testing.T) { }) Convey("and the expected external calls are made with the correct parameters", func() { - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) - So(mockedDataStore.CheckDatasetExistsCalls()[0].ID, ShouldEqual, "123") - So(mockedDataStore.CheckDatasetExistsCalls()[0].State, ShouldEqual, "") + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(mockedDataStore.GetDatasetCalls()[0].ID, ShouldEqual, "123") So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(mockedDataStore.CheckEditionExistsCalls()[0].ID, ShouldEqual, "123") So(mockedDataStore.CheckEditionExistsCalls()[0].EditionID, ShouldEqual, "2017") So(mockedDataStore.CheckEditionExistsCalls()[0].State, ShouldEqual, "") - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) So(mockedDataStore.GetVersionCalls()[0].DatasetID, ShouldEqual, "123") So(mockedDataStore.GetVersionCalls()[0].EditionID, ShouldEqual, "2017") So(mockedDataStore.GetVersionCalls()[0].Version, ShouldEqual, "1") @@ -1322,7 +1333,10 @@ func TestPutVersionReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetVersionFunc: func(string, string, string, string) (*models.Version, error) { - return &models.Version{}, nil + return &models.Version{State: models.AssociatedState}, nil + }, + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, } @@ -1330,7 +1344,8 @@ func TestPutVersionReturnsError(t *testing.T) { api.router.ServeHTTP(w, r) So(w.Body.String(), ShouldEqual, "Failed to parse json body\n") So(w.Code, ShouldEqual, http.StatusBadRequest) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) }) @@ -1348,11 +1363,11 @@ func TestPutVersionReturnsError(t *testing.T) { So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return errInternal + GetVersionFunc: func(string, string, string, string) (*models.Version, error) { + return nil, errInternal }, - CheckEditionExistsFunc: func(string, string, string) error { - return nil + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, } @@ -1360,8 +1375,8 @@ func TestPutVersionReturnsError(t *testing.T) { api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) So(w.Body.String(), ShouldEqual, "internal error\n") - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) }) @@ -1379,8 +1394,11 @@ func TestPutVersionReturnsError(t *testing.T) { So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return errs.ErrDatasetNotFound + GetVersionFunc: func(string, string, string, string) (*models.Version, error) { + return &models.Version{}, errs.ErrVersionNotFound + }, + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound }, CheckEditionExistsFunc: func(string, string, string) error { return nil @@ -1391,7 +1409,8 @@ func TestPutVersionReturnsError(t *testing.T) { api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) So(w.Body.String(), ShouldEqual, "Dataset not found\n") - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) }) @@ -1410,8 +1429,11 @@ func TestPutVersionReturnsError(t *testing.T) { So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return nil + GetVersionFunc: func(string, string, string, string) (*models.Version, error) { + return &models.Version{}, errs.ErrVersionNotFound + }, + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, CheckEditionExistsFunc: func(string, string, string) error { return errs.ErrEditionNotFound @@ -1422,9 +1444,9 @@ func TestPutVersionReturnsError(t *testing.T) { api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) So(w.Body.String(), ShouldEqual, "Edition not found\n") - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) }) @@ -1442,15 +1464,15 @@ func TestPutVersionReturnsError(t *testing.T) { So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return nil + GetVersionFunc: func(string, string, string, string) (*models.Version, error) { + return &models.Version{}, errs.ErrVersionNotFound + }, + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, CheckEditionExistsFunc: func(string, string, string) error { return nil }, - GetVersionFunc: func(string, string, string, string) (*models.Version, error) { - return nil, errs.ErrVersionNotFound - }, UpdateVersionFunc: func(string, *models.Version) error { return nil }, @@ -1460,9 +1482,9 @@ func TestPutVersionReturnsError(t *testing.T) { api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) So(w.Body.String(), ShouldEqual, "Version not found\n") - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) }) @@ -1480,57 +1502,19 @@ func TestPutVersionReturnsError(t *testing.T) { So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return nil - }, - } - - api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock) - api.router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusUnauthorized) - So(w.Body.String(), ShouldEqual, "No authentication header provided\n") - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 0) - So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) - }) - - Convey("Given the version doc is 'published', when we try to set state to 'completed', then we see a status of forbidden", t, func() { - generatorMock := &mocks.DownloadsGeneratorMock{ - GenerateFunc: func(string, string, string, string) error { - return nil - }, - } - - var b string - b = versionPayload - r, err := http.NewRequest("PUT", "http://localhost:22000/datasets/123/editions/2017/versions/1", bytes.NewBufferString(b)) - r.Header.Add("internal-token", "coffee") - So(err, ShouldBeNil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return nil - }, - CheckEditionExistsFunc: func(string, string, string) error { - return nil - }, GetVersionFunc: func(string, string, string, string) (*models.Version, error) { return &models.Version{ - State: models.PublishedState, + State: "associated", }, nil }, - UpdateVersionFunc: func(string, *models.Version) error { - return nil - }, } api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock) api.router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusForbidden) - So(w.Body.String(), ShouldEqual, "unable to update document, already published\n") - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 0) + So(w.Code, ShouldEqual, http.StatusUnauthorized) + So(w.Body.String(), ShouldEqual, "No authentication header provided\n") + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) }) Convey("When the version document has already been published return status forbidden", t, func() { @@ -1547,31 +1531,22 @@ func TestPutVersionReturnsError(t *testing.T) { So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return nil - }, - CheckEditionExistsFunc: func(string, string, string) error { - return nil - }, GetVersionFunc: func(string, string, string, string) (*models.Version, error) { return &models.Version{ State: models.PublishedState, }, nil }, - UpdateVersionFunc: func(string, *models.Version) error { - return nil + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, } api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock) api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusForbidden) - So(w.Body.String(), ShouldEqual, "unable to update document, already published\n") - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) + So(w.Body.String(), ShouldEqual, "unable to update version as it has been published\n") So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 0) - So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) }) Convey("When the request body is invalid return status bad request", t, func() { @@ -1588,15 +1563,15 @@ func TestPutVersionReturnsError(t *testing.T) { So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - CheckDatasetExistsFunc: func(string, string) error { - return nil + GetVersionFunc: func(string, string, string, string) (*models.Version, error) { + return &models.Version{State: "associated"}, nil + }, + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil }, CheckEditionExistsFunc: func(string, string, string) error { return nil }, - GetVersionFunc: func(string, string, string, string) (*models.Version, error) { - return &models.Version{}, nil - }, UpdateVersionFunc: func(string, *models.Version) error { return nil }, @@ -1606,9 +1581,9 @@ func TestPutVersionReturnsError(t *testing.T) { api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) So(w.Body.String(), ShouldEqual, "Missing collection_id for association between version and a collection\n") - So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) }) @@ -1698,11 +1673,14 @@ func TestGetDimensionsReturnsErrors(t *testing.T) { func TestGetDimensionOptionsReturnsOk(t *testing.T) { t.Parallel() - Convey("", t, func() { + Convey("When a valid dimension is provided then a list of options can be returned successfully", t, func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions/age/options", nil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - GetDimensionOptionsFunc: func(datasetID, editionID, versionID, dimensions string) (*models.DimensionOptionResults, error) { + GetVersionFunc: func(datasetID, edition, version, state string) (*models.Version, error) { + return &models.Version{}, nil + }, + GetDimensionOptionsFunc: func(version *models.Version, dimensions string) (*models.DimensionOptionResults, error) { return &models.DimensionOptionResults{}, nil }, } @@ -1715,12 +1693,12 @@ func TestGetDimensionOptionsReturnsOk(t *testing.T) { func TestGetDimensionOptionsReturnsErrors(t *testing.T) { t.Parallel() - Convey("", t, func() { + Convey("When the version doesn't exist in a request for dimension options, then return not found", t, func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions/age/options", nil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - GetDimensionOptionsFunc: func(datasetID, editionID, versionID, dimensions string) (*models.DimensionOptionResults, error) { - return nil, errs.ErrDatasetNotFound + GetVersionFunc: func(datasetID, edition, version, state string) (*models.Version, error) { + return nil, errs.ErrVersionNotFound }, } @@ -1729,11 +1707,14 @@ func TestGetDimensionOptionsReturnsErrors(t *testing.T) { So(w.Code, ShouldEqual, http.StatusNotFound) }) - Convey("", t, func() { + Convey("When an internal error causes failure to retrieve dimension options, then return internal server error", t, func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions/age/options", nil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - GetDimensionOptionsFunc: func(datasetID, editionID, versionID, dimensions string) (*models.DimensionOptionResults, error) { + GetVersionFunc: func(datasetID, edition, version, state string) (*models.Version, error) { + return &models.Version{}, nil + }, + GetDimensionOptionsFunc: func(version *models.Version, dimensions string) (*models.DimensionOptionResults, error) { return nil, errInternal }, } diff --git a/apierrors/errors.go b/apierrors/errors.go index 4fd97b8d..a72d43c0 100644 --- a/apierrors/errors.go +++ b/apierrors/errors.go @@ -8,6 +8,7 @@ var ( ErrEditionNotFound = errors.New("Edition not found") ErrVersionNotFound = errors.New("Version not found") ErrDimensionNodeNotFound = errors.New("Dimension node not found") + ErrDimensionNotFound = errors.New("Dimension not found") ErrDimensionsNotFound = errors.New("Dimensions not found") ErrInstanceNotFound = errors.New("Instance not found") ) diff --git a/config/config.go b/config/config.go index 44db8155..260997d8 100644 --- a/config/config.go +++ b/config/config.go @@ -1,6 +1,7 @@ package config import ( + "encoding/json" "time" "github.com/kelseyhightower/envconfig" @@ -9,12 +10,12 @@ import ( // Configuration structure which hold information for configuring the import API type Configuration struct { BindAddr string `envconfig:"BIND_ADDR"` - KafkaAddr []string `envconfig:"KAFKA_ADDR"` + KafkaAddr []string `envconfig:"KAFKA_ADDR" json:"-"` GenerateDownloadsTopic string `envconfig:"GENERATE_DOWNLOADS_TOPIC"` CodeListAPIURL string `envconfig:"CODE_LIST_API_URL"` DatasetAPIURL string `envconfig:"DATASET_API_URL"` WebsiteURL string `envconfig:"WEBSITE_URL"` - SecretKey string `envconfig:"SECRET_KEY"` + SecretKey string `envconfig:"SECRET_KEY" json:"-"` GracefulShutdownTimeout time.Duration `envconfig:"GRACEFUL_SHUTDOWN_TIMEOUT"` HealthCheckTimeout time.Duration `envconfig:"HEALTHCHECK_TIMEOUT"` MongoConfig MongoConfig @@ -22,7 +23,7 @@ type Configuration struct { // MongoConfig contains the config required to connect to MongoDB. type MongoConfig struct { - BindAddr string `envconfig:"MONGODB_BIND_ADDR"` + BindAddr string `envconfig:"MONGODB_BIND_ADDR" json:"-"` Collection string `envconfig:"MONGODB_COLLECTION"` Database string `envconfig:"MONGODB_DATABASE"` } @@ -54,3 +55,10 @@ func Get() (*Configuration, error) { return cfg, envconfig.Process("", cfg) } + +// String is implemented to prevent sensitive fields being logged. +// The config is returned as JSON with sensitive fields omitted. +func (config Configuration) String() string { + json, _ := json.Marshal(config) + return string(json) +} diff --git a/instance/instance.go b/instance/instance.go index fb4e27c5..30e906ae 100644 --- a/instance/instance.go +++ b/instance/instance.go @@ -109,6 +109,81 @@ func (s *Store) Add(w http.ResponseWriter, r *http.Request) { log.Debug("add instance", log.Data{"instance": instance}) } +// UpdateDimension updates label and/or description for a specific dimension within an instance +func (s *Store) UpdateDimension(w http.ResponseWriter, r *http.Request) { + + vars := mux.Vars(r) + id := vars["id"] + dimension := vars["dimension"] + + // Get instance + instance, err := s.GetInstance(id) + if err != nil { + log.ErrorC("Failed to GET instance when attempting to update a dimension of that instance.", err, log.Data{"instance": id}) + handleErrorType(err, w) + return + } + + // Early return if instance is already published + if instance.State == models.PublishedState { + log.Debug("unable to update instance/version, already published", nil) + w.WriteHeader(http.StatusForbidden) + return + } + + // Read and unmarshal request body + bytes, err := ioutil.ReadAll(r.Body) + if err != nil { + log.ErrorC("Error reading response.body.", err, nil) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + var dim *models.CodeList + + err = json.Unmarshal(bytes, &dim) + if err != nil { + log.ErrorC("Failing to model models.Codelist resource based on request", err, log.Data{"instance": id, "dimension": dimension}) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Update instance-dimension + notFound := true + for i := range instance.Dimensions { + + // For the chosen dimension + if instance.Dimensions[i].Name == dimension { + notFound = false + // Assign update info, conditionals to allow updating of both or either without blanking other + if dim.Label != "" { + instance.Dimensions[i].Label = dim.Label + } + if dim.Description != "" { + instance.Dimensions[i].Description = dim.Description + } + break + } + + } + + if notFound { + log.ErrorC("dimension not found", errs.ErrDimensionNotFound, log.Data{"instance": id, "dimension": dimension}) + handleErrorType(errs.ErrDimensionNotFound, w) + return + } + + // Update instance + if err = s.UpdateInstance(id, instance); err != nil { + log.ErrorC("Failed to update instance with new dimension label/description.", err, log.Data{"instance": id, "dimension": dimension}) + handleErrorType(err, w) + return + } + + log.Debug("updated dimension", log.Data{"instance": id, "dimension": dimension}) + +} + //Update a specific instance func (s *Store) Update(w http.ResponseWriter, r *http.Request) { vars := mux.Vars(r) @@ -134,35 +209,37 @@ func (s *Store) Update(w http.ResponseWriter, r *http.Request) { instance.Links = updateLinks(instance, currentInstance) logData := log.Data{"instance_id": id, "current_state": currentInstance.State, "requested_state": instance.State} - switch instance.State { - case models.CompletedState: - if err = validateInstanceUpdate(models.SubmittedState, currentInstance, instance); err != nil { - log.Error(err, logData) - http.Error(w, err.Error(), http.StatusForbidden) - return - } - case models.EditionConfirmedState: - if err = validateInstanceUpdate(models.CompletedState, currentInstance, instance); err != nil { - log.Error(err, logData) - http.Error(w, err.Error(), http.StatusForbidden) - return - } - case models.AssociatedState: - if err = validateInstanceUpdate(models.EditionConfirmedState, currentInstance, instance); err != nil { - log.Error(err, logData) - http.Error(w, err.Error(), http.StatusForbidden) - return - } + if instance.State != currentInstance.State { + switch instance.State { + case models.CompletedState: + if err = validateInstanceUpdate(models.SubmittedState, currentInstance, instance); err != nil { + log.Error(err, logData) + http.Error(w, err.Error(), http.StatusForbidden) + return + } + case models.EditionConfirmedState: + if err = validateInstanceUpdate(models.CompletedState, currentInstance, instance); err != nil { + log.Error(err, logData) + http.Error(w, err.Error(), http.StatusForbidden) + return + } + case models.AssociatedState: + if err = validateInstanceUpdate(models.EditionConfirmedState, currentInstance, instance); err != nil { + log.Error(err, logData) + http.Error(w, err.Error(), http.StatusForbidden) + return + } - // TODO Update dataset.next state to associated and add collection id - case models.PublishedState: - if err = validateInstanceUpdate(models.AssociatedState, currentInstance, instance); err != nil { - log.Error(err, logData) - http.Error(w, err.Error(), http.StatusForbidden) - return - } + // TODO Update dataset.next state to associated and add collection id + case models.PublishedState: + if err = validateInstanceUpdate(models.AssociatedState, currentInstance, instance); err != nil { + log.Error(err, logData) + http.Error(w, err.Error(), http.StatusForbidden) + return + } - // TODO Update both edition and dataset states to published + // TODO Update both edition and dataset states to published + } } if instance.State == models.EditionConfirmedState { @@ -286,16 +363,16 @@ func (s *Store) getEdition(datasetID, edition, instanceID string) (*models.Editi } func validateInstanceUpdate(expectedState string, currentInstance, instance *models.Instance) error { - if currentInstance.State != expectedState { - err := fmt.Errorf("Unable to update resource, expected resource to have a state of %s", expectedState) - return err - } - if instance.State == models.EditionConfirmedState && currentInstance.Edition == "" && instance.Edition == "" { - err := errors.New("Unable to update resource, missing a value for the edition") - return err + var err error + if currentInstance.State == models.PublishedState { + err = fmt.Errorf("Unable to update resource state, as the version has been published") + } else if currentInstance.State != expectedState { + err = fmt.Errorf("Unable to update resource, expected resource to have a state of %s", expectedState) + } else if instance.State == models.EditionConfirmedState && currentInstance.Edition == "" && instance.Edition == "" { + err = errors.New("Unable to update resource, missing a value for the edition") } - return nil + return err } func (s *Store) defineInstanceLinks(instance *models.Instance, editionDoc *models.Edition) *models.InstanceLinks { @@ -378,7 +455,7 @@ func (s *Store) UpdateImportTask(w http.ResponseWriter, r *http.Request) { validationErrs = append(validationErrs, fmt.Errorf("bad request - invalid task state value for import observations: %v", tasks.ImportObservations.State)) } else if err := s.UpdateImportObservationsTaskState(id, tasks.ImportObservations.State); err != nil { log.Error(err, nil) - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, "Failed to update import observations task state", http.StatusInternalServerError) return } } @@ -391,7 +468,21 @@ func (s *Store) UpdateImportTask(w http.ResponseWriter, r *http.Request) { validationErrs = append(validationErrs, fmt.Errorf("bad request - invalid task state value: %v", task.State)) } else if err := s.UpdateBuildHierarchyTaskState(id, task.DimensionName, task.State); err != nil { log.Error(err, nil) - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, "Failed to update build hierarchy task state", http.StatusInternalServerError) + return + } + } + } + } + + if tasks.BuildSearchIndexTasks != nil { + for _, task := range tasks.BuildSearchIndexTasks { + if task.State != "" { + if task.State != models.CompletedState { + validationErrs = append(validationErrs, fmt.Errorf("bad request - invalid task state value: %v", task.State)) + } else if err := s.UpdateBuildSearchTaskState(id, task.DimensionName, task.State); err != nil { + log.Error(err, nil) + http.Error(w, "Failed to update build search index task state", http.StatusInternalServerError) return } } @@ -469,7 +560,7 @@ func unmarshalInstance(reader io.Reader, post bool) (*models.Instance, error) { func handleErrorType(err error, w http.ResponseWriter) { status := http.StatusInternalServerError - if err == errs.ErrDatasetNotFound || err == errs.ErrEditionNotFound || err == errs.ErrVersionNotFound || err == errs.ErrDimensionNodeNotFound || err == errs.ErrInstanceNotFound { + if err == errs.ErrDatasetNotFound || err == errs.ErrEditionNotFound || err == errs.ErrVersionNotFound || err == errs.ErrDimensionNotFound || err == errs.ErrDimensionNodeNotFound || err == errs.ErrInstanceNotFound { status = http.StatusNotFound } @@ -488,3 +579,32 @@ func writeBody(w http.ResponseWriter, bytes []byte) { http.Error(w, err.Error(), http.StatusInternalServerError) } } + +// PublishCheck Checks if an instance has been published +type PublishCheck struct { + Datastore store.Storer +} + +// Check wraps a HTTP handle. Checks that the state is not published +func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request)) http.HandlerFunc { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + vars := mux.Vars(r) + id := vars["id"] + instance, err := d.Datastore.GetInstance(id) + if err != nil { + log.Error(err, nil) + handleErrorType(err, w) + return + } + + if instance.State == models.PublishedState { + err = errors.New("unable to update instance as it has been published") + log.Error(err, log.Data{"instance": instance}) + http.Error(w, err.Error(), http.StatusForbidden) + return + } + + handle(w, r) + }) +} diff --git a/instance/instance_external_test.go b/instance/instance_external_test.go index 3ccbf40d..4ec7ea5f 100644 --- a/instance/instance_external_test.go +++ b/instance/instance_external_test.go @@ -17,7 +17,7 @@ import ( ) const secretKey = "coffee" -const host = "http://locahost:8080" +const host = "http://localhost:8080" var internalError = errors.New("internal error") @@ -374,7 +374,7 @@ func TestUpdatePublishedInstanceToCompletedReturnsForbidden(t *testing.T) { return currentInstanceTestData, nil }, UpdateInstanceFunc: func(id string, i *models.Instance) error { - return internalError + return nil }, } @@ -387,7 +387,7 @@ func TestUpdatePublishedInstanceToCompletedReturnsForbidden(t *testing.T) { }) } -func TestUpdateCompletedInstanceToCompletedReturnsForbidden(t *testing.T) { +func TestUpdateEditionConfirmedInstanceToCompletedReturnsForbidden(t *testing.T) { t.Parallel() Convey("update to an instance returns an internal error", t, func() { body := strings.NewReader(`{"state":"completed"}`) @@ -401,7 +401,7 @@ func TestUpdateCompletedInstanceToCompletedReturnsForbidden(t *testing.T) { ID: "4567", }, }, - State: models.CompletedState, + State: models.EditionConfirmedState, } mockedDataStore := &storetest.StorerMock{ @@ -409,7 +409,7 @@ func TestUpdateCompletedInstanceToCompletedReturnsForbidden(t *testing.T) { return currentInstanceTestData, nil }, UpdateInstanceFunc: func(id string, i *models.Instance) error { - return internalError + return nil }, } @@ -599,3 +599,204 @@ func TestStore_UpdateImportTask_ReturnsInternalError(t *testing.T) { So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 1) }) } + +func TestUpdateInstanceReturnsErrorWhenStateIsPublished(t *testing.T) { + t.Parallel() + Convey("when an instance has a state of published, then put request to change to it to completed ", t, func() { + body := strings.NewReader(`{"state":"completed"}`) + r := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.PublishedState}, nil + }, + UpdateInstanceFunc: func(id string, i *models.Instance) error { + return nil + }, + } + + instance := &instance.Store{Host: host, Storer: mockedDataStore} + instance.Update(w, r) + + So(w.Code, ShouldEqual, http.StatusForbidden) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + }) +} + +func TestUpdateDimensionReturnsNotFound(t *testing.T) { + t.Parallel() + Convey("When update dimension return status not found", t, func() { + r := createRequestWithToken("PUT", "http://localhost:22000/instances/dimensions/age", nil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return nil, errs.ErrInstanceNotFound + }, + } + + instance := &instance.Store{Host: host, Storer: mockedDataStore} + instance.UpdateDimension(w, r) + + So(w.Code, ShouldEqual, http.StatusNotFound) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + }) +} + +func TestUpdateDimensionReturnsForbidden(t *testing.T) { + t.Parallel() + Convey("When update dimension returns forbidden (for already published) ", t, func() { + r := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", nil) + w := httptest.NewRecorder() + + currentInstanceTestData := &models.Instance{ + State: models.PublishedState, + } + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return currentInstanceTestData, nil + }, + } + + instance := &instance.Store{Host: host, Storer: mockedDataStore} + instance.UpdateDimension(w, r) + + So(w.Code, ShouldEqual, http.StatusForbidden) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + }) +} + +func TestUpdateDimensionReturnsBadRequest(t *testing.T) { + t.Parallel() + Convey("When update dimension returns bad request", t, func() { + body := strings.NewReader("{") + r := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", body) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{}, nil + }, + } + + instance := &instance.Store{Host: host, Storer: mockedDataStore} + instance.UpdateDimension(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + }) +} + +func TestUpdateDimensionReturnsNotFoundWithWrongName(t *testing.T) { + t.Parallel() + Convey("When update dimension fails to update an instance", t, func() { + body := strings.NewReader(`{"label":"notages"}`) + r := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/notage", body) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState, + InstanceID: "123", + Dimensions: []models.CodeList{{Name: "age", ID: "age"}}}, nil + }, + UpdateInstanceFunc: func(id string, i *models.Instance) error { + return nil + }, + } + + instance := &instance.Store{Host: host, Storer: mockedDataStore} + + router := mux.NewRouter() + router.HandleFunc("/instances/{id}/dimensions/{dimension}", instance.UpdateDimension).Methods("PUT") + + router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusNotFound) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + }) +} + +func TestUpdateDimensionReturnsOk(t *testing.T) { + t.Parallel() + Convey("When update dimension fails to update an instance", t, func() { + body := strings.NewReader(`{"label":"ages"}`) + r := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", body) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState, + InstanceID: "123", + Dimensions: []models.CodeList{{Name: "age", ID: "age"}}}, nil + }, + UpdateInstanceFunc: func(id string, i *models.Instance) error { + return nil + }, + } + + instance := &instance.Store{Host: host, Storer: mockedDataStore} + router := mux.NewRouter() + router.HandleFunc("/instances/{id}/dimensions/{dimension}", instance.UpdateDimension).Methods("PUT") + + router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 1) + }) +} + +func TestStore_UpdateImportTask_UpdateBuildSearchIndexTask_InvalidState(t *testing.T) { + + t.Parallel() + Convey("update to an import task with an invalid state returns http 400 response", t, func() { + body := strings.NewReader(`{"build_search_indexes":[{"state":"notvalid"}]}`) + r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } + + instance := &instance.Store{Host: host, Storer: mockedDataStore} + + instance.UpdateImportTask(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + }) +} + +func TestStore_UpdateImportTask_UpdateBuildSearchIndexTask(t *testing.T) { + + t.Parallel() + Convey("update to an import task returns http 200 response if no errors occur", t, func() { + body := strings.NewReader(`{"build_search_indexes":[{"state":"completed"}]}`) + r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } + + instance := &instance.Store{Host: host, Storer: mockedDataStore} + + instance.UpdateImportTask(w, r) + + So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 1) + }) +} diff --git a/instance/instance_internal_test.go b/instance/instance_internal_test.go index dc6f9c72..71b184ad 100644 --- a/instance/instance_internal_test.go +++ b/instance/instance_internal_test.go @@ -115,7 +115,7 @@ func TestUnmarshalImportTaskWithInvalidJson(t *testing.T) { }) } -func TestUnmarshalImportTask(t *testing.T) { +func TestUnmarshalImportTask_ImportObservations(t *testing.T) { Convey("Create an import observation task with valid json", t, func() { task, err := unmarshalImportTasks(strings.NewReader(`{"import_observations":{"state":"completed"}}`)) So(err, ShouldBeNil) @@ -124,3 +124,23 @@ func TestUnmarshalImportTask(t *testing.T) { So(task.ImportObservations.State, ShouldEqual, "completed") }) } + +func TestUnmarshalImportTask_BuildHierarchies(t *testing.T) { + Convey("Create an import observation task with valid json", t, func() { + task, err := unmarshalImportTasks(strings.NewReader(`{"build_hierarchies":[{"state":"completed"}]}`)) + So(err, ShouldBeNil) + So(task, ShouldNotBeNil) + So(task.BuildHierarchyTasks, ShouldNotBeNil) + So(task.BuildHierarchyTasks[0].State, ShouldEqual, "completed") + }) +} + +func TestUnmarshalImportTask_BuildSearch(t *testing.T) { + Convey("Create an import observation task with valid json", t, func() { + task, err := unmarshalImportTasks(strings.NewReader(`{"build_search_indexes":[{"state":"completed"}]}`)) + So(err, ShouldBeNil) + So(task, ShouldNotBeNil) + So(task.BuildSearchIndexTasks, ShouldNotBeNil) + So(task.BuildSearchIndexTasks[0].State, ShouldEqual, "completed") + }) +} diff --git a/main.go b/main.go index fe3f3115..a588fb16 100644 --- a/main.go +++ b/main.go @@ -34,9 +34,7 @@ func main() { os.Exit(1) } - sanitized := *cfg - sanitized.SecretKey = "" - log.Info("config on startup", log.Data{"config": sanitized}) + log.Info("config on startup", log.Data{"config": cfg}) generateDownloadsProducer, err := kafka.NewProducer(cfg.KafkaAddr, cfg.GenerateDownloadsTopic, 0) if err != nil { diff --git a/models/dimension.go b/models/dimension.go index 5f92d7ed..92c236b9 100644 --- a/models/dimension.go +++ b/models/dimension.go @@ -19,6 +19,7 @@ type Dimension struct { Links DimensionLink `bson:"links,omitempty" json:"links,omitempty"` Name string `bson:"name,omitempty" json:"dimension,omitempty"` LastUpdated time.Time `bson:"last_updated,omitempty" json:"-"` + Label string `bson:"label,omitempty" json:"label,omitempty"` } // DimensionLink contains all links needed for a dimension diff --git a/models/instance.go b/models/instance.go index 7b239b05..0500e9d7 100644 --- a/models/instance.go +++ b/models/instance.go @@ -27,16 +27,17 @@ type Instance struct { ImportTasks *InstanceImportTasks `bson:"import_tasks,omitempty" json:"import_tasks"` } -// InstanceImportTasks +// InstanceImportTasks represents all of the tasks required to complete an import job. type InstanceImportTasks struct { - ImportObservations *ImportObservationsTask `bson:"import_observations,omitempty" json:"import_observations"` - BuildHierarchyTasks []*BuildHierarchyTask `bson:"build_hierarchies,omitempty" json:"build_hierarchies"` + ImportObservations *ImportObservationsTask `bson:"import_observations,omitempty" json:"import_observations"` + BuildHierarchyTasks []*BuildHierarchyTask `bson:"build_hierarchies,omitempty" json:"build_hierarchies"` + BuildSearchIndexTasks []*BuildSearchIndexTask `bson:"build_search_indexes,omitempty" json:"build_search_indexes"` } // ImportObservationsTask represents the task of importing instance observation data into the database. type ImportObservationsTask struct { - State string `bson:"state,omitempty" json:"state,omitempty"` - InsertedObservations int `bson:"total_inserted_observations" json:"total_inserted_observations,omitempty"` + State string `bson:"state,omitempty" json:"state,omitempty"` + InsertedObservations int64 `bson:"total_inserted_observations" json:"total_inserted_observations"` } // BuildHierarchyTask represents a task of importing a single hierarchy. @@ -46,12 +47,19 @@ type BuildHierarchyTask struct { CodeListID string `bson:"code_list_id,omitempty" json:"code_list_id,omitempty"` } +// BuildSearchIndexTask represents a task of importing a single search index into search. +type BuildSearchIndexTask struct { + State string `bson:"state,omitempty" json:"state,omitempty"` + DimensionName string `bson:"dimension_name,omitempty" json:"dimension_name,omitempty"` +} + // CodeList for a dimension within an instance type CodeList struct { Description string `json:"description"` HRef string `json:"href"` ID string `json:"id"` Name string `json:"name"` + Label string `json:"label"` } // InstanceLinks holds all links for an instance diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index bc1aac51..a4fd57b1 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -282,11 +282,11 @@ func buildVersionQuery(id, editionID, state string, versionID int) bson.M { } // UpdateDataset updates an existing dataset document -func (m *Mongo) UpdateDataset(id string, dataset *models.Dataset) (err error) { +func (m *Mongo) UpdateDataset(id string, dataset *models.Dataset, currentState string) (err error) { s := m.Session.Copy() defer s.Close() - updates := createDatasetUpdateQuery(id, dataset) + updates := createDatasetUpdateQuery(id, dataset, currentState) update := bson.M{"$set": updates, "$setOnInsert": bson.M{"next.last_updated": time.Now()}} if err = s.DB(m.Database).C("datasets").UpdateId(id, update); err != nil { if err == mgo.ErrNotFound { @@ -298,7 +298,7 @@ func (m *Mongo) UpdateDataset(id string, dataset *models.Dataset) (err error) { return nil } -func createDatasetUpdateQuery(id string, dataset *models.Dataset) bson.M { +func createDatasetUpdateQuery(id string, dataset *models.Dataset, currentState string) bson.M { updates := make(bson.M, 0) log.Debug("building update query for dataset resource", log.Data{"dataset_id": id, "dataset": dataset, "updates": updates}) @@ -385,6 +385,10 @@ func createDatasetUpdateQuery(id string, dataset *models.Dataset) bson.M { if dataset.State != "" { updates["next.state"] = dataset.State + } else { + if currentState == models.PublishedState { + updates["next.state"] = models.CreatedState + } } if dataset.Theme != "" { diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index 422553be..8e33e82d 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -226,7 +226,7 @@ func TestDatasetUpdateQuery(t *testing.T) { URI: "http://ons.gov.uk/dataset/123/landing-page", } - selector := createDatasetUpdateQuery("123", dataset) + selector := createDatasetUpdateQuery("123", dataset, models.CreatedState) So(selector, ShouldNotBeNil) So(selector, ShouldResemble, expectedUpdate) }) @@ -239,7 +239,7 @@ func TestDatasetUpdateQuery(t *testing.T) { expectedUpdate := bson.M{ "next.national_statistic": &nationalStatistic, } - selector := createDatasetUpdateQuery("123", dataset) + selector := createDatasetUpdateQuery("123", dataset, models.CreatedState) So(selector, ShouldNotBeNil) So(selector, ShouldResemble, expectedUpdate) }) @@ -247,7 +247,7 @@ func TestDatasetUpdateQuery(t *testing.T) { Convey("When national statistic is not set", t, func() { dataset := &models.Dataset{} - selector := createDatasetUpdateQuery("123", dataset) + selector := createDatasetUpdateQuery("123", dataset, models.CreatedState) So(selector, ShouldNotBeNil) So(selector, ShouldResemble, bson.M{}) }) diff --git a/mongo/dimension_store.go b/mongo/dimension_store.go index 514d2a56..20266abf 100644 --- a/mongo/dimension_store.go +++ b/mongo/dimension_store.go @@ -89,19 +89,13 @@ func (m *Mongo) GetDimensions(datasetID, versionID string) ([]bson.M, error) { } // GetDimensionOptions returns all dimension options for a dimensions within a dataset. -func (m *Mongo) GetDimensionOptions(datasetID, editionID, versionID, dimension string) (*models.DimensionOptionResults, error) { +func (m *Mongo) GetDimensionOptions(version *models.Version, dimension string) (*models.DimensionOptionResults, error) { s := m.Session.Copy() defer s.Close() - version, err := m.GetVersion(datasetID, editionID, versionID, models.PublishedState) - if err != nil { - return nil, err - } - var values []models.PublicDimensionOption iter := s.DB(m.Database).C(dimensionOptions).Find(bson.M{"instance_id": version.ID, "name": dimension}).Iter() - err = iter.All(&values) - if err != nil { + if err := iter.All(&values); err != nil { return nil, err } diff --git a/mongo/instance_store.go b/mongo/instance_store.go index 3d47d163..4ddbad3e 100644 --- a/mongo/instance_store.go +++ b/mongo/instance_store.go @@ -128,7 +128,9 @@ func (m *Mongo) UpdateObservationInserted(id string, observationInserted int64) err := s.DB(m.Database).C(instanceCollection).Update(bson.M{"id": id}, bson.M{ "$inc": bson.M{"import_tasks.import_observations.total_inserted_observations": observationInserted}, - "$set": bson.M{"last_updated": time.Now().UTC()}}) + "$set": bson.M{"last_updated": time.Now().UTC()}, + }, + ) if err == mgo.ErrNotFound { return errs.ErrInstanceNotFound @@ -150,7 +152,8 @@ func (m *Mongo) UpdateImportObservationsTaskState(id string, state string) error bson.M{ "$set": bson.M{"import_tasks.import_observations.state": state}, "$currentDate": bson.M{"last_updated": true}, - }) + }, + ) if err == mgo.ErrNotFound { return errs.ErrInstanceNotFound @@ -181,3 +184,22 @@ func (m *Mongo) UpdateBuildHierarchyTaskState(id, dimension, state string) (err err = s.DB(m.Database).C(instanceCollection).Update(selector, update) return } + +// UpdateBuildSearchTaskState updates the state of a build search task. +func (m *Mongo) UpdateBuildSearchTaskState(id, dimension, state string) (err error) { + s := m.Session.Copy() + defer s.Close() + + selector := bson.M{ + "id": id, + "import_tasks.build_search_indexes.dimension_name": dimension, + } + + update := bson.M{ + "$set": bson.M{"import_tasks.build_search_indexes.$.state": state}, + "$currentDate": bson.M{"last_updated": true}, + } + + err = s.DB(m.Database).C(instanceCollection).Update(selector, update) + return +} diff --git a/store/datastore.go b/store/datastore.go index 3c335797..4b800541 100644 --- a/store/datastore.go +++ b/store/datastore.go @@ -26,7 +26,7 @@ type Storer interface { GetDatasets() ([]models.DatasetUpdate, error) GetDimensionNodesFromInstance(ID string) (*models.DimensionNodeResults, error) GetDimensions(datasetID, versionID string) ([]bson.M, error) - GetDimensionOptions(datasetID, editionID, versionID, dimension string) (*models.DimensionOptionResults, error) + GetDimensionOptions(version *models.Version, dimension string) (*models.DimensionOptionResults, error) GetEdition(ID, editionID, state string) (*models.Edition, error) GetEditions(ID, state string) (*models.EditionResults, error) GetInstances(filters []string) (*models.InstanceResults, error) @@ -35,7 +35,7 @@ type Storer interface { GetUniqueDimensionValues(ID, dimension string) (*models.DimensionValues, error) GetVersion(datasetID, editionID, version, state string) (*models.Version, error) GetVersions(datasetID, editionID, state string) (*models.VersionResults, error) - UpdateDataset(ID string, dataset *models.Dataset) error + UpdateDataset(ID string, dataset *models.Dataset, currentState string) error UpdateDatasetWithAssociation(ID, state string, version *models.Version) error UpdateDimensionNodeID(dimension *models.DimensionOption) error UpdateEdition(datasetID, edition string, latestVersion *models.Version) error @@ -43,6 +43,7 @@ type Storer interface { UpdateObservationInserted(ID string, observationInserted int64) error UpdateImportObservationsTaskState(id, state string) error UpdateBuildHierarchyTaskState(id, dimension, state string) error + UpdateBuildSearchTaskState(id, dimension, state string) error UpdateVersion(ID string, version *models.Version) error UpsertContact(ID string, update interface{}) error UpsertDataset(ID string, datasetDoc *models.DatasetUpdate) error diff --git a/store/datastoretest/datastore.go b/store/datastoretest/datastore.go index 3ccd3139..ce639670 100755 --- a/store/datastoretest/datastore.go +++ b/store/datastoretest/datastore.go @@ -32,6 +32,7 @@ var ( lockStorerMockGetVersions sync.RWMutex lockStorerMockPing sync.RWMutex lockStorerMockUpdateBuildHierarchyTaskState sync.RWMutex + lockStorerMockUpdateBuildSearchTaskState sync.RWMutex lockStorerMockUpdateDataset sync.RWMutex lockStorerMockUpdateDatasetWithAssociation sync.RWMutex lockStorerMockUpdateDimensionNodeID sync.RWMutex @@ -76,7 +77,7 @@ var ( // GetDimensionNodesFromInstanceFunc: func(ID string) (*models.DimensionNodeResults, error) { // panic("TODO: mock out the GetDimensionNodesFromInstance method") // }, -// GetDimensionOptionsFunc: func(datasetID string, editionID string, versionID string, dimension string) (*models.DimensionOptionResults, error) { +// GetDimensionOptionsFunc: func(version *models.Version, dimension string) (*models.DimensionOptionResults, error) { // panic("TODO: mock out the GetDimensionOptions method") // }, // GetDimensionsFunc: func(datasetID string, versionID string) ([]bson.M, error) { @@ -112,7 +113,10 @@ var ( // UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { // panic("TODO: mock out the UpdateBuildHierarchyTaskState method") // }, -// UpdateDatasetFunc: func(ID string, dataset *models.Dataset) error { +// UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { +// panic("TODO: mock out the UpdateBuildSearchTaskState method") +// }, +// UpdateDatasetFunc: func(ID string, dataset *models.Dataset, currentState string) error { // panic("TODO: mock out the UpdateDataset method") // }, // UpdateDatasetWithAssociationFunc: func(ID string, state string, version *models.Version) error { @@ -180,7 +184,7 @@ type StorerMock struct { GetDimensionNodesFromInstanceFunc func(ID string) (*models.DimensionNodeResults, error) // GetDimensionOptionsFunc mocks the GetDimensionOptions method. - GetDimensionOptionsFunc func(datasetID string, editionID string, versionID string, dimension string) (*models.DimensionOptionResults, error) + GetDimensionOptionsFunc func(version *models.Version, dimension string) (*models.DimensionOptionResults, error) // GetDimensionsFunc mocks the GetDimensions method. GetDimensionsFunc func(datasetID string, versionID string) ([]bson.M, error) @@ -215,8 +219,11 @@ type StorerMock struct { // UpdateBuildHierarchyTaskStateFunc mocks the UpdateBuildHierarchyTaskState method. UpdateBuildHierarchyTaskStateFunc func(id string, dimension string, state string) error + // UpdateBuildSearchTaskStateFunc mocks the UpdateBuildSearchTaskState method. + UpdateBuildSearchTaskStateFunc func(id string, dimension string, state string) error + // UpdateDatasetFunc mocks the UpdateDataset method. - UpdateDatasetFunc func(ID string, dataset *models.Dataset) error + UpdateDatasetFunc func(ID string, dataset *models.Dataset, currentState string) error // UpdateDatasetWithAssociationFunc mocks the UpdateDatasetWithAssociation method. UpdateDatasetWithAssociationFunc func(ID string, state string, version *models.Version) error @@ -301,12 +308,8 @@ type StorerMock struct { } // GetDimensionOptions holds details about calls to the GetDimensionOptions method. GetDimensionOptions []struct { - // DatasetID is the datasetID argument value. - DatasetID string - // EditionID is the editionID argument value. - EditionID string - // VersionID is the versionID argument value. - VersionID string + // Version is the version argument value. + Version *models.Version // Dimension is the dimension argument value. Dimension string } @@ -384,8 +387,17 @@ type StorerMock struct { } // UpdateBuildHierarchyTaskState holds details about calls to the UpdateBuildHierarchyTaskState method. UpdateBuildHierarchyTaskState []struct { - // Id is the id argument value. - Id string + // ID is the id argument value. + ID string + // Dimension is the dimension argument value. + Dimension string + // State is the state argument value. + State string + } + // UpdateBuildSearchTaskState holds details about calls to the UpdateBuildSearchTaskState method. + UpdateBuildSearchTaskState []struct { + // ID is the id argument value. + ID string // Dimension is the dimension argument value. Dimension string // State is the state argument value. @@ -397,6 +409,8 @@ type StorerMock struct { ID string // Dataset is the dataset argument value. Dataset *models.Dataset + // CurrentState is the currentState argument value. + CurrentState string } // UpdateDatasetWithAssociation holds details about calls to the UpdateDatasetWithAssociation method. UpdateDatasetWithAssociation []struct { @@ -423,8 +437,8 @@ type StorerMock struct { } // UpdateImportObservationsTaskState holds details about calls to the UpdateImportObservationsTaskState method. UpdateImportObservationsTaskState []struct { - // Id is the id argument value. - Id string + // ID is the id argument value. + ID string // State is the state argument value. State string } @@ -742,40 +756,32 @@ func (mock *StorerMock) GetDimensionNodesFromInstanceCalls() []struct { } // GetDimensionOptions calls GetDimensionOptionsFunc. -func (mock *StorerMock) GetDimensionOptions(datasetID string, editionID string, versionID string, dimension string) (*models.DimensionOptionResults, error) { +func (mock *StorerMock) GetDimensionOptions(version *models.Version, dimension string) (*models.DimensionOptionResults, error) { if mock.GetDimensionOptionsFunc == nil { panic("moq: StorerMock.GetDimensionOptionsFunc is nil but Storer.GetDimensionOptions was just called") } callInfo := struct { - DatasetID string - EditionID string - VersionID string + Version *models.Version Dimension string }{ - DatasetID: datasetID, - EditionID: editionID, - VersionID: versionID, + Version: version, Dimension: dimension, } lockStorerMockGetDimensionOptions.Lock() mock.calls.GetDimensionOptions = append(mock.calls.GetDimensionOptions, callInfo) lockStorerMockGetDimensionOptions.Unlock() - return mock.GetDimensionOptionsFunc(datasetID, editionID, versionID, dimension) + return mock.GetDimensionOptionsFunc(version, dimension) } // GetDimensionOptionsCalls gets all the calls that were made to GetDimensionOptions. // Check the length with: // len(mockedStorer.GetDimensionOptionsCalls()) func (mock *StorerMock) GetDimensionOptionsCalls() []struct { - DatasetID string - EditionID string - VersionID string + Version *models.Version Dimension string } { var calls []struct { - DatasetID string - EditionID string - VersionID string + Version *models.Version Dimension string } lockStorerMockGetDimensionOptions.RLock() @@ -1144,11 +1150,11 @@ func (mock *StorerMock) UpdateBuildHierarchyTaskState(id string, dimension strin panic("moq: StorerMock.UpdateBuildHierarchyTaskStateFunc is nil but Storer.UpdateBuildHierarchyTaskState was just called") } callInfo := struct { - Id string + ID string Dimension string State string }{ - Id: id, + ID: id, Dimension: dimension, State: state, } @@ -1162,12 +1168,12 @@ func (mock *StorerMock) UpdateBuildHierarchyTaskState(id string, dimension strin // Check the length with: // len(mockedStorer.UpdateBuildHierarchyTaskStateCalls()) func (mock *StorerMock) UpdateBuildHierarchyTaskStateCalls() []struct { - Id string + ID string Dimension string State string } { var calls []struct { - Id string + ID string Dimension string State string } @@ -1177,34 +1183,77 @@ func (mock *StorerMock) UpdateBuildHierarchyTaskStateCalls() []struct { return calls } +// UpdateBuildSearchTaskState calls UpdateBuildSearchTaskStateFunc. +func (mock *StorerMock) UpdateBuildSearchTaskState(id string, dimension string, state string) error { + if mock.UpdateBuildSearchTaskStateFunc == nil { + panic("moq: StorerMock.UpdateBuildSearchTaskStateFunc is nil but Storer.UpdateBuildSearchTaskState was just called") + } + callInfo := struct { + ID string + Dimension string + State string + }{ + ID: id, + Dimension: dimension, + State: state, + } + lockStorerMockUpdateBuildSearchTaskState.Lock() + mock.calls.UpdateBuildSearchTaskState = append(mock.calls.UpdateBuildSearchTaskState, callInfo) + lockStorerMockUpdateBuildSearchTaskState.Unlock() + return mock.UpdateBuildSearchTaskStateFunc(id, dimension, state) +} + +// UpdateBuildSearchTaskStateCalls gets all the calls that were made to UpdateBuildSearchTaskState. +// Check the length with: +// len(mockedStorer.UpdateBuildSearchTaskStateCalls()) +func (mock *StorerMock) UpdateBuildSearchTaskStateCalls() []struct { + ID string + Dimension string + State string +} { + var calls []struct { + ID string + Dimension string + State string + } + lockStorerMockUpdateBuildSearchTaskState.RLock() + calls = mock.calls.UpdateBuildSearchTaskState + lockStorerMockUpdateBuildSearchTaskState.RUnlock() + return calls +} + // UpdateDataset calls UpdateDatasetFunc. -func (mock *StorerMock) UpdateDataset(ID string, dataset *models.Dataset) error { +func (mock *StorerMock) UpdateDataset(ID string, dataset *models.Dataset, currentState string) error { if mock.UpdateDatasetFunc == nil { panic("moq: StorerMock.UpdateDatasetFunc is nil but Storer.UpdateDataset was just called") } callInfo := struct { - ID string - Dataset *models.Dataset + ID string + Dataset *models.Dataset + CurrentState string }{ - ID: ID, - Dataset: dataset, + ID: ID, + Dataset: dataset, + CurrentState: currentState, } lockStorerMockUpdateDataset.Lock() mock.calls.UpdateDataset = append(mock.calls.UpdateDataset, callInfo) lockStorerMockUpdateDataset.Unlock() - return mock.UpdateDatasetFunc(ID, dataset) + return mock.UpdateDatasetFunc(ID, dataset, currentState) } // UpdateDatasetCalls gets all the calls that were made to UpdateDataset. // Check the length with: // len(mockedStorer.UpdateDatasetCalls()) func (mock *StorerMock) UpdateDatasetCalls() []struct { - ID string - Dataset *models.Dataset + ID string + Dataset *models.Dataset + CurrentState string } { var calls []struct { - ID string - Dataset *models.Dataset + ID string + Dataset *models.Dataset + CurrentState string } lockStorerMockUpdateDataset.RLock() calls = mock.calls.UpdateDataset @@ -1327,10 +1376,10 @@ func (mock *StorerMock) UpdateImportObservationsTaskState(id string, state strin panic("moq: StorerMock.UpdateImportObservationsTaskStateFunc is nil but Storer.UpdateImportObservationsTaskState was just called") } callInfo := struct { - Id string + ID string State string }{ - Id: id, + ID: id, State: state, } lockStorerMockUpdateImportObservationsTaskState.Lock() @@ -1343,11 +1392,11 @@ func (mock *StorerMock) UpdateImportObservationsTaskState(id string, state strin // Check the length with: // len(mockedStorer.UpdateImportObservationsTaskStateCalls()) func (mock *StorerMock) UpdateImportObservationsTaskStateCalls() []struct { - Id string + ID string State string } { var calls []struct { - Id string + ID string State string } lockStorerMockUpdateImportObservationsTaskState.RLock() diff --git a/swagger.yaml b/swagger.yaml index 8b5bf1bf..e28e4311 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -591,6 +591,31 @@ paths: $ref: '#/responses/InstanceNotFound' 500: $ref: '#/responses/InternalError' + /instances/{instance_id}/dimensions/{dimension}: + put: + tags: + - "Private user" + summary: "Update dimension description and/or label" + description: "Update the label and/or description of a dimension within an instance, by providing dimension name and properties to over write" + parameters: + - $ref: '#/parameters/instance_id' + - $ref: '#/parameters/dimension' + - $ref: '#/parameters/instance' + security: + - InternalAPIKey: [] + responses: + 200: + description: "The instance has been updated" + 400: + $ref: '#/responses/InvalidRequestError' + 401: + $ref: '#/responses/UnauthorisedError' + 403: + $ref: '#/responses/ForbiddenError' + 404: + $ref: '#/responses/InstanceNotFound' + 500: + $ref: '#/responses/InternalError' /instances/{instance_id}/dimensions/{dimension}/options: get: tags: @@ -1191,6 +1216,17 @@ definitions: code_list_id: type: string description: "The ID of the codelist that this hierarchy represents" + build_search_indexes: + type: array + items: + type: object + properties: + state: + type: string + description: "The state of the import observations task" + dimension_name: + type: string + description: "The name of the dimension the search index represents" Codelist: type: object properties: @@ -1802,4 +1838,4 @@ definitions: type: string href: description: "A URL to the version this resource relates to" - type: string \ No newline at end of file + type: string