diff --git a/api/api.go b/api/api.go index 4807338a..0b086591 100644 --- a/api/api.go +++ b/api/api.go @@ -6,6 +6,7 @@ import ( "encoding/json" "io/ioutil" "net/http" + "strconv" "time" errs "github.com/ONSdigital/dp-dataset-api/apierrors" @@ -17,7 +18,6 @@ import ( "github.com/ONSdigital/dp-dataset-api/url" "github.com/ONSdigital/go-ns/audit" "github.com/ONSdigital/go-ns/common" - "github.com/ONSdigital/go-ns/handlers/requestID" "github.com/ONSdigital/go-ns/healthcheck" "github.com/ONSdigital/go-ns/identity" "github.com/ONSdigital/go-ns/log" @@ -30,39 +30,39 @@ import ( var httpServer *server.Server const ( - datasetDocType = "dataset" - editionDocType = "edition" - versionDocType = "version" - observationDocType = "observation" downloadServiceToken = "X-Download-Service-Token" dimensionDocType = "dimension" dimensionOptionDocType = "dimension-option" // audit actions - getDatasetsAction = "getDatasets" - getDatasetAction = "getDataset" - deleteDatasetAction = "deleteDataset" - addDatasetAction = "addDataset" - putDatasetAction = "putDataset" - getEditionsAction = "getEditions" - getEditionAction = "getEdition" - getVersionsAction = "getVersions" - getVersionAction = "getVersion" - getDimensionsAction = "getDimensions" - getMetadataAction = "getMetadata" - - // audit results - actionAttempted = "attempted" - actionSuccessful = "successful" - actionUnsuccessful = "unsuccessful" + getDatasetsAction = "getDatasets" + getDatasetAction = "getDataset" + putDatasetAction = "putDataset" + getEditionsAction = "getEditions" + getEditionAction = "getEdition" + getVersionsAction = "getVersions" + getVersionAction = "getVersion" + updateVersionAction = "updateVersion" + publishVersionAction = "publishVersion" + associateVersionAction = "associateVersionAction" + deleteDatasetAction = "deleteDataset" + addDatasetAction = "addDataset" + getDimensionsAction = "getDimensions" + getDimensionOptionsAction = "getDimensionOptionsAction" + getMetadataAction = "getMetadata" auditError = "error while attempting to record audit event, failing request" auditActionErr = "failed to audit action" + + hasDownloads = "has_downloads" ) +var trueStringified = strconv.FormatBool(true) + // PublishCheck Checks if an version has been published type PublishCheck struct { Datastore store.Storer + Auditor audit.AuditorService } //API provides an interface for the routes @@ -75,6 +75,7 @@ type DownloadsGenerator interface { Generate(datasetID, instanceID, edition, version string) error } +// Auditor is an alias for the auditor service type Auditor audit.AuditorService // DatasetAPI manages importing filters against a dataset @@ -86,7 +87,7 @@ type DatasetAPI struct { internalToken string downloadServiceToken string EnablePrePublishView bool - router *mux.Router + Router *mux.Router urlBuilder *url.Builder downloadGenerator DownloadsGenerator healthCheckTimeout time.Duration @@ -94,14 +95,10 @@ type DatasetAPI struct { auditor Auditor } -func setJSONContentType(w http.ResponseWriter) { - w.Header().Set("Content-Type", "application/json") -} - // CreateDatasetAPI manages all the routes configured to API func CreateDatasetAPI(cfg config.Configuration, dataStore store.DataStore, urlBuilder *url.Builder, errorChan chan error, downloadsGenerator DownloadsGenerator, auditor Auditor, observationStore ObservationStore) { router := mux.NewRouter() - routes(cfg, router, dataStore, urlBuilder, downloadsGenerator, auditor, observationStore) + Routes(cfg, router, dataStore, urlBuilder, downloadsGenerator, auditor, observationStore) healthcheckHandler := healthcheck.NewMiddleware(healthcheck.Do) middleware := alice.New(healthcheckHandler) @@ -125,7 +122,8 @@ func CreateDatasetAPI(cfg config.Configuration, dataStore store.DataStore, urlBu }() } -func routes(cfg config.Configuration, router *mux.Router, dataStore store.DataStore, urlBuilder *url.Builder, downloadGenerator DownloadsGenerator, auditor Auditor, observationStore ObservationStore) *DatasetAPI { +// Routes represents a list of endpoints that exist with this api +func Routes(cfg config.Configuration, router *mux.Router, dataStore store.DataStore, urlBuilder *url.Builder, downloadGenerator DownloadsGenerator, auditor Auditor, observationStore ObservationStore) *DatasetAPI { api := DatasetAPI{ dataStore: dataStore, @@ -135,123 +133,81 @@ func routes(cfg config.Configuration, router *mux.Router, dataStore store.DataSt serviceAuthToken: cfg.ServiceAuthToken, downloadServiceToken: cfg.DownloadServiceSecretKey, EnablePrePublishView: cfg.EnablePrivateEnpoints, - router: router, + Router: router, urlBuilder: urlBuilder, downloadGenerator: downloadGenerator, healthCheckTimeout: cfg.HealthCheckTimeout, auditor: auditor, } - api.router.HandleFunc("/datasets", api.getDatasets).Methods("GET") - api.router.HandleFunc("/datasets/{id}", api.getDataset).Methods("GET") - api.router.HandleFunc("/datasets/{id}/editions", api.getEditions).Methods("GET") - api.router.HandleFunc("/datasets/{id}/editions/{edition}", api.getEdition).Methods("GET") - api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions", api.getVersions).Methods("GET") - api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}", api.getVersion).Methods("GET") - api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}/metadata", api.getMetadata).Methods("GET") - api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}/observations", api.getObservations).Methods("GET") - api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}/dimensions", api.getDimensions).Methods("GET") - api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}/dimensions/{dimension}/options", api.getDimensionOptions).Methods("GET") + api.Router.HandleFunc("/datasets", api.getDatasets).Methods("GET") + api.Router.HandleFunc("/datasets/{id}", api.getDataset).Methods("GET") + api.Router.HandleFunc("/datasets/{id}/editions", api.getEditions).Methods("GET") + api.Router.HandleFunc("/datasets/{id}/editions/{edition}", api.getEdition).Methods("GET") + api.Router.HandleFunc("/datasets/{id}/editions/{edition}/versions", api.getVersions).Methods("GET") + api.Router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}", api.getVersion).Methods("GET") + api.Router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}/metadata", api.getMetadata).Methods("GET") + api.Router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}/observations", api.getObservations).Methods("GET") + api.Router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}/dimensions", api.getDimensions).Methods("GET") + api.Router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}/dimensions/{dimension}/options", api.getDimensionOptions).Methods("GET") if cfg.EnablePrivateEnpoints { log.Debug("private endpoints have been enabled", nil) - versionPublishChecker := PublishCheck{Datastore: dataStore.Backend} - api.router.HandleFunc("/datasets/{id}", identity.Check(api.addDataset)).Methods("POST") - api.router.HandleFunc("/datasets/{id}", identity.Check(api.putDataset)).Methods("PUT") - api.router.HandleFunc("/datasets/{id}", identity.Check(api.deleteDataset)).Methods("DELETE") - api.router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}", identity.Check(versionPublishChecker.Check(api.putVersion))).Methods("PUT") - - instanceAPI := instance.Store{Host: api.host, Storer: api.dataStore.Backend} - instancePublishChecker := instance.PublishCheck{Datastore: dataStore.Backend} - api.router.HandleFunc("/instances", identity.Check(instanceAPI.GetList)).Methods("GET") - api.router.HandleFunc("/instances", identity.Check(instanceAPI.Add)).Methods("POST") - api.router.HandleFunc("/instances/{id}", identity.Check(instanceAPI.Get)).Methods("GET") - api.router.HandleFunc("/instances/{id}", identity.Check(instancePublishChecker.Check(instanceAPI.Update))).Methods("PUT") - api.router.HandleFunc("/instances/{id}/dimensions/{dimension}", identity.Check(instancePublishChecker.Check(instanceAPI.UpdateDimension))).Methods("PUT") - api.router.HandleFunc("/instances/{id}/events", identity.Check(instanceAPI.AddEvent)).Methods("POST") - api.router.HandleFunc("/instances/{id}/inserted_observations/{inserted_observations}", - identity.Check(instancePublishChecker.Check(instanceAPI.UpdateObservations))).Methods("PUT") - api.router.HandleFunc("/instances/{id}/import_tasks", identity.Check(instancePublishChecker.Check(instanceAPI.UpdateImportTask))).Methods("PUT") - - dimension := dimension.Store{Storer: api.dataStore.Backend} - api.router.HandleFunc("/instances/{id}/dimensions", identity.Check(dimension.GetNodes)).Methods("GET") - api.router.HandleFunc("/instances/{id}/dimensions", identity.Check(instancePublishChecker.Check(dimension.Add))).Methods("POST") - api.router.HandleFunc("/instances/{id}/dimensions/{dimension}/options", identity.Check(dimension.GetUnique)).Methods("GET") - api.router.HandleFunc("/instances/{id}/dimensions/{dimension}/options/{value}/node_id/{node_id}", - identity.Check(instancePublishChecker.Check(dimension.AddNodeID))).Methods("PUT") + versionPublishChecker := PublishCheck{Auditor: auditor, Datastore: dataStore.Backend} + api.Router.HandleFunc("/datasets/{id}", identity.Check(api.addDataset)).Methods("POST") + api.Router.HandleFunc("/datasets/{id}", identity.Check(api.putDataset)).Methods("PUT") + api.Router.HandleFunc("/datasets/{id}", identity.Check(api.deleteDataset)).Methods("DELETE") + api.Router.HandleFunc("/datasets/{id}/editions/{edition}/versions/{version}", identity.Check(versionPublishChecker.Check(api.putVersion, updateVersionAction))).Methods("PUT") + + instanceAPI := instance.Store{Host: api.host, Storer: api.dataStore.Backend, Auditor: auditor} + instancePublishChecker := instance.PublishCheck{Auditor: auditor, Datastore: dataStore.Backend} + api.Router.HandleFunc("/instances", identity.Check(instanceAPI.GetList)).Methods("GET") + api.Router.HandleFunc("/instances", identity.Check(instanceAPI.Add)).Methods("POST") + api.Router.HandleFunc("/instances/{id}", identity.Check(instanceAPI.Get)).Methods("GET") + api.Router.HandleFunc("/instances/{id}", identity.Check(instancePublishChecker.Check(instanceAPI.Update, instance.UpdateInstanceAction))).Methods("PUT") + api.Router.HandleFunc("/instances/{id}/dimensions/{dimension}", identity.Check(instancePublishChecker.Check(instanceAPI.UpdateDimension, instance.UpdateDimensionAction))).Methods("PUT") + api.Router.HandleFunc("/instances/{id}/events", identity.Check(instanceAPI.AddEvent)).Methods("POST") + api.Router.HandleFunc("/instances/{id}/inserted_observations/{inserted_observations}", + identity.Check(instancePublishChecker.Check(instanceAPI.UpdateObservations, instance.UpdateInsertedObservationsAction))).Methods("PUT") + api.Router.HandleFunc("/instances/{id}/import_tasks", identity.Check(instancePublishChecker.Check(instanceAPI.UpdateImportTask, instance.UpdateImportTasksAction))).Methods("PUT") + + dimensionAPI := dimension.Store{Auditor: auditor, Storer: api.dataStore.Backend} + api.Router.HandleFunc("/instances/{id}/dimensions", identity.Check(dimensionAPI.GetDimensionsHandler)).Methods("GET") + api.Router.HandleFunc("/instances/{id}/dimensions", identity.Check(instancePublishChecker.Check(dimensionAPI.AddHandler, dimension.PostDimensionsAction))).Methods("POST") + api.Router.HandleFunc("/instances/{id}/dimensions/{dimension}/options", identity.Check(dimensionAPI.GetUniqueDimensionAndOptionsHandler)).Methods("GET") + api.Router.HandleFunc("/instances/{id}/dimensions/{dimension}/options/{value}/node_id/{node_id}", + identity.Check(instancePublishChecker.Check(dimensionAPI.AddNodeIDHandler, dimension.PutNodeIDAction))).Methods("PUT") } return &api } -func handleErrorType(docType string, err error, w http.ResponseWriter) { - log.Error(err, nil) - - switch docType { - default: - if err == errs.ErrEditionNotFound || err == errs.ErrVersionNotFound || err == errs.ErrDimensionNodeNotFound || err == errs.ErrInstanceNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - case "dataset": - if err == errs.ErrDatasetNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else if err == errs.ErrDeleteDatasetNotFound { - http.Error(w, err.Error(), http.StatusNoContent) - } else if err == errs.ErrDeletePublishedDatasetForbidden || err == errs.ErrAddDatasetAlreadyExists { - http.Error(w, err.Error(), http.StatusForbidden) - } else if err == errs.ErrAddUpdateDatasetBadRequest { - http.Error(w, err.Error(), http.StatusBadRequest) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - case "edition": - if err == errs.ErrDatasetNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else if err == errs.ErrEditionNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - case "version": - if err == errs.ErrDatasetNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else if err == errs.ErrEditionNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else if err == errs.ErrVersionNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - case "dimension": - if err == errs.ErrDatasetNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else if err == errs.ErrEditionNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else if err == errs.ErrVersionNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else if err == errs.ErrDimensionsNotFound { - http.Error(w, err.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - } -} - // Check wraps a HTTP handle. Checks that the state is not published -func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request)) http.HandlerFunc { +func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request), action string) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - + ctx := r.Context() vars := mux.Vars(r) id := vars["id"] edition := vars["edition"] version := vars["version"] + data := log.Data{"dataset_id": id, "edition": edition, "version": version} + auditParams := common.Params{"dataset_id": id, "edition": edition, "version": version} + + if auditErr := d.Auditor.Record(ctx, action, audit.Attempted, auditParams); auditErr != nil { + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) + return + } currentVersion, err := d.Datastore.GetVersion(id, edition, version, "") if err != nil { if err != errs.ErrVersionNotFound { + log.ErrorCtx(ctx, errors.WithMessage(err, "errored whilst retrieving version resource"), data) + + if auditErr := d.Auditor.Record(ctx, action, audit.Unsuccessful, auditParams); auditErr != nil { + err = errs.ErrInternalServer + } + http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -263,22 +219,32 @@ func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request)) ht if currentVersion != nil { if currentVersion.State == models.PublishedState { defer func() { - if err := r.Body.Close(); err != nil { - log.ErrorC("could not close response body", err, nil) + if bodyCloseErr := r.Body.Close(); bodyCloseErr != nil { + log.ErrorCtx(ctx, errors.Wrap(bodyCloseErr, "could not close response body"), data) } }() - versionDoc, err := models.CreateVersion(r.Body) - if err != nil { - log.ErrorC("failed to model version resource based on request", err, log.Data{"dataset_id": id, "edition": edition, "version": version}) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } + // We can allow public download links to be modified by the exporter + // services when a version is published. Note that a new version will be + // created which contain only the download information to prevent any + // forbidden fields from being set on the published version + + // TODO Logic here might require it's own endpoint, + // possibly /datasets/.../versions//downloads + if action == updateVersionAction { + versionDoc, err := models.CreateVersion(r.Body) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "failed to model version resource based on request"), data) + + if auditErr := d.Auditor.Record(ctx, action, audit.Unsuccessful, auditParams); auditErr != nil { + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) + return + } + + http.Error(w, err.Error(), http.StatusBadRequest) + return + } - // We can allow public download links to be modified by the exporters when a version is published. - // Note that a new version will be created which contain only the download information to prevent - // any forbidden fields from being set on the published version - if r.Method == "PUT" { if versionDoc.Downloads != nil { newVersion := new(models.Version) if versionDoc.Downloads.CSV != nil && versionDoc.Downloads.CSV.Public != "" { @@ -292,6 +258,7 @@ func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request)) ht }, } } + if versionDoc.Downloads.XLS != nil && versionDoc.Downloads.XLS.Public != "" { newVersion = &models.Version{ Downloads: &models.DownloadList{ @@ -304,15 +271,27 @@ func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request)) ht } } if newVersion != nil { - b, err := json.Marshal(newVersion) + var b []byte + b, err = json.Marshal(newVersion) if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "failed to marshal new version resource based on request"), data) + + if auditErr := d.Auditor.Record(ctx, action, audit.Unsuccessful, auditParams); auditErr != nil { + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) + return + } + http.Error(w, err.Error(), http.StatusForbidden) return } - if err := r.Body.Close(); err != nil { - log.ErrorC("could not close response body", err, nil) + if err = r.Body.Close(); err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "could not close response body"), data) } + + // Set variable `has_downloads` to true to prevent request + // triggering version from being republished + vars[hasDownloads] = trueStringified r.Body = ioutil.NopCloser(bytes.NewBuffer(b)) handle(w, r) return @@ -321,7 +300,13 @@ func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request)) ht } err = errors.New("unable to update version as it has been published") - log.Error(err, log.Data{"version": currentVersion}) + data["version"] = currentVersion + log.ErrorCtx(ctx, err, data) + if auditErr := d.Auditor.Record(ctx, action, audit.Unsuccessful, auditParams); auditErr != nil { + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) + return + } + http.Error(w, err.Error(), http.StatusForbidden) return } @@ -331,6 +316,10 @@ func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request)) ht }) } +func setJSONContentType(w http.ResponseWriter) { + w.Header().Set("Content-Type", "application/json") +} + func (api *DatasetAPI) authenticate(r *http.Request, logData map[string]interface{}) (bool, map[string]interface{}) { var authorised bool @@ -359,50 +348,6 @@ func (api *DatasetAPI) authenticate(r *http.Request, logData map[string]interfac return authorised, logData } -func handleAuditingFailure(w http.ResponseWriter, err error, logData log.Data) { - log.ErrorC(auditError, err, logData) - http.Error(w, "internal server error", http.StatusInternalServerError) -} - -func auditActionFailure(ctx context.Context, auditedAction string, auditedResult string, err error, logData log.Data) { - if logData == nil { - logData = log.Data{} - } - - logData["auditAction"] = auditedAction - logData["auditResult"] = auditedResult - - logError(ctx, errors.WithMessage(err, auditActionErr), logData) -} - -func logError(ctx context.Context, err error, data log.Data) { - if data == nil { - data = log.Data{} - } - reqID := requestID.Get(ctx) - if user := common.User(ctx); user != "" { - data["user"] = user - } - if caller := common.Caller(ctx); caller != "" { - data["caller"] = caller - } - log.ErrorC(reqID, err, data) -} - -func logInfo(ctx context.Context, message string, data log.Data) { - if data == nil { - data = log.Data{} - } - reqID := requestID.Get(ctx) - if user := common.User(ctx); user != "" { - data["user"] = user - } - if caller := common.Caller(ctx); caller != "" { - data["caller"] = caller - } - log.InfoC(reqID, message, data) -} - // Close represents the graceful shutting down of the http server func Close(ctx context.Context) error { if err := httpServer.Shutdown(ctx); err != nil { diff --git a/api/dataset.go b/api/dataset.go index c9ea158a..213e604b 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -9,73 +9,92 @@ import ( errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" + "github.com/ONSdigital/go-ns/audit" "github.com/ONSdigital/go-ns/common" "github.com/ONSdigital/go-ns/log" "github.com/gorilla/mux" "github.com/pkg/errors" ) +var ( + // errors that should return a 403 status + datasetsForbidden = map[error]bool{ + errs.ErrDeletePublishedDatasetForbidden: true, + errs.ErrAddDatasetAlreadyExists: true, + } + + // errors that should return a 404 status + datasetsNotFound = map[error]bool{ + errs.ErrDatasetNotFound: true, + } + + // errors that should return a 204 status + datasetsNoContent = map[error]bool{ + errs.ErrDeleteDatasetNotFound: true, + } + + // errors that should return a 400 status + datasetsBadRequest = map[error]bool{ + errs.ErrAddUpdateDatasetBadRequest: true, + } +) + func (api *DatasetAPI) getDatasets(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - if err := api.auditor.Record(ctx, getDatasetsAction, actionAttempted, nil); err != nil { - auditActionFailure(ctx, getDatasetsAction, actionAttempted, err, nil) - handleDatasetAPIErr(ctx, errs.ErrAuditActionAttemptedFailure, w, nil) + if err := api.auditor.Record(ctx, getDatasetsAction, audit.Attempted, nil); err != nil { + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) return } b, err := func() ([]byte, error) { - results, err := api.dataStore.Backend.GetDatasets() + datasets, err := api.dataStore.Backend.GetDatasets() if err != nil { - logError(ctx, errors.WithMessage(err, "api endpoint getDatasets datastore.GetDatasets returned an error"), nil) + log.ErrorCtx(ctx, errors.WithMessage(err, "api endpoint getDatasets datastore.GetDatasets returned an error"), nil) return nil, err } authorised, logData := api.authenticate(r, log.Data{}) var b []byte - if authorised { + var datasetsResponse interface{} + if authorised { // User has valid authentication to get raw dataset document - datasets := &models.DatasetUpdateResults{} - datasets.Items = results - b, err = json.Marshal(datasets) - if err != nil { - logError(ctx, errors.WithMessage(err, "api endpoint getDatasets failed to marshal dataset resource into bytes"), logData) - return nil, err - } + datasetsResponse = &models.DatasetUpdateResults{Items: datasets} } else { - // User is not authenticated and hence has only access to current sub document - datasets := &models.DatasetResults{} - datasets.Items = mapResults(results) + datasetsResponse = &models.DatasetResults{Items: mapResults(datasets)} + } - b, err = json.Marshal(datasets) - if err != nil { - logError(ctx, errors.WithMessage(err, "api endpoint getDatasets failed to marshal dataset resource into bytes"), logData) - return nil, err - } + b, err = json.Marshal(datasetsResponse) + + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "api endpoint getDatasets failed to marshal dataset resource into bytes"), logData) + return nil, err } - return b, err + + return b, nil }() if err != nil { - if auditErr := api.auditor.Record(ctx, getDatasetsAction, actionUnsuccessful, nil); auditErr != nil { - auditActionFailure(ctx, getDatasetsAction, actionUnsuccessful, auditErr, nil) + if auditErr := api.auditor.Record(ctx, getDatasetsAction, audit.Unsuccessful, nil); auditErr != nil { + err = auditErr } handleDatasetAPIErr(ctx, err, w, nil) return } - if auditErr := api.auditor.Record(ctx, getDatasetsAction, actionSuccessful, nil); auditErr != nil { - auditActionFailure(ctx, getDatasetsAction, actionSuccessful, auditErr, nil) + if auditErr := api.auditor.Record(ctx, getDatasetsAction, audit.Successful, nil); auditErr != nil { + handleDatasetAPIErr(ctx, auditErr, w, nil) + return } setJSONContentType(w) - _, err = w.Write(b) - if err != nil { - logError(ctx, errors.WithMessage(err, "api endpoint getDatasets error writing response body"), nil) - http.Error(w, err.Error(), http.StatusInternalServerError) + if _, err = w.Write(b); err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "api endpoint getDatasets error writing response body"), nil) + handleDatasetAPIErr(ctx, err, w, nil) + return } - logInfo(ctx, "api endpoint getDatasets request successful", nil) + log.InfoCtx(ctx, "api endpoint getDatasets request successful", nil) } func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) { @@ -85,8 +104,7 @@ func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) { logData := log.Data{"dataset_id": id} auditParams := common.Params{"dataset_id": id} - if auditErr := api.auditor.Record(ctx, getDatasetAction, actionAttempted, auditParams); auditErr != nil { - auditActionFailure(ctx, getDatasetAction, actionAttempted, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getDatasetAction, audit.Attempted, auditParams); auditErr != nil { handleDatasetAPIErr(ctx, errs.ErrInternalServer, w, logData) return } @@ -94,62 +112,65 @@ func (api *DatasetAPI) getDataset(w http.ResponseWriter, r *http.Request) { b, err := func() ([]byte, error) { dataset, err := api.dataStore.Backend.GetDataset(id) if err != nil { - logError(ctx, errors.WithMessage(err, "getDataset endpoint: dataStore.Backend.GetDataset returned an error"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getDataset endpoint: dataStore.Backend.GetDataset returned an error"), logData) return nil, err } authorised, logData := api.authenticate(r, logData) var b []byte + var datasetResponse interface{} + + // var marshallErr error if !authorised { // User is not authenticated and hence has only access to current sub document if dataset.Current == nil { - logInfo(ctx, "getDataste endpoint: published dataset not found", logData) + log.InfoCtx(ctx, "getDataste endpoint: published dataset not found", logData) return nil, errs.ErrDatasetNotFound } + log.InfoCtx(ctx, "getDataset endpoint: caller authorised returning dataset current sub document", logData) + dataset.Current.ID = dataset.ID - b, err = json.Marshal(dataset.Current) - if err != nil { - logError(ctx, errors.WithMessage(err, "getDataset endpoint: failed to marshal dataset current sub document resource into bytes"), logData) - return nil, err - } + datasetResponse = dataset.Current } else { // User has valid authentication to get raw dataset document if dataset == nil { - logInfo(ctx, "getDataset endpoint: published or unpublished dataset not found", logData) + log.InfoCtx(ctx, "getDataset endpoint: published or unpublished dataset not found", logData) return nil, errs.ErrDatasetNotFound } - b, err = json.Marshal(dataset) - if err != nil { - logError(ctx, errors.WithMessage(err, "getDataset endpoint: failed to marshal dataset current sub document resource into bytes"), logData) - return nil, err - } + log.InfoCtx(ctx, "getDataset endpoint: caller not authorised returning dataset", logData) + datasetResponse = dataset + } + + b, err = json.Marshal(datasetResponse) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "getDataset endpoint: failed to marshal dataset resource into bytes"), logData) + return nil, err } + return b, nil }() if err != nil { - if auditErr := api.auditor.Record(ctx, getDatasetAction, actionUnsuccessful, auditParams); auditErr != nil { - auditActionFailure(ctx, getDatasetAction, actionUnsuccessful, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getDatasetAction, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr } handleDatasetAPIErr(ctx, err, w, logData) return } - if auditErr := api.auditor.Record(ctx, getDatasetAction, actionSuccessful, auditParams); auditErr != nil { - auditActionFailure(ctx, getDatasetAction, actionSuccessful, auditErr, logData) - handleDatasetAPIErr(ctx, errs.ErrInternalServer, w, logData) + if auditErr := api.auditor.Record(ctx, getDatasetAction, audit.Successful, auditParams); auditErr != nil { + handleDatasetAPIErr(ctx, auditErr, w, logData) return } setJSONContentType(w) - _, err = w.Write(b) - if err != nil { - logError(ctx, errors.WithMessage(err, "getDataset endpoint: error writing bytes to response"), logData) - http.Error(w, err.Error(), http.StatusInternalServerError) + if _, err = w.Write(b); err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "getDataset endpoint: error writing bytes to response"), logData) + handleDatasetAPIErr(ctx, err, w, logData) } - logInfo(ctx, "getDataset endpoint: request successful", logData) + log.InfoCtx(ctx, "getDataset endpoint: request successful", logData) } func (api *DatasetAPI) addDataset(w http.ResponseWriter, r *http.Request) { @@ -160,9 +181,8 @@ func (api *DatasetAPI) addDataset(w http.ResponseWriter, r *http.Request) { logData := log.Data{"dataset_id": datasetID} auditParams := common.Params{"dataset_id": datasetID} - if err := api.auditor.Record(ctx, addDatasetAction, actionAttempted, auditParams); err != nil { - auditActionFailure(ctx, addDatasetAction, actionAttempted, err, logData) - handleDatasetAPIErr(ctx, errs.ErrInternalServer, w, logData) + if auditErr := api.auditor.Record(ctx, addDatasetAction, audit.Attempted, auditParams); auditErr != nil { + handleDatasetAPIErr(ctx, auditErr, w, logData) return } @@ -171,18 +191,18 @@ func (api *DatasetAPI) addDataset(w http.ResponseWriter, r *http.Request) { _, err := api.dataStore.Backend.GetDataset(datasetID) if err != nil { if err != errs.ErrDatasetNotFound { - logError(ctx, errors.WithMessage(err, "addDataset endpoint: error checking if dataset exists"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "addDataset endpoint: error checking if dataset exists"), logData) return nil, err } } else { - logError(ctx, errors.WithMessage(errs.ErrAddDatasetAlreadyExists, "addDataset endpoint: unable to create a dataset that already exists"), logData) + log.ErrorCtx(ctx, errors.WithMessage(errs.ErrAddDatasetAlreadyExists, "addDataset endpoint: unable to create a dataset that already exists"), logData) return nil, errs.ErrAddDatasetAlreadyExists } defer r.Body.Close() dataset, err := models.CreateDataset(r.Body) if err != nil { - logError(ctx, errors.WithMessage(err, "addDataset endpoint: failed to model dataset resource based on request"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "addDataset endpoint: failed to model dataset resource based on request"), logData) return nil, errs.ErrAddUpdateDatasetBadRequest } @@ -210,38 +230,33 @@ func (api *DatasetAPI) addDataset(w http.ResponseWriter, r *http.Request) { if err = api.dataStore.Backend.UpsertDataset(datasetID, datasetDoc); err != nil { logData["new_dataset"] = datasetID - logError(ctx, errors.WithMessage(err, "addDataset endpoint: failed to insert dataset resource to datastore"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "addDataset endpoint: failed to insert dataset resource to datastore"), logData) return nil, err } b, err := json.Marshal(datasetDoc) if err != nil { - logError(ctx, errors.WithMessage(err, "addDataset endpoint: failed to marshal dataset resource into bytes"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "addDataset endpoint: failed to marshal dataset resource into bytes"), logData) return nil, err } return b, nil }() if err != nil { - if auditErr := api.auditor.Record(ctx, addDatasetAction, actionUnsuccessful, auditParams); auditErr != nil { - auditActionFailure(ctx, addDatasetAction, actionUnsuccessful, auditErr, logData) - } + api.auditor.Record(ctx, addDatasetAction, audit.Unsuccessful, auditParams) handleDatasetAPIErr(ctx, err, w, logData) return } - if auditErr := api.auditor.Record(ctx, addDatasetAction, actionSuccessful, auditParams); auditErr != nil { - auditActionFailure(ctx, addDatasetAction, actionSuccessful, auditErr, logData) - } + api.auditor.Record(ctx, addDatasetAction, audit.Successful, auditParams) setJSONContentType(w) w.WriteHeader(http.StatusCreated) - _, err = w.Write(b) - if err != nil { - logError(ctx, errors.WithMessage(err, "addDataset endpoint: error writing bytes to response"), logData) + if _, err = w.Write(b); err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "addDataset endpoint: error writing bytes to response"), logData) http.Error(w, err.Error(), http.StatusInternalServerError) } - logInfo(ctx, "addDataset endpoint: request completed successfully", logData) + log.InfoCtx(ctx, "addDataset endpoint: request completed successfully", logData) } func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) { @@ -251,9 +266,8 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) { data := log.Data{"dataset_id": datasetID} auditParams := common.Params{"dataset_id": datasetID} - if err := api.auditor.Record(ctx, putDatasetAction, actionAttempted, auditParams); err != nil { - auditActionFailure(ctx, putDatasetAction, actionAttempted, err, data) - handleDatasetAPIErr(ctx, err, w, data) + if auditErr := api.auditor.Record(ctx, putDatasetAction, audit.Attempted, auditParams); auditErr != nil { + handleDatasetAPIErr(ctx, auditErr, w, data) return } @@ -262,24 +276,24 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) { dataset, err := models.CreateDataset(r.Body) if err != nil { - logError(ctx, errors.WithMessage(err, "putDataset endpoint: failed to model dataset resource based on request"), data) + log.ErrorCtx(ctx, errors.WithMessage(err, "putDataset endpoint: failed to model dataset resource based on request"), data) return errs.ErrAddUpdateDatasetBadRequest } currentDataset, err := api.dataStore.Backend.GetDataset(datasetID) if err != nil { - logError(ctx, errors.WithMessage(err, "putDataset endpoint: datastore.getDataset returned an error"), data) + log.ErrorCtx(ctx, errors.WithMessage(err, "putDataset endpoint: datastore.getDataset returned an error"), data) return err } if dataset.State == models.PublishedState { - if err := api.publishDataset(currentDataset, nil); err != nil { - logError(ctx, errors.WithMessage(err, "putDataset endpoint: failed to update dataset document to published"), data) + if err := api.publishDataset(ctx, currentDataset, nil); err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "putDataset endpoint: failed to update dataset document to published"), data) return err } } else { if err := api.dataStore.Backend.UpdateDataset(datasetID, dataset, currentDataset.Next.State); err != nil { - logError(ctx, errors.WithMessage(err, "putDataset endpoint: failed to update dataset resource"), data) + log.ErrorCtx(ctx, errors.WithMessage(err, "putDataset endpoint: failed to update dataset resource"), data) return err } } @@ -287,24 +301,19 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) { }() if err != nil { - if err := api.auditor.Record(ctx, putDatasetAction, actionUnsuccessful, auditParams); err != nil { - auditActionFailure(ctx, putDatasetAction, actionUnsuccessful, err, data) - } - + api.auditor.Record(ctx, putDatasetAction, audit.Unsuccessful, auditParams) handleDatasetAPIErr(ctx, err, w, data) return } - if err := api.auditor.Record(ctx, putDatasetAction, actionSuccessful, auditParams); err != nil { - auditActionFailure(ctx, putDatasetAction, actionSuccessful, err, data) - } + api.auditor.Record(ctx, putDatasetAction, audit.Successful, auditParams) setJSONContentType(w) w.WriteHeader(http.StatusOK) - logInfo(ctx, "putDataset endpoint: request successful", data) + log.InfoCtx(ctx, "putDataset endpoint: request successful", data) } -func (api *DatasetAPI) publishDataset(currentDataset *models.DatasetUpdate, version *models.Version) error { +func (api *DatasetAPI) publishDataset(ctx context.Context, currentDataset *models.DatasetUpdate, version *models.Version) error { if version != nil { currentDataset.Next.CollectionID = "" @@ -328,7 +337,7 @@ func (api *DatasetAPI) publishDataset(currentDataset *models.DatasetUpdate, vers } if err := api.dataStore.Backend.UpsertDataset(currentDataset.ID, newDataset); err != nil { - log.ErrorC("unable to update dataset", err, log.Data{"dataset_id": currentDataset.ID}) + log.ErrorCtx(ctx, errors.WithMessage(err, "unable to update dataset"), log.Data{"dataset_id": currentDataset.ID}) return err } @@ -342,9 +351,8 @@ func (api *DatasetAPI) deleteDataset(w http.ResponseWriter, r *http.Request) { logData := log.Data{"dataset_id": datasetID} auditParams := common.Params{"dataset_id": datasetID} - if err := api.auditor.Record(ctx, deleteDatasetAction, actionAttempted, auditParams); err != nil { - auditActionFailure(ctx, deleteDatasetAction, actionAttempted, err, logData) - handleDatasetAPIErr(ctx, err, w, logData) + if auditErr := api.auditor.Record(ctx, deleteDatasetAction, audit.Attempted, auditParams); auditErr != nil { + handleDatasetAPIErr(ctx, auditErr, w, logData) return } @@ -353,40 +361,35 @@ func (api *DatasetAPI) deleteDataset(w http.ResponseWriter, r *http.Request) { currentDataset, err := api.dataStore.Backend.GetDataset(datasetID) if err == errs.ErrDatasetNotFound { - log.Debug("cannot delete dataset, it does not exist", logData) + log.InfoCtx(ctx, "cannot delete dataset, it does not exist", logData) return errs.ErrDeleteDatasetNotFound } if err != nil { - log.ErrorC("failed to run query for existing dataset", err, logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "failed to run query for existing dataset"), logData) return err } if currentDataset.Current != nil && currentDataset.Current.State == models.PublishedState { - log.ErrorC("unable to delete a published dataset", errs.ErrDeletePublishedDatasetForbidden, logData) + log.ErrorCtx(ctx, errors.WithMessage(errs.ErrDeletePublishedDatasetForbidden, "unable to delete a published dataset"), logData) return errs.ErrDeletePublishedDatasetForbidden } if err := api.dataStore.Backend.DeleteDataset(datasetID); err != nil { - log.ErrorC("failed to delete dataset", err, logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "failed to delete dataset"), logData) return err } - log.Debug("dataset deleted successfully", logData) + log.InfoCtx(ctx, "dataset deleted successfully", logData) return nil }() if err != nil { - if auditErr := api.auditor.Record(ctx, deleteDatasetAction, actionUnsuccessful, auditParams); auditErr != nil { - auditActionFailure(ctx, deleteDatasetAction, actionUnsuccessful, auditErr, logData) - } + api.auditor.Record(ctx, deleteDatasetAction, audit.Unsuccessful, auditParams) handleDatasetAPIErr(ctx, err, w, logData) return } - if err := api.auditor.Record(ctx, deleteDatasetAction, actionSuccessful, auditParams); err != nil { - auditActionFailure(ctx, deleteDatasetAction, actionSuccessful, err, logData) - // fall through and return the origin status code as the action has been carried out at this point. - } + api.auditor.Record(ctx, deleteDatasetAction, audit.Successful, auditParams) w.WriteHeader(http.StatusNoContent) log.Debug("delete dataset", logData) } @@ -411,21 +414,20 @@ func handleDatasetAPIErr(ctx context.Context, err error, w http.ResponseWriter, var status int switch { - case err == errs.ErrDeletePublishedDatasetForbidden: - status = http.StatusForbidden - case err == errs.ErrAddDatasetAlreadyExists: + case datasetsForbidden[err]: status = http.StatusForbidden - case err == errs.ErrDatasetNotFound: + case datasetsNotFound[err]: status = http.StatusNotFound - case err == errs.ErrDeleteDatasetNotFound: + case datasetsNoContent[err]: status = http.StatusNoContent - case err == errs.ErrAddUpdateDatasetBadRequest: + case datasetsBadRequest[err]: status = http.StatusBadRequest default: + err = errs.ErrInternalServer status = http.StatusInternalServerError } data["responseStatus"] = status - logError(ctx, errors.WithMessage(err, "request unsuccessful"), data) + log.ErrorCtx(ctx, errors.WithMessage(err, "request unsuccessful"), data) http.Error(w, err.Error(), status) } diff --git a/api/dataset_test.go b/api/dataset_test.go index 6aa7f2ff..f0283c39 100644 --- a/api/dataset_test.go +++ b/api/dataset_test.go @@ -8,18 +8,17 @@ import ( "io" "net/http" "net/http/httptest" - "strings" "testing" "time" errs "github.com/ONSdigital/dp-dataset-api/apierrors" - "github.com/ONSdigital/dp-dataset-api/mocks" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/store" - "github.com/ONSdigital/dp-dataset-api/store/datastoretest" + storetest "github.com/ONSdigital/dp-dataset-api/store/datastoretest" "github.com/ONSdigital/dp-dataset-api/url" "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/audit/audit_mock" "github.com/ONSdigital/go-ns/common" "github.com/gorilla/mux" @@ -35,19 +34,15 @@ const ( ) var ( - errInternal = errors.New("internal error") - errBadRequest = errors.New("bad request") - errNotFound = errors.New("not found") - datasetPayload = `{"contacts":[{"email":"testing@hotmail.com","name":"John Cox","telephone":"01623 456789"}],"description":"census","links":{"access_rights":{"href":"http://ons.gov.uk/accessrights"}},"title":"CensusEthnicity","theme":"population","periodicity":"yearly","state":"completed","next_release":"2016-04-04","publisher":{"name":"The office of national statistics","type":"government department","url":"https://www.ons.gov.uk/"}}` urlBuilder = url.NewBuilder("localhost:20000") genericMockedObservationStore = &mocks.ObservationStoreMock{} - auditParams = common.Params{"dataset_id": "123-456"} + genericAuditParams = common.Params{"dataset_id": "123-456"} ) // GetAPIWithMockedDatastore also used in other tests, so exported -func GetAPIWithMockedDatastore(mockedDataStore store.Storer, mockedGeneratedDownloads DownloadsGenerator, mockAuditor Auditor, mockedObservationStore ObservationStore) *DatasetAPI { +func GetAPIWithMockedDatastore(mockedDataStore store.Storer, mockedGeneratedDownloads DownloadsGenerator, auditMock Auditor, mockedObservationStore ObservationStore) *DatasetAPI { cfg, err := config.Get() So(err, ShouldBeNil) cfg.ServiceAuthToken = authToken @@ -55,7 +50,7 @@ func GetAPIWithMockedDatastore(mockedDataStore store.Storer, mockedGeneratedDown cfg.EnablePrivateEnpoints = true cfg.HealthCheckTimeout = healthTimeout - return routes(*cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedGeneratedDownloads, mockAuditor, mockedObservationStore) + return Routes(*cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedGeneratedDownloads, auditMock, mockedObservationStore) } func createRequestWithAuth(method, URL string, body io.Reader) (*http.Request, error) { @@ -66,17 +61,6 @@ func createRequestWithAuth(method, URL string, body io.Reader) (*http.Request, e return r, err } -func verifyAuditRecordCalls(c struct { - Ctx context.Context - Action string - Result string - Params common.Params -}, expectedAction string, expectedResult string, expectedParams common.Params) { - So(c.Action, ShouldEqual, expectedAction) - So(c.Result, ShouldEqual, expectedResult) - So(c.Params, ShouldResemble, expectedParams) -} - func TestGetDatasetsReturnsOK(t *testing.T) { t.Parallel() Convey("A successful request to get dataset returns 200 OK response", t, func() { @@ -88,18 +72,18 @@ func TestGetDatasetsReturnsOK(t *testing.T) { }, } - mockAuditor := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, mockAuditor, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - - recCalls := mockAuditor.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getDatasetsAction, actionAttempted, nil) - verifyAuditRecordCalls(recCalls[1], getDatasetsAction, actionSuccessful, nil) So(len(mockedDataStore.GetDatasetsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDatasetsAction, Result: audit.Attempted, Params: nil}, + audit_mock.Expected{Action: getDatasetsAction, Result: audit.Successful, Params: nil}, + ) }) } @@ -110,26 +94,29 @@ func TestGetDatasetsReturnsErrorIfAuditAttemptFails(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetsFunc: func() ([]models.DatasetUpdate, error) { - return nil, errInternal + return nil, errs.ErrInternalServer }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { return errors.New("boom!") } api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 1) - verifyAuditRecordCalls(recCalls[0], getDatasetsAction, actionAttempted, nil) + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{ + Action: getDatasetsAction, + Result: audit.Attempted, + Params: nil, + }, + ) }) Convey("When auditing get datasets errors an internal server error is returned", t, func() { @@ -137,30 +124,29 @@ func TestGetDatasetsReturnsErrorIfAuditAttemptFails(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetsFunc: func() ([]models.DatasetUpdate, error) { - return nil, errInternal + return nil, errs.ErrInternalServer }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getDatasetsAction && result == actionUnsuccessful { + if action == getDatasetsAction && result == audit.Unsuccessful { return errors.New("boom!") } return nil } api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(strings.TrimSpace(w.Body.String()), ShouldEqual, errInternal.Error()) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getDatasetsAction, actionAttempted, nil) - verifyAuditRecordCalls(recCalls[1], getDatasetsAction, actionUnsuccessful, nil) + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDatasetsAction, Result: audit.Attempted, Params: nil}, + audit_mock.Expected{Action: getDatasetsAction, Result: audit.Unsuccessful, Params: nil}, + ) }) } @@ -171,29 +157,28 @@ func TestGetDatasetsReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetsFunc: func() ([]models.DatasetUpdate, error) { - return nil, errInternal + return nil, errs.ErrInternalServer }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getDatasetsAction, actionAttempted, nil) - verifyAuditRecordCalls(recCalls[1], getDatasetsAction, actionUnsuccessful, nil) - + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDatasetsAction, Result: audit.Attempted, Params: nil}, + audit_mock.Expected{Action: getDatasetsAction, Result: audit.Unsuccessful, Params: nil}, + ) }) } -func TestGetDatasetsAuditActionSuccessfulError(t *testing.T) { +func TestGetDatasetsAuditSuccessfulError(t *testing.T) { t.Parallel() - Convey("when a successful request to get dataset fails to audit action successful then a 200 response is returned", t, func() { + Convey("when a successful request to get dataset fails to audit action successful then a 500 response is returned", t, func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets", nil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -202,24 +187,18 @@ func TestGetDatasetsAuditActionSuccessfulError(t *testing.T) { }, } - mockAuditor := getMockAuditor() - mockAuditor.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getDatasetsAction && result == actionSuccessful { - return errors.New("boom") - } - return nil - } - - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, mockAuditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) - - recCalls := mockAuditor.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getDatasetsAction, actionAttempted, nil) - verifyAuditRecordCalls(recCalls[1], getDatasetsAction, actionSuccessful, nil) + auditMock := audit_mock.NewErroring(getDatasetsAction, audit.Successful) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetsCalls()), ShouldEqual, 1) - So(w.Code, ShouldEqual, http.StatusOK) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDatasetsAction, Result: audit.Attempted, Params: nil}, + audit_mock.Expected{Action: getDatasetsAction, Result: audit.Successful, Params: nil}, + ) }) } @@ -234,18 +213,18 @@ func TestGetDatasetReturnsOK(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getDatasetAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getDatasetAction, actionSuccessful, auditParams) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDatasetAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getDatasetAction, Result: audit.Successful, Params: genericAuditParams}, + ) }) Convey("When dataset document has only a next sub document and request is authorised return status 200", t, func() { @@ -259,17 +238,18 @@ func TestGetDatasetReturnsOK(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getDatasetAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getDatasetAction, actionSuccessful, auditParams) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDatasetAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getDatasetAction, Result: audit.Successful, Params: genericAuditParams}, + ) }) } @@ -280,21 +260,22 @@ func TestGetDatasetReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetFunc: func(id string) (*models.DatasetUpdate, error) { - return nil, errInternal + return nil, errs.ErrInternalServer }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - verifyAuditRecordCalls(recCalls[0], getDatasetAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getDatasetAction, actionUnsuccessful, auditParams) - So(w.Code, ShouldEqual, http.StatusInternalServerError) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDatasetAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getDatasetAction, Result: audit.Unsuccessful, Params: genericAuditParams}, + ) }) Convey("When dataset document has only a next sub document return status 404", t, func() { @@ -306,11 +287,18 @@ func TestGetDatasetReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDatasetAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getDatasetAction, Result: audit.Unsuccessful, Params: genericAuditParams}, + ) }) Convey("When there is no dataset document return status 404", t, func() { @@ -324,24 +312,24 @@ func TestGetDatasetReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - verifyAuditRecordCalls(recCalls[0], getDatasetAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getDatasetAction, actionUnsuccessful, auditParams) So(w.Code, ShouldEqual, http.StatusNotFound) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDatasetAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getDatasetAction, Result: audit.Unsuccessful, Params: genericAuditParams}, + ) }) } func TestGetDatasetAuditingErrors(t *testing.T) { Convey("given auditing attempted action returns an error", t, func() { - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { return errors.New("auditing error") } @@ -353,24 +341,28 @@ func TestGetDatasetAuditingErrors(t *testing.T) { mockDatastore := &storetest.StorerMock{} api := GetAPIWithMockedDatastore(mockDatastore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 1) - verifyAuditRecordCalls(recCalls[0], getDatasetAction, actionAttempted, auditParams) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(strings.TrimSpace(w.Body.String()), ShouldEqual, errs.ErrInternalServer.Error()) + assertInternalServerErr(w) So(len(mockDatastore.GetDatasetCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{ + Action: getDatasetAction, + Result: audit.Attempted, + Params: genericAuditParams, + }, + ) }) }) }) Convey("given audit action successful returns an error", t, func() { - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getDatasetAction && result == actionSuccessful { + if action == getDatasetAction && result == audit.Successful { return errors.New("auditing error") } return nil @@ -385,27 +377,27 @@ func TestGetDatasetAuditingErrors(t *testing.T) { return &models.DatasetUpdate{ID: "123", Current: &models.Dataset{ID: "123"}}, nil }, } - api := GetAPIWithMockedDatastore(mockDatastore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockDatastore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getDatasetAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getDatasetAction, actionSuccessful, auditParams) - So(len(mockDatastore.GetDatasetCalls()), ShouldEqual, 1) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(strings.TrimSpace(w.Body.String()), ShouldEqual, errs.ErrInternalServer.Error()) + assertInternalServerErr(w) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDatasetAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getDatasetAction, Result: audit.Successful, Params: genericAuditParams}, + ) }) }) }) Convey("given auditing action unsuccessful returns an error", t, func() { - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getDatasetAction && result == actionUnsuccessful { + if action == getDatasetAction && result == audit.Unsuccessful { return errors.New("auditing error") } return nil @@ -422,18 +414,17 @@ func TestGetDatasetAuditingErrors(t *testing.T) { } api := GetAPIWithMockedDatastore(mockDatastore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(strings.TrimSpace(w.Body.String()), ShouldEqual, "get dataset error") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getDatasetAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getDatasetAction, actionUnsuccessful, auditParams) - + assertInternalServerErr(w) So(len(mockDatastore.GetDatasetCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDatasetAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getDatasetAction, Result: audit.Unsuccessful, Params: genericAuditParams}, + ) }) }) }) @@ -456,14 +447,22 @@ func TestPostDatasetsReturnsCreated(t *testing.T) { return nil }, } - mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{Next: &models.Dataset{}}) - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{Next: &models.Dataset{}}) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusCreated) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 2) + + p := common.Params{"dataset_id": "123"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: addDatasetAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: addDatasetAction, Result: audit.Successful, Params: p}, + ) }) } @@ -481,18 +480,25 @@ func TestPostDatasetReturnsError(t *testing.T) { return nil, errs.ErrDatasetNotFound }, UpsertDatasetFunc: func(string, *models.DatasetUpdate) error { - return errBadRequest + return errs.ErrAddUpdateDatasetBadRequest }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "Failed to parse json body\n") - + So(w.Body.String(), ShouldContainSubstring, errs.ErrUnableToParseJSON.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) + + p := common.Params{"dataset_id": "123"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: addDatasetAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: addDatasetAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When the api cannot connect to datastore return an internal server error", t, func() { @@ -504,21 +510,27 @@ func TestPostDatasetReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { - return nil, errInternal + return nil, errs.ErrInternalServer }, UpsertDatasetFunc: func(string, *models.DatasetUpdate) error { return nil }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) + + p := common.Params{"dataset_id": "123"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: addDatasetAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: addDatasetAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When the request does not contain a valid internal token returns 401", t, func() { @@ -535,14 +547,17 @@ func TestPostDatasetReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusUnauthorized) So(w.Body.String(), ShouldResemble, "unauthenticated request\n") - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 0) + auditMock.AssertRecordCalls() }) Convey("When the dataset already exists and a request is sent to create the same dataset return status forbidden", t, func() { @@ -565,14 +580,21 @@ func TestPostDatasetReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusForbidden) So(w.Body.String(), ShouldResemble, "forbidden - dataset already exists\n") - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) + + p := common.Params{"dataset_id": "123"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: addDatasetAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: addDatasetAction, Result: audit.Unsuccessful, Params: p}, + ) }) } @@ -580,12 +602,7 @@ func TestPostDatasetAuditErrors(t *testing.T) { ap := common.Params{"dataset_id": "123"} Convey("given audit action attempted returns an error", t, func() { - auditor := getMockAuditorFunc(func(a string, r string) error { - if a == addDatasetAction && r == actionAttempted { - return errors.New("auditing error") - } - return nil - }) + auditMock := audit_mock.NewErroring(addDatasetAction, audit.Attempted) Convey("when add dataset is called", func() { r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString("{")) @@ -593,30 +610,28 @@ func TestPostDatasetAuditErrors(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{} - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(strings.TrimSpace(w.Body.String()), ShouldEqual, errs.ErrInternalServer.Error()) - + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 1) - verifyAuditRecordCalls(calls[0], addDatasetAction, actionAttempted, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{ + Action: addDatasetAction, + Result: audit.Attempted, + Params: ap, + }, + ) }) }) }) Convey("given audit action unsuccessful returns an error", t, func() { - auditor := getMockAuditorFunc(func(a string, r string) error { - if a == addDatasetAction && r == actionUnsuccessful { - return errors.New("auditing error") - } - return nil - }) + auditMock := audit_mock.NewErroring(addDatasetAction, audit.Unsuccessful) Convey("when datastore getdataset returns an error", func() { r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString("{")) @@ -628,19 +643,19 @@ func TestPostDatasetAuditErrors(t *testing.T) { return nil, errors.New("get dataset error") }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(strings.TrimSpace(w.Body.String()), ShouldEqual, "get dataset error") - + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], addDatasetAction, actionAttempted, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: addDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: addDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) @@ -654,20 +669,20 @@ func TestPostDatasetAuditErrors(t *testing.T) { return &models.DatasetUpdate{}, nil }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 403 status is returned", func() { So(w.Code, ShouldEqual, http.StatusForbidden) - So(strings.TrimSpace(w.Body.String()), ShouldEqual, errs.ErrAddDatasetAlreadyExists.Error()) - + So(w.Body.String(), ShouldContainSubstring, errs.ErrAddDatasetAlreadyExists.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], addDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], addDatasetAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: addDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: addDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) @@ -684,31 +699,25 @@ func TestPostDatasetAuditErrors(t *testing.T) { return errors.New("upsert datset error") }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(strings.TrimSpace(w.Body.String()), ShouldEqual, "upsert datset error") - + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 1) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], addDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], addDatasetAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: addDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: addDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) }) Convey("given audit action successful returns an error", t, func() { - auditor := getMockAuditorFunc(func(a string, r string) error { - if a == addDatasetAction && r == actionSuccessful { - return errors.New("auditing error") - } - return nil - }) + auditMock := audit_mock.NewErroring(addDatasetAction, audit.Successful) Convey("when add dataset is successful", func() { r, err := createRequestWithAuth("POST", "http://localhost:22000/datasets/123", bytes.NewBufferString(datasetPayload)) @@ -723,18 +732,19 @@ func TestPostDatasetAuditErrors(t *testing.T) { return nil }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 201 status is returned", func() { So(w.Code, ShouldEqual, http.StatusCreated) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 1) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], addDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], addDatasetAction, actionSuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: addDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: addDatasetAction, Result: audit.Successful, Params: ap}, + ) }) }) }) @@ -763,24 +773,25 @@ func TestPutDatasetReturnsSuccessfully(t *testing.T) { } mockedDataStore.UpdateDataset("123", dataset, models.CreatedState) - auditor := createAuditor("", "") - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 2) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], putDatasetAction, actionAttempted, common.Params{"dataset_id": "123"}) - verifyAuditRecordCalls(calls[1], putDatasetAction, actionSuccessful, common.Params{"dataset_id": "123"}) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: putDatasetAction, Result: audit.Attempted, Params: common.Params{"dataset_id": "123"}}, + audit_mock.Expected{Action: putDatasetAction, Result: audit.Successful, Params: common.Params{"dataset_id": "123"}}, + ) }) } func TestPutDatasetReturnsError(t *testing.T) { t.Parallel() + ap := common.Params{"dataset_id": "123"} Convey("When the request contain malformed json a bad request status is returned", t, func() { var b string b = "{" @@ -794,25 +805,24 @@ func TestPutDatasetReturnsError(t *testing.T) { return &models.DatasetUpdate{Next: &models.Dataset{}}, nil }, UpdateDatasetFunc: func(string, *models.Dataset, string) error { - return errBadRequest + return errs.ErrAddUpdateDatasetBadRequest }, } - auditor := createAuditor("", "") - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "Failed to parse json body\n") - + So(w.Body.String(), ShouldContainSubstring, errs.ErrUnableToParseJSON.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], putDatasetAction, actionAttempted, common.Params{"dataset_id": "123"}) - verifyAuditRecordCalls(calls[1], putDatasetAction, actionUnsuccessful, common.Params{"dataset_id": "123"}) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: putDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: putDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the api cannot connect to datastore return an internal server error", t, func() { @@ -828,29 +838,29 @@ func TestPutDatasetReturnsError(t *testing.T) { return &models.DatasetUpdate{Next: &models.Dataset{State: models.CreatedState}}, nil }, UpdateDatasetFunc: func(string, *models.Dataset, string) error { - return errInternal + return errs.ErrInternalServer }, } dataset := &models.Dataset{ Title: "CPI", } - mockedDataStore.UpdateDataset("123", dataset, models.CreatedState) - auditor := createAuditor("", "") - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) + mockedDataStore.UpdateDataset("123", dataset, models.CreatedState) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") - + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 2) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], putDatasetAction, actionAttempted, common.Params{"dataset_id": "123"}) - verifyAuditRecordCalls(calls[1], putDatasetAction, actionUnsuccessful, common.Params{"dataset_id": "123"}) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: putDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: putDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the dataset document cannot be found return status not found ", t, func() { @@ -870,23 +880,24 @@ func TestPutDatasetReturnsError(t *testing.T) { }, } - auditor := createAuditor("", "") - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Dataset not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], putDatasetAction, actionAttempted, common.Params{"dataset_id": "123"}) - verifyAuditRecordCalls(calls[1], putDatasetAction, actionUnsuccessful, common.Params{"dataset_id": "123"}) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: putDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: putDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) - Convey("When the request is not authorised to update dataset return status not found", t, func() { + Convey("When the request is not authorised to update dataset return status unauthorised", t, func() { var b string b = "{\"edition\":\"2017\",\"state\":\"created\",\"license\":\"ONS\",\"release_date\":\"2017-04-04\",\"version\":\"1\"}" r, err := http.NewRequest("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(b)) @@ -902,16 +913,19 @@ func TestPutDatasetReturnsError(t *testing.T) { }, } - auditor := createAuditor("", "") - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusUnauthorized) So(w.Body.String(), ShouldResemble, "unauthenticated request\n") So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 0) - So(len(auditor.RecordCalls()), ShouldEqual, 0) + So(len(auditMock.RecordCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 0) + auditMock.AssertRecordCalls() }) } @@ -920,7 +934,7 @@ func TestPutDatasetAuditErrors(t *testing.T) { t.Parallel() Convey("given audit action attempted returns an error", t, func() { - auditor := createAuditor(putDatasetAction, actionAttempted) + auditMock := audit_mock.NewErroring(putDatasetAction, audit.Attempted) Convey("when put dataset is called", func() { r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(datasetPayload)) @@ -936,24 +950,28 @@ func TestPutDatasetAuditErrors(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 1) - verifyAuditRecordCalls(calls[0], putDatasetAction, actionAttempted, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{ + Action: putDatasetAction, + Result: audit.Attempted, + Params: ap, + }, + ) }) }) }) Convey("given audit action successful returns an error", t, func() { - auditor := createAuditor(putDatasetAction, actionSuccessful) + auditMock := audit_mock.NewErroring(putDatasetAction, audit.Successful) Convey("when a put dataset request is successful", func() { r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString(datasetPayload)) @@ -969,24 +987,25 @@ func TestPutDatasetAuditErrors(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 200 status is returned", func() { So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 1) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], putDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], putDatasetAction, actionSuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: putDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: putDatasetAction, Result: audit.Successful, Params: ap}, + ) }) }) }) Convey("given audit action unsuccessful returns an error", t, func() { - auditor := createAuditor(putDatasetAction, actionUnsuccessful) + auditMock := audit_mock.NewErroring(putDatasetAction, audit.Unsuccessful) Convey("when a put dataset request contains an invalid dataset body", func() { r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123", bytes.NewBufferString("`zxcvbnm,./")) @@ -1002,18 +1021,19 @@ func TestPutDatasetAuditErrors(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 400 status is returned", func() { So(w.Code, ShouldEqual, http.StatusBadRequest) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], putDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], putDatasetAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: putDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: putDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) @@ -1028,18 +1048,19 @@ func TestPutDatasetAuditErrors(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 400 status is returned", func() { So(w.Code, ShouldEqual, http.StatusNotFound) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], putDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], putDatasetAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: putDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: putDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) @@ -1063,18 +1084,19 @@ func TestPutDatasetAuditErrors(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 400 status is returned", func() { So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 1) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], putDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], putDatasetAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: putDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: putDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) @@ -1092,18 +1114,19 @@ func TestPutDatasetAuditErrors(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 400 status is returned", func() { So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 1) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], putDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], putDatasetAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: putDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: putDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) }) @@ -1125,20 +1148,20 @@ func TestDeleteDatasetReturnsSuccessfully(t *testing.T) { }, } - auditorMock := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - - calls := auditorMock.RecordCalls() - ap := common.Params{"dataset_id": "123"} + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNoContent) - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], deleteDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], deleteDatasetAction, actionSuccessful, ap) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "123"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Successful, Params: ap}, + ) }) } @@ -1158,20 +1181,20 @@ func TestDeleteDatasetReturnsError(t *testing.T) { }, } - auditorMock := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusForbidden) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 0) - calls := auditorMock.RecordCalls() ap := common.Params{"dataset_id": "123"} - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], deleteDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], deleteDatasetAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the api cannot connect to datastore return an internal server error", t, func() { @@ -1185,26 +1208,24 @@ func TestDeleteDatasetReturnsError(t *testing.T) { return &models.DatasetUpdate{Next: &models.Dataset{State: models.CreatedState}}, nil }, DeleteDatasetFunc: func(string) error { - return errInternal + return errs.ErrInternalServer }, } - auditorMock := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 1) - calls := auditorMock.RecordCalls() ap := common.Params{"dataset_id": "123"} - - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], deleteDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], deleteDatasetAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the dataset document cannot be found return status not found ", t, func() { @@ -1222,21 +1243,20 @@ func TestDeleteDatasetReturnsError(t *testing.T) { }, } - auditorMock := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNoContent) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 0) - calls := auditorMock.RecordCalls() ap := common.Params{"dataset_id": "123"} - - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], deleteDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], deleteDatasetAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the dataset document cannot be queried return status 500 ", t, func() { @@ -1254,20 +1274,20 @@ func TestDeleteDatasetReturnsError(t *testing.T) { }, } - auditorMock := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetCalls()), ShouldEqual, 0) - calls := auditorMock.RecordCalls() ap := common.Params{"dataset_id": "123"} - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], deleteDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], deleteDatasetAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the request is not authorised to delete the dataset return status not found", t, func() { @@ -1278,27 +1298,25 @@ func TestDeleteDatasetReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{} - auditorMock := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusUnauthorized) So(w.Body.String(), ShouldResemble, "unauthenticated request\n") So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 0) - So(len(auditorMock.RecordCalls()), ShouldEqual, 0) + So(len(auditMock.RecordCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 0) + auditMock.AssertRecordCalls() }) } func TestDeleteDatasetAuditActionAttemptedError(t *testing.T) { t.Parallel() Convey("given audit action attempted returns an error", t, func() { - auditorMock := &audit.AuditorServiceMock{ - RecordFunc: func(ctx context.Context, action string, result string, params common.Params) error { - return errors.New("audit error") - }, - } + auditMock := audit_mock.NewErroring(deleteDatasetAction, audit.Attempted) Convey("when delete dataset is called", func() { r, err := createRequestWithAuth("DELETE", "http://localhost:22000/datasets/123", nil) @@ -1306,39 +1324,31 @@ func TestDeleteDatasetAuditActionAttemptedError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{} - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { So(w.Code, ShouldEqual, http.StatusInternalServerError) - - calls := auditorMock.RecordCalls() - ap := common.Params{"dataset_id": "123"} - So(len(calls), ShouldEqual, 1) - verifyAuditRecordCalls(calls[0], deleteDatasetAction, actionAttempted, ap) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "123"} + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{ + Action: deleteDatasetAction, + Result: audit.Attempted, + Params: ap, + }, + ) }) }) }) } -func TestDeleteDatasetAuditActionUnsuccessfulError(t *testing.T) { - getAuditor := func() *audit.AuditorServiceMock { - return &audit.AuditorServiceMock{ - RecordFunc: func(ctx context.Context, action string, result string, params common.Params) error { - if deleteDatasetAction == action && result == actionUnsuccessful { - return errors.New("audit error") - } - return nil - }, - } - } - +func TestDeleteDatasetAuditauditUnsuccessfulError(t *testing.T) { Convey("given auditing action unsuccessful returns an errors", t, func() { - auditorMock := getAuditor() + auditMock := audit_mock.NewErroring(deleteDatasetAction, audit.Unsuccessful) Convey("when attempting to delete a dataset that does not exist", func() { @@ -1351,27 +1361,24 @@ func TestDeleteDatasetAuditActionUnsuccessfulError(t *testing.T) { return nil, errs.ErrDatasetNotFound }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 204 status is returned", func() { So(w.Code, ShouldEqual, http.StatusNoContent) - - calls := auditorMock.RecordCalls() - ap := common.Params{"dataset_id": "123"} - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], deleteDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], deleteDatasetAction, actionUnsuccessful, ap) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "123"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) Convey("when dataStore.Backend.GetDataset returns an error", func() { - auditorMock = getAuditor() - r, err := createRequestWithAuth("DELETE", "http://localhost:22000/datasets/123", nil) So(err, ShouldBeNil) @@ -1381,27 +1388,26 @@ func TestDeleteDatasetAuditActionUnsuccessfulError(t *testing.T) { return nil, errors.New("dataStore.Backend.GetDataset error") }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock = audit_mock.NewErroring(deleteDatasetAction, audit.Unsuccessful) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusInternalServerError) - - calls := auditorMock.RecordCalls() - ap := common.Params{"dataset_id": "123"} - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], deleteDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], deleteDatasetAction, actionUnsuccessful, ap) - + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "123"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) Convey("when attempting to delete a published dataset", func() { - auditorMock = getAuditor() - r, err := createRequestWithAuth("DELETE", "http://localhost:22000/datasets/123", nil) So(err, ShouldBeNil) @@ -1411,27 +1417,26 @@ func TestDeleteDatasetAuditActionUnsuccessfulError(t *testing.T) { return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock = audit_mock.NewErroring(deleteDatasetAction, audit.Unsuccessful) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 403 status is returned", func() { So(w.Code, ShouldEqual, http.StatusForbidden) - - calls := auditorMock.RecordCalls() - ap := common.Params{"dataset_id": "123"} - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], deleteDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], deleteDatasetAction, actionUnsuccessful, ap) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "123"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) Convey("when dataStore.Backend.DeleteDataset returns an error", func() { - auditorMock = getAuditor() - r, err := createRequestWithAuth("DELETE", "http://localhost:22000/datasets/123", nil) So(err, ShouldBeNil) @@ -1444,21 +1449,22 @@ func TestDeleteDatasetAuditActionUnsuccessfulError(t *testing.T) { return errors.New("DeleteDatasetFunc error") }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock = audit_mock.NewErroring(deleteDatasetAction, audit.Unsuccessful) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusInternalServerError) - - calls := auditorMock.RecordCalls() - ap := common.Params{"dataset_id": "123"} - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], deleteDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], deleteDatasetAction, actionUnsuccessful, ap) - + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "123"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) }) @@ -1479,57 +1485,24 @@ func TestDeleteDatasetAuditActionSuccessfulError(t *testing.T) { }, } - auditorMock := &audit.AuditorServiceMock{ - RecordFunc: func(ctx context.Context, action string, result string, params common.Params) error { - if deleteDatasetAction == action && result == actionSuccessful { - return errors.New("audit error") - } - return nil - }, - } + auditMock := audit_mock.NewErroring(deleteDatasetAction, audit.Successful) + Convey("when delete dataset is called", func() { - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 204 status is returned", func() { So(w.Code, ShouldEqual, http.StatusNoContent) - - calls := auditorMock.RecordCalls() - ap := common.Params{"dataset_id": "123"} - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], deleteDatasetAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], deleteDatasetAction, actionSuccessful, ap) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.DeleteDatasetCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "123"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: deleteDatasetAction, Result: audit.Successful, Params: ap}, + ) }) }) }) } - -func getMockAuditor() *audit.AuditorServiceMock { - return &audit.AuditorServiceMock{ - RecordFunc: func(ctx context.Context, action string, result string, params common.Params) error { - return nil - }, - } -} - -func createAuditor(a string, r string) *audit.AuditorServiceMock { - return &audit.AuditorServiceMock{ - RecordFunc: func(ctx context.Context, action string, result string, params common.Params) error { - if action == a && result == r { - return errors.New("auditing error") - } - return nil - }, - } -} - -func getMockAuditorFunc(f func(action string, result string) error) *audit.AuditorServiceMock { - return &audit.AuditorServiceMock{ - RecordFunc: func(ctx context.Context, action string, result string, params common.Params) error { - return f(action, result) - }, - } -} diff --git a/api/dimensions.go b/api/dimensions.go index 6a0b3383..ceda87a5 100644 --- a/api/dimensions.go +++ b/api/dimensions.go @@ -5,14 +5,16 @@ import ( "encoding/json" "fmt" "net/http" + "strconv" errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" + "github.com/ONSdigital/go-ns/audit" "github.com/ONSdigital/go-ns/common" "github.com/ONSdigital/go-ns/log" + "github.com/gedge/mgo/bson" "github.com/gorilla/mux" "github.com/pkg/errors" - "gopkg.in/mgo.v2/bson" ) func (api *DatasetAPI) getDimensions(w http.ResponseWriter, r *http.Request) { @@ -24,9 +26,8 @@ func (api *DatasetAPI) getDimensions(w http.ResponseWriter, r *http.Request) { logData := log.Data{"dataset_id": datasetID, "edition": edition, "version": version} auditParams := common.Params{"dataset_id": datasetID, "edition": edition, "version": version} - if err := api.auditor.Record(ctx, getDimensionsAction, actionAttempted, auditParams); err != nil { - auditActionFailure(ctx, getDimensionsAction, actionAttempted, err, logData) - handleDimensionsErr(ctx, w, err) + if err := api.auditor.Record(ctx, getDimensionsAction, audit.Attempted, auditParams); err != nil { + handleDimensionsErr(ctx, w, err, logData) return } @@ -40,25 +41,25 @@ func (api *DatasetAPI) getDimensions(w http.ResponseWriter, r *http.Request) { versionDoc, err := api.dataStore.Backend.GetVersion(datasetID, edition, version, state) if err != nil { - logError(ctx, errors.WithMessage(err, "getDimensions endpoint: datastore.getversion returned an error"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getDimensions endpoint: datastore.getversion returned an error"), logData) return nil, err } if err = models.CheckState("version", versionDoc.State); err != nil { logData["state"] = versionDoc.State - logError(ctx, errors.WithMessage(err, "getDimensions endpoint: unpublished version has an invalid state"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getDimensions endpoint: unpublished version has an invalid state"), logData) return nil, err } dimensions, err := api.dataStore.Backend.GetDimensions(datasetID, versionDoc.ID) if err != nil { - logError(ctx, errors.WithMessage(err, "getDimensions endpoint: failed to get version dimensions"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getDimensions endpoint: failed to get version dimensions"), logData) return nil, err } results, err := api.createListOfDimensions(versionDoc, dimensions) if err != nil { - logError(ctx, errors.WithMessage(err, "getDimensions endpoint: failed to convert bson to dimension"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getDimensions endpoint: failed to convert bson to dimension"), logData) return nil, err } @@ -66,33 +67,32 @@ func (api *DatasetAPI) getDimensions(w http.ResponseWriter, r *http.Request) { b, err := json.Marshal(listOfDimensions) if err != nil { - logError(ctx, errors.WithMessage(err, "getDimensions endpoint: failed to marshal list of dimension resources into bytes"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getDimensions endpoint: failed to marshal list of dimension resources into bytes"), logData) return nil, err } return b, nil }() - if err != nil { - if auditErr := api.auditor.Record(ctx, getDimensionsAction, actionUnsuccessful, auditParams); auditErr != nil { - auditActionFailure(ctx, getDimensionsAction, actionUnsuccessful, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getDimensionsAction, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr } - handleDimensionsErr(ctx, w, err) + handleDimensionsErr(ctx, w, err, logData) return } - if auditErr := api.auditor.Record(ctx, getDimensionsAction, actionSuccessful, auditParams); auditErr != nil { - auditActionFailure(ctx, getDimensionsAction, actionSuccessful, auditErr, logData) - handleDimensionsErr(ctx, w, auditErr) + if auditErr := api.auditor.Record(ctx, getDimensionsAction, audit.Successful, auditParams); auditErr != nil { + handleDimensionsErr(ctx, w, auditErr, logData) return } setJSONContentType(w) _, err = w.Write(b) if err != nil { - logError(ctx, errors.WithMessage(err, "getDimensions endpoint: error writing bytes to response"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getDimensions endpoint: error writing bytes to response"), logData) http.Error(w, err.Error(), http.StatusInternalServerError) } - logInfo(ctx, "getDimensions endpoint: request successful", logData) + + log.InfoCtx(ctx, "getDimensions endpoint: request successful", logData) } func (api *DatasetAPI) createListOfDimensions(versionDoc *models.Version, dimensions []bson.M) ([]models.Dimension, error) { @@ -142,6 +142,7 @@ func convertBSONToDimensionOption(data interface{}) (*models.DimensionOption, er } func (api *DatasetAPI) getDimensionOptions(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) datasetID := vars["id"] editionID := vars["edition"] @@ -149,43 +150,63 @@ func (api *DatasetAPI) getDimensionOptions(w http.ResponseWriter, r *http.Reques dimension := vars["dimension"] logData := log.Data{"dataset_id": datasetID, "edition": editionID, "version": versionID, "dimension": dimension} + auditParams := common.Params{"dataset_id": datasetID, "edition": editionID, "version": versionID, "dimension": dimension} + + if err := api.auditor.Record(ctx, getDimensionOptionsAction, audit.Attempted, auditParams); err != nil { + handleDimensionsErr(ctx, w, err, logData) + return + } authorised, logData := api.authenticate(r, logData) + auditParams["authorised"] = strconv.FormatBool(authorised) var state string if !authorised { state = models.PublishedState } - version, err := api.dataStore.Backend.GetVersion(datasetID, editionID, versionID, state) - if err != nil { - log.ErrorC("failed to get version", err, logData) - handleErrorType(versionDocType, err, w) - return - } + b, err := func() ([]byte, error) { + version, err := api.dataStore.Backend.GetVersion(datasetID, editionID, versionID, state) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "failed to get version"), logData) + return nil, err + } - if err = models.CheckState("version", version.State); err != nil { - log.ErrorC("unpublished version has an invalid state", err, log.Data{"state": version.State}) - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + if err = models.CheckState("version", version.State); err != nil { + logData["version_state"] = version.State + log.ErrorCtx(ctx, errors.WithMessage(err, "unpublished version has an invalid state"), logData) + return nil, err + } - results, err := api.dataStore.Backend.GetDimensionOptions(version, dimension) + results, err := api.dataStore.Backend.GetDimensionOptions(version, dimension) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "failed to get a list of dimension options"), logData) + return nil, err + } + + for i := range results.Items { + results.Items[i].Links.Version.HRef = fmt.Sprintf("%s/datasets/%s/editions/%s/versions/%s", + api.host, datasetID, editionID, versionID) + } + + b, err := json.Marshal(results) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "failed to marshal list of dimension option resources into bytes"), logData) + return nil, err + } + + return b, nil + }() if err != nil { - log.ErrorC("failed to get a list of dimension options", err, logData) - handleErrorType(dimensionOptionDocType, err, w) + if auditErr := api.auditor.Record(ctx, getDimensionOptionsAction, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr + } + handleDimensionsErr(ctx, w, err, logData) return } - for i := range results.Items { - results.Items[i].Links.Version.HRef = fmt.Sprintf("%s/datasets/%s/editions/%s/versions/%s", - api.host, datasetID, editionID, versionID) - } - - b, err := json.Marshal(results) - if err != nil { - log.ErrorC("failed to marshal list of dimension option resources into bytes", err, logData) - http.Error(w, err.Error(), http.StatusInternalServerError) + if auditErr := api.auditor.Record(ctx, getDimensionOptionsAction, audit.Successful, auditParams); auditErr != nil { + handleDimensionsErr(ctx, w, auditErr, logData) return } @@ -199,22 +220,22 @@ func (api *DatasetAPI) getDimensionOptions(w http.ResponseWriter, r *http.Reques log.Debug("get dimension options", logData) } -func handleDimensionsErr(ctx context.Context, w http.ResponseWriter, err error) { - var responseStatus int +func handleDimensionsErr(ctx context.Context, w http.ResponseWriter, err error, data log.Data) { + if data == nil { + data = log.Data{} + } + var status int + response := err switch { - case err == errs.ErrDatasetNotFound: - responseStatus = http.StatusNotFound - case err == errs.ErrEditionNotFound: - responseStatus = http.StatusNotFound - case err == errs.ErrVersionNotFound: - responseStatus = http.StatusNotFound - case err == errs.ErrDimensionsNotFound: - responseStatus = http.StatusNotFound + case errs.NotFoundMap[err]: + status = http.StatusNotFound default: - responseStatus = http.StatusInternalServerError + status = http.StatusInternalServerError + response = errs.ErrInternalServer } - logError(ctx, errors.WithMessage(err, "request unsuccessful"), log.Data{"responseStatus": responseStatus}) - http.Error(w, err.Error(), responseStatus) + data["response_status"] = status + log.ErrorCtx(ctx, errors.WithMessage(err, "request unsuccessful"), data) + http.Error(w, response.Error(), status) } diff --git a/api/dimensions_test.go b/api/dimensions_test.go index 0afc5d26..d4817378 100644 --- a/api/dimensions_test.go +++ b/api/dimensions_test.go @@ -9,9 +9,11 @@ import ( "github.com/ONSdigital/dp-dataset-api/mocks" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/store/datastoretest" + "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/audit/audit_mock" "github.com/ONSdigital/go-ns/common" + "github.com/gedge/mgo/bson" . "github.com/smartystreets/goconvey/convey" - "gopkg.in/mgo.v2/bson" ) func TestGetDimensionsReturnsOk(t *testing.T) { @@ -28,23 +30,24 @@ func TestGetDimensionsReturnsOk(t *testing.T) { }, } - auditor := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 1) - calls := auditor.RecordCalls() ap := common.Params{ "dataset_id": "123", "edition": "2017", "version": "1", } - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getDimensionsAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getDimensionsAction, actionSuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Successful, Params: ap}, + ) }) } @@ -61,24 +64,24 @@ func TestGetDimensionsReturnsErrors(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetVersionFunc: func(datasetID, edition, version, state string) (*models.Version, error) { - return nil, errInternal + return nil, errs.ErrInternalServer }, } - auditor := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, "internal error\n") - + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getDimensionsAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getDimensionsAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the request contain an invalid version return not found", t, func() { @@ -90,21 +93,20 @@ func TestGetDimensionsReturnsErrors(t *testing.T) { }, } - auditor := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldEqual, "Version not found\n") - + So(w.Body.String(), ShouldContainSubstring, errs.ErrVersionNotFound.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getDimensionsAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getDimensionsAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When there are no dimensions then return not found error", t, func() { @@ -119,20 +121,20 @@ func TestGetDimensionsReturnsErrors(t *testing.T) { }, } - auditor := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldEqual, "Dimensions not found\n") - + So(w.Body.String(), ShouldContainSubstring, errs.ErrDimensionsNotFound.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 1) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getDimensionsAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getDimensionsAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the version has an invalid state return internal server error", t, func() { @@ -144,20 +146,20 @@ func TestGetDimensionsReturnsErrors(t *testing.T) { }, } - auditor := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, "Incorrect resource state\n") - + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getDimensionsAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getDimensionsAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) } @@ -166,30 +168,34 @@ func TestGetDimensionsAuditingErrors(t *testing.T) { ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} Convey("given audit action attempted returns an error", t, func() { - auditor := createAuditor(getDimensionsAction, actionAttempted) + auditMock := audit_mock.NewErroring(getDimensionsAction, audit.Attempted) Convey("when get dimensions is called", func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions", nil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{} - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 1) - verifyAuditRecordCalls(calls[0], getDimensionsAction, actionAttempted, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{ + Action: getDimensionsAction, + Result: audit.Attempted, + Params: ap, + }, + ) }) }) }) Convey("given audit action successful returns an error", t, func() { - auditor := createAuditor(getDimensionsAction, actionSuccessful) + auditMock := audit_mock.NewErroring(getDimensionsAction, audit.Successful) Convey("when get dimensions is called", func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions", nil) @@ -202,25 +208,26 @@ func TestGetDimensionsAuditingErrors(t *testing.T) { return []bson.M{}, nil }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 1) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getDimensionsAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getDimensionsAction, actionSuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Successful, Params: ap}, + ) }) }) }) Convey("given audit action unsuccessful returns an error", t, func() { - auditor := createAuditor(getDimensionsAction, actionUnsuccessful) + auditMock := audit_mock.NewErroring(getDimensionsAction, audit.Unsuccessful) Convey("when datastore.getVersion returns an error", func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions", nil) @@ -230,19 +237,20 @@ func TestGetDimensionsAuditingErrors(t *testing.T) { return nil, errs.ErrVersionNotFound }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusNotFound) + So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getDimensionsAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getDimensionsAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) @@ -254,19 +262,20 @@ func TestGetDimensionsAuditingErrors(t *testing.T) { return &models.Version{State: "BROKEN"}, nil }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 0) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getDimensionsAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getDimensionsAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) @@ -281,19 +290,20 @@ func TestGetDimensionsAuditingErrors(t *testing.T) { return nil, errs.ErrDimensionsNotFound }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusNotFound) + So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 1) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getDimensionsAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getDimensionsAction, actionUnsuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) }) @@ -313,10 +323,20 @@ func TestGetDimensionOptionsReturnsOk(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDimensionOptionsCalls()), ShouldEqual, 1) + + ap := common.Params{"authorised": "false", "dataset_id": "123", "edition": "2017", "version": "1", "dimension": "age"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Successful, Params: ap}, + ) }) } @@ -331,14 +351,21 @@ func TestGetDimensionOptionsReturnsErrors(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldEqual, "Version not found\n") - + So(w.Body.String(), ShouldContainSubstring, errs.ErrVersionNotFound.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionOptionsCalls()), ShouldEqual, 0) + + ap := common.Params{"authorised": "false", "dataset_id": "123", "edition": "2017", "version": "1", "dimension": "age"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When an internal error causes failure to retrieve dimension options, then return internal server error", t, func() { @@ -349,18 +376,25 @@ func TestGetDimensionOptionsReturnsErrors(t *testing.T) { return &models.Version{State: models.AssociatedState}, nil }, GetDimensionOptionsFunc: func(version *models.Version, dimensions string) (*models.DimensionOptionResults, error) { - return nil, errInternal + return nil, errs.ErrInternalServer }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, "internal error\n") - + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionOptionsCalls()), ShouldEqual, 1) + + ap := common.Params{"authorised": "false", "dataset_id": "123", "edition": "2017", "version": "1", "dimension": "age"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the version has an invalid state return internal server error", t, func() { @@ -372,13 +406,175 @@ func TestGetDimensionOptionsReturnsErrors(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, "Incorrect resource state\n") - + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDimensionOptionsCalls()), ShouldEqual, 0) + + ap := common.Params{"authorised": "false", "dataset_id": "123", "edition": "2017", "version": "1", "dimension": "age"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) +} + +func TestGetDimensionOptionsAuditingErrors(t *testing.T) { + t.Parallel() + + Convey("given audit action attempted returns an error", t, func() { + auditMock := audit_mock.NewErroring(getDimensionOptionsAction, audit.Attempted) + + Convey("when get dimensions options is called", func() { + r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions/age/options", nil) + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{} + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + So(len(mockedDataStore.GetDimensionOptionsCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1", "dimension": "age"} + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{ + Action: getDimensionOptionsAction, + Result: audit.Attempted, + Params: ap, + }, + ) + }) + }) + }) + + Convey("given audit action successful returns an error", t, func() { + auditMock := audit_mock.NewErroring(getDimensionOptionsAction, audit.Successful) + + Convey("when get dimension options is called", func() { + r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions/age/options", nil) + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetVersionFunc: func(datasetID, edition, version, state string) (*models.Version, error) { + return &models.Version{State: models.AssociatedState}, nil + }, + GetDimensionOptionsFunc: func(version *models.Version, dimensions string) (*models.DimensionOptionResults, error) { + return &models.DimensionOptionResults{}, nil + }, + } + + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDimensionOptionsCalls()), ShouldEqual, 1) + + ap := common.Params{"authorised": "false", "dataset_id": "123", "edition": "2017", "version": "1", "dimension": "age"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Successful, Params: ap}, + ) + }) + }) + }) + + Convey("given audit action unsuccessful returns an error", t, func() { + auditMock := audit_mock.NewErroring(getDimensionOptionsAction, audit.Unsuccessful) + + Convey("when datastore.getVersion returns an error", func() { + r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions/age/options", nil) + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetVersionFunc: func(datasetID, edition, version, state string) (*models.Version, error) { + return nil, errs.ErrVersionNotFound + }, + } + + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 0) + + ap := common.Params{"authorised": "false", "dataset_id": "123", "edition": "2017", "version": "1", "dimension": "age"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) + }) + + Convey("when the version in not in a valid state", func() { + r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions/age/options", nil) + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetVersionFunc: func(datasetID, edition, version, state string) (*models.Version, error) { + return &models.Version{State: "BROKEN"}, nil + }, + } + + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDimensionsCalls()), ShouldEqual, 0) + + ap := common.Params{"authorised": "false", "dataset_id": "123", "edition": "2017", "version": "1", "dimension": "age"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) + }) + + Convey("when datastore.getDataset returns an error", func() { + r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions/age/options", nil) + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetVersionFunc: func(datasetID, edition, version, state string) (*models.Version, error) { + return &models.Version{State: models.AssociatedState}, nil + }, + GetDimensionOptionsFunc: func(version *models.Version, dimensions string) (*models.DimensionOptionResults, error) { + return nil, errs.ErrDimensionNotFound + }, + } + + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDimensionOptionsCalls()), ShouldEqual, 1) + + ap := common.Params{"authorised": "false", "dataset_id": "123", "edition": "2017", "version": "1", "dimension": "age"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getDimensionOptionsAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) + }) }) } diff --git a/api/editions.go b/api/editions.go index d69347d6..9d508a44 100644 --- a/api/editions.go +++ b/api/editions.go @@ -4,178 +4,184 @@ import ( "encoding/json" "net/http" + errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" + "github.com/ONSdigital/go-ns/audit" "github.com/ONSdigital/go-ns/common" "github.com/ONSdigital/go-ns/log" "github.com/gorilla/mux" + "github.com/pkg/errors" ) func (api *DatasetAPI) getEditions(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) id := vars["id"] logData := log.Data{"dataset_id": id} auditParams := common.Params{"dataset_id": id} - if auditErr := api.auditor.Record(r.Context(), getEditionsAction, actionAttempted, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) + if auditErr := api.auditor.Record(r.Context(), getEditionsAction, audit.Attempted, auditParams); auditErr != nil { + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) return } - authorised, logData := api.authenticate(r, logData) - - var state string - if !authorised { - state = models.PublishedState - } - - logData["state"] = state - log.Info("about to check resources exist", logData) - - if err := api.dataStore.Backend.CheckDatasetExists(id, state); err != nil { - log.ErrorC("unable to find dataset", err, logData) - if auditErr := api.auditor.Record(r.Context(), getEditionsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + b, err := func() ([]byte, error) { + authorised, logData := api.authenticate(r, logData) + var state string + if !authorised { + state = models.PublishedState } - handleErrorType(editionDocType, err, w) - return - } - results, err := api.dataStore.Backend.GetEditions(id, state) - if err != nil { - log.ErrorC("unable to find editions for dataset", err, logData) + logData["state"] = state + log.Info("about to check resources exist", logData) - if auditErr := api.auditor.Record(r.Context(), getEditionsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if err := api.dataStore.Backend.CheckDatasetExists(id, state); err != nil { + audit.LogError(ctx, errors.WithMessage(err, "getEditions endpoint: unable to find dataset"), logData) + return nil, err } - handleErrorType(editionDocType, err, w) - return - } - - var logMessage string - var b []byte - - if authorised { - - // User has valid authentication to get raw edition document - b, err = json.Marshal(results) + results, err := api.dataStore.Backend.GetEditions(id, state) if err != nil { - log.ErrorC("failed to marshal a list of edition resources into bytes", err, logData) - http.Error(w, err.Error(), http.StatusInternalServerError) - return + audit.LogError(ctx, errors.WithMessage(err, "getEditions endpoint: unable to find editions for dataset"), logData) + return nil, err } - logMessage = "get all editions with auth" - } else { + var editionBytes []byte + + if authorised { + + // User has valid authentication to get raw edition document + editionBytes, err = json.Marshal(results) + if err != nil { + audit.LogError(ctx, errors.WithMessage(err, "getEditions endpoint: failed to marshal a list of edition resources into bytes"), logData) + return nil, err + } + audit.LogInfo(ctx, "getEditions endpoint: get all edition with auth", logData) + + } else { + // User is not authenticated and hence has only access to current sub document + var publicResults []*models.Edition + for i := range results.Items { + publicResults = append(publicResults, results.Items[i].Current) + } + + editionBytes, err = json.Marshal(&models.EditionResults{Items: publicResults}) + if err != nil { + audit.LogError(ctx, errors.WithMessage(err, "getEditions endpoint: failed to marshal a list of edition resources into bytes"), logData) + return nil, err + } + audit.LogInfo(ctx, "getEditions endpoint: get all edition without auth", logData) + } + return editionBytes, nil + }() - // User is not authenticated and hance has only access to current sub document - var publicResults []*models.Edition - for i := range results.Items { - publicResults = append(publicResults, results.Items[i].Current) + if err != nil { + if auditErr := api.auditor.Record(ctx, getEditionsAction, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr } - b, err = json.Marshal(&models.EditionResults{Items: publicResults}) - if err != nil { - log.ErrorC("failed to marshal a list of public edition resources into bytes", err, logData) - http.Error(w, err.Error(), http.StatusInternalServerError) - return + if err == errs.ErrDatasetNotFound || err == errs.ErrEditionNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) } - logMessage = "get all editions without auth" + return } - if auditErr := api.auditor.Record(r.Context(), getEditionsAction, actionSuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) + if auditErr := api.auditor.Record(r.Context(), getEditionsAction, audit.Successful, auditParams); auditErr != nil { + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) return } setJSONContentType(w) _, err = w.Write(b) if err != nil { - log.Error(err, logData) - http.Error(w, err.Error(), http.StatusInternalServerError) + audit.LogError(ctx, errors.WithMessage(err, "getEditions endpoint: failed writing bytes to response"), logData) + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) } - log.Debug(logMessage, log.Data{"dataset_id": id}) + audit.LogInfo(ctx, "getEditions endpoint: request successful", logData) } func (api *DatasetAPI) getEdition(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) id := vars["id"] editionID := vars["edition"] - logData := log.Data{"dataset_id": id, "edition": editionID} auditParams := common.Params{"dataset_id": id, "edition": editionID} + logData := audit.ToLogData(auditParams) - if auditErr := api.auditor.Record(r.Context(), getEditionAction, actionAttempted, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) + if auditErr := api.auditor.Record(r.Context(), getEditionAction, audit.Attempted, auditParams); auditErr != nil { + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) return } - authorised, logData := api.authenticate(r, logData) + b, err := func() ([]byte, error) { + authorised, logData := api.authenticate(r, logData) - var state string - if !authorised { - state = models.PublishedState - } - - if err := api.dataStore.Backend.CheckDatasetExists(id, state); err != nil { - log.ErrorC("unable to find dataset", err, logData) - if auditErr := api.auditor.Record(r.Context(), getEditionAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + var state string + if !authorised { + state = models.PublishedState } - handleErrorType(editionDocType, err, w) - return - } - edition, err := api.dataStore.Backend.GetEdition(id, editionID, state) - if err != nil { - log.ErrorC("unable to find edition", err, logData) - if auditErr := api.auditor.Record(r.Context(), getEditionAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if err := api.dataStore.Backend.CheckDatasetExists(id, state); err != nil { + audit.LogError(ctx, errors.WithMessage(err, "getEdition endpoint: unable to find dataset"), logData) + return nil, err } - handleErrorType(editionDocType, err, w) - return - } - - var logMessage string - var b []byte - if authorised { - - // User has valid authentication to get raw edition document - b, err = json.Marshal(edition) + edition, err := api.dataStore.Backend.GetEdition(id, editionID, state) if err != nil { - log.ErrorC("failed to marshal edition resource into bytes", err, logData) - http.Error(w, err.Error(), http.StatusInternalServerError) - return + audit.LogError(ctx, errors.WithMessage(err, "getEdition endpoint: unable to find edition"), logData) + return nil, err } - logMessage = "get edition with auth" - } else { + var b []byte + + if authorised { + // User has valid authentication to get raw edition document + b, err = json.Marshal(edition) + if err != nil { + audit.LogError(ctx, errors.WithMessage(err, "getEdition endpoint: failed to marshal edition resource into bytes"), logData) + return nil, err + } + audit.LogInfo(ctx, "getEdition endpoint: get edition with auth", logData) + } else { + + // User is not authenticated and hence has only access to current sub document + b, err = json.Marshal(edition.Current) + if err != nil { + audit.LogError(ctx, errors.WithMessage(err, "getEdition endpoint: failed to marshal edition resource into bytes"), logData) + return nil, err + } + audit.LogInfo(ctx, "getEdition endpoint: get edition without auth", logData) + } + return b, nil + }() - // User is not authenticated and hance has only access to current sub document - b, err = json.Marshal(edition.Current) - if err != nil { - log.ErrorC("failed to marshal public edition resource into bytes", err, logData) - http.Error(w, err.Error(), http.StatusInternalServerError) - return + if err != nil { + if auditErr := api.auditor.Record(ctx, getEditionAction, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr } - logMessage = "get public edition without auth" + + if err == errs.ErrDatasetNotFound || err == errs.ErrEditionNotFound { + http.Error(w, err.Error(), http.StatusNotFound) + } else { + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) + } + return } - if auditErr := api.auditor.Record(r.Context(), getEditionAction, actionSuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getEditionAction, audit.Successful, auditParams); auditErr != nil { + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) return } setJSONContentType(w) _, err = w.Write(b) if err != nil { - log.Error(err, logData) - http.Error(w, err.Error(), http.StatusInternalServerError) + audit.LogError(ctx, errors.WithMessage(err, "getEdition endpoint: failed to write byte to response"), logData) + http.Error(w, errs.ErrInternalServer.Error(), http.StatusInternalServerError) + return } - log.Debug(logMessage, logData) + audit.LogInfo(ctx, "getEdition endpoint: request successful", logData) } diff --git a/api/editions_test.go b/api/editions_test.go index 958ff690..05521cb1 100644 --- a/api/editions_test.go +++ b/api/editions_test.go @@ -11,6 +11,8 @@ import ( "github.com/ONSdigital/dp-dataset-api/mocks" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/store/datastoretest" + "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/audit/audit_mock" "github.com/ONSdigital/go-ns/common" . "github.com/smartystreets/goconvey/convey" ) @@ -34,18 +36,19 @@ func TestGetEditionsReturnsOK(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionsAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getEditionsAction, actionSuccessful, auditParams) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionsAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getEditionsAction, Result: audit.Successful, Params: genericAuditParams}, + ) }) } @@ -63,22 +66,25 @@ func TestGetEditionsAuditingError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { return errors.New("get editions action attempted audit event error") } api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, internalServerErr) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 1) - verifyAuditRecordCalls(recCalls[0], getEditionsAction, actionAttempted, auditParams) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetEditionsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{ + Action: getEditionsAction, + Result: audit.Attempted, + Params: genericAuditParams, + }, + ) }) Convey("given auditing get editions action successful returns an error then a 500 response is returned", t, func() { @@ -93,27 +99,19 @@ func TestGetEditionsAuditingError(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getEditionsAction && result == actionSuccessful { - return errors.New("audit error") - } - return nil - } - + auditMock := audit_mock.NewErroring(getEditionsAction, audit.Successful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, internalServerErr) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionsAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getEditionsAction, actionSuccessful, auditParams) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionsAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getEditionsAction, Result: audit.Successful, Params: genericAuditParams}, + ) }) Convey("When the dataset does not exist and auditing the action result fails then return status 500", t, func() { @@ -126,27 +124,19 @@ func TestGetEditionsAuditingError(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getEditionsAction && result == actionUnsuccessful { - return errors.New(auditError) - } - return nil - } + auditMock := audit_mock.NewErroring(getEditionsAction, audit.Unsuccessful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionsAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getEditionsAction, actionUnsuccessful, auditParams) - + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionsAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getEditionsAction, Result: audit.Unsuccessful, Params: genericAuditParams}, + ) }) Convey("When no published editions exist against a published dataset and auditing unsuccessful errors return status 500", t, func() { @@ -161,27 +151,19 @@ func TestGetEditionsAuditingError(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getEditionsAction && result == actionUnsuccessful { - return errors.New(auditError) - } - return nil - } + auditMock := audit_mock.NewErroring(getEditionsAction, audit.Unsuccessful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, internalServerErr) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionsAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getEditionsAction, actionUnsuccessful, auditParams) - + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionsAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getEditionsAction, Result: audit.Unsuccessful, Params: genericAuditParams}, + ) }) } @@ -192,25 +174,24 @@ func TestGetEditionsReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(datasetID, state string) error { - return errInternal + return errs.ErrInternalServer }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionsAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getEditionsAction, actionUnsuccessful, auditParams) - + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionsAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getEditionsAction, Result: audit.Unsuccessful, Params: genericAuditParams}, + ) }) Convey("When the dataset does not exist return status not found", t, func() { @@ -223,21 +204,20 @@ func TestGetEditionsReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Dataset not found\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionsAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getEditionsAction, actionUnsuccessful, auditParams) - + So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionsAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getEditionsAction, Result: audit.Unsuccessful, Params: genericAuditParams}, + ) }) Convey("When no editions exist against an existing dataset return status not found", t, func() { @@ -253,21 +233,20 @@ func TestGetEditionsReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Edition not found\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionsAction, actionAttempted, auditParams) - verifyAuditRecordCalls(recCalls[1], getEditionsAction, actionUnsuccessful, auditParams) - + So(w.Body.String(), ShouldContainSubstring, errs.ErrEditionNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionsAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getEditionsAction, Result: audit.Unsuccessful, Params: genericAuditParams}, + ) }) Convey("When no published editions exist against a published dataset return status not found", t, func() { @@ -282,14 +261,20 @@ func TestGetEditionsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Edition not found\n") - + So(w.Body.String(), ShouldContainSubstring, errs.ErrEditionNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionsAction, Result: audit.Attempted, Params: genericAuditParams}, + audit_mock.Expected{Action: getEditionsAction, Result: audit.Unsuccessful, Params: genericAuditParams}, + ) }) } @@ -307,20 +292,20 @@ func TestGetEditionReturnsOK(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - p := common.Params{"dataset_id": "123-456", "edition": "678"} + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getEditionAction, actionSuccessful, p) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionCalls()), ShouldEqual, 1) + + p := common.Params{"dataset_id": "123-456", "edition": "678"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getEditionAction, Result: audit.Successful, Params: p}, + ) }) } @@ -331,27 +316,25 @@ func TestGetEditionReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(datasetID, state string) error { - return errInternal + return errs.ErrInternalServer }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - p := common.Params{"dataset_id": "123-456", "edition": "678"} + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") - - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getEditionAction, actionUnsuccessful, p) - + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionCalls()), ShouldEqual, 0) + + p := common.Params{"dataset_id": "123-456", "edition": "678"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getEditionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When the dataset does not exist return status not found", t, func() { @@ -364,23 +347,21 @@ func TestGetEditionReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - p := common.Params{"dataset_id": "123-456", "edition": "678"} + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Dataset not found\n") - - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getEditionAction, actionUnsuccessful, p) - + So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionCalls()), ShouldEqual, 0) + + p := common.Params{"dataset_id": "123-456", "edition": "678"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getEditionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When edition does not exist for a dataset return status not found", t, func() { @@ -396,22 +377,21 @@ func TestGetEditionReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - p := common.Params{"dataset_id": "123-456", "edition": "678"} + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Edition not found\n") - - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getEditionAction, actionUnsuccessful, p) + So(w.Body.String(), ShouldContainSubstring, errs.ErrEditionNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionCalls()), ShouldEqual, 1) + + p := common.Params{"dataset_id": "123-456", "edition": "678"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getEditionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When edition is not published for a dataset return status not found", t, func() { @@ -426,21 +406,21 @@ func TestGetEditionReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - p := common.Params{"dataset_id": "123-456", "edition": "678"} + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Edition not found\n") - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getEditionAction, actionUnsuccessful, p) + So(w.Body.String(), ShouldContainSubstring, errs.ErrEditionNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionCalls()), ShouldEqual, 1) + + p := common.Params{"dataset_id": "123-456", "edition": "678"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getEditionAction, Result: audit.Unsuccessful, Params: p}, + ) }) } @@ -451,23 +431,22 @@ func TestGetEditionAuditErrors(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{} - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - return errors.New("auditing error") - } + auditMock := audit_mock.NewErroring(getEditionAction, audit.Attempted) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - p := common.Params{"dataset_id": "123-456", "edition": "678"} - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - So(len(recCalls), ShouldEqual, 1) - verifyAuditRecordCalls(recCalls[0], getEditionAction, actionAttempted, p) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetEditionCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{ + Action: getEditionAction, + Result: audit.Attempted, + Params: common.Params{"dataset_id": "123-456", "edition": "678"}, + }, + ) }) Convey("when check dataset exists errors and auditing action unsuccessful errors then a 500 status is returned", t, func() { @@ -479,27 +458,20 @@ func TestGetEditionAuditErrors(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getEditionAction && result == actionUnsuccessful { - return errors.New("auditing error") - } - return nil - } + auditMock := audit_mock.NewErroring(getEditionAction, audit.Unsuccessful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - p := common.Params{"dataset_id": "123-456", "edition": "678"} - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getEditionAction, actionUnsuccessful, p) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionCalls()), ShouldEqual, 0) + + p := common.Params{"dataset_id": "123-456", "edition": "678"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getEditionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("when check edition exists errors and auditing action unsuccessful errors then a 500 status is returned", t, func() { @@ -515,27 +487,20 @@ func TestGetEditionAuditErrors(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getEditionAction && result == actionUnsuccessful { - return errors.New("auditing error") - } - return nil - } + auditMock := audit_mock.NewErroring(getEditionAction, audit.Unsuccessful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - p := common.Params{"dataset_id": "123-456", "edition": "678"} - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getEditionAction, actionUnsuccessful, p) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionCalls()), ShouldEqual, 1) + + p := common.Params{"dataset_id": "123-456", "edition": "678"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getEditionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("when get edition audit even successful errors then a 500 status is returned", t, func() { @@ -550,27 +515,19 @@ func TestGetEditionAuditErrors(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getEditionAction && result == actionSuccessful { - return errors.New("error") - } - return nil - } - + auditMock := audit_mock.NewErroring(getEditionAction, audit.Successful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - p := common.Params{"dataset_id": "123-456", "edition": "678"} - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getEditionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getEditionAction, actionSuccessful, p) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetEditionCalls()), ShouldEqual, 1) + + p := common.Params{"dataset_id": "123-456", "edition": "678"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getEditionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getEditionAction, Result: audit.Successful, Params: p}, + ) }) } diff --git a/api/metadata.go b/api/metadata.go index f3b1f1fb..69b0317a 100644 --- a/api/metadata.go +++ b/api/metadata.go @@ -6,6 +6,7 @@ import ( errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" + "github.com/ONSdigital/go-ns/audit" "github.com/ONSdigital/go-ns/common" "github.com/ONSdigital/go-ns/log" "github.com/gorilla/mux" @@ -18,11 +19,10 @@ func (api *DatasetAPI) getMetadata(w http.ResponseWriter, r *http.Request) { datasetID := vars["id"] edition := vars["edition"] version := vars["version"] - logData := log.Data{"dataset_id": datasetID, "edition": edition, "version": version} auditParams := common.Params{"dataset_id": datasetID, "edition": edition, "version": version} + logData := audit.ToLogData(auditParams) - if auditErr := api.auditor.Record(ctx, getMetadataAction, actionAttempted, auditParams); auditErr != nil { - auditActionFailure(ctx, getMetadataAction, actionAttempted, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getMetadataAction, audit.Attempted, auditParams); auditErr != nil { handleMetadataErr(w, auditErr) return } @@ -30,7 +30,7 @@ func (api *DatasetAPI) getMetadata(w http.ResponseWriter, r *http.Request) { b, err := func() ([]byte, error) { datasetDoc, err := api.dataStore.Backend.GetDataset(datasetID) if err != nil { - logError(ctx, errors.WithMessage(err, "getMetadata endpoint: get datastore.getDataset returned an error"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getMetadata endpoint: get datastore.getDataset returned an error"), logData) return nil, err } @@ -42,7 +42,7 @@ func (api *DatasetAPI) getMetadata(w http.ResponseWriter, r *http.Request) { // Check for current sub document if datasetDoc.Current == nil || datasetDoc.Current.State != models.PublishedState { logData["dataset"] = datasetDoc.Current - logError(ctx, errors.New("getMetadata endpoint: caller not is authorised and dataset but currently unpublished"), logData) + log.ErrorCtx(ctx, errors.New("getMetadata endpoint: caller not is authorised and dataset but currently unpublished"), logData) return nil, errs.ErrDatasetNotFound } @@ -50,19 +50,19 @@ func (api *DatasetAPI) getMetadata(w http.ResponseWriter, r *http.Request) { } if err = api.dataStore.Backend.CheckEditionExists(datasetID, edition, state); err != nil { - logError(ctx, errors.WithMessage(err, "getMetadata endpoint: failed to find edition for dataset"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getMetadata endpoint: failed to find edition for dataset"), logData) return nil, err } versionDoc, err := api.dataStore.Backend.GetVersion(datasetID, edition, version, state) if err != nil { - logError(ctx, errors.WithMessage(err, "getMetadata endpoint: failed to find version for dataset edition"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getMetadata endpoint: failed to find version for dataset edition"), logData) return nil, errs.ErrMetadataVersionNotFound } if err = models.CheckState("version", versionDoc.State); err != nil { logData["state"] = versionDoc.State - logError(ctx, errors.WithMessage(err, "getMetadata endpoint: unpublished version has an invalid state"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getMetadata endpoint: unpublished version has an invalid state"), logData) return nil, err } @@ -76,33 +76,32 @@ func (api *DatasetAPI) getMetadata(w http.ResponseWriter, r *http.Request) { b, err := json.Marshal(metaDataDoc) if err != nil { - logError(ctx, errors.WithMessage(err, "getMetadata endpoint: failed to marshal metadata resource into bytes"), logData) + log.ErrorCtx(ctx, errors.WithMessage(err, "getMetadata endpoint: failed to marshal metadata resource into bytes"), logData) return nil, err } return b, err }() if err != nil { - if auditErr := api.auditor.Record(ctx, getMetadataAction, actionUnsuccessful, auditParams); auditErr != nil { - auditActionFailure(ctx, getMetadataAction, actionUnsuccessful, auditErr, logData) + log.ErrorCtx(ctx, err, logData) + if auditErr := api.auditor.Record(ctx, getMetadataAction, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr } handleMetadataErr(w, err) return } - if auditErr := api.auditor.Record(ctx, getMetadataAction, actionSuccessful, auditParams); auditErr != nil { - auditActionFailure(ctx, getMetadataAction, actionSuccessful, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getMetadataAction, audit.Successful, auditParams); auditErr != nil { handleMetadataErr(w, auditErr) return } setJSONContentType(w) - _, err = w.Write(b) - if err != nil { - logError(ctx, errors.WithMessage(err, "getMetadata endpoint: failed to write bytes to response"), logData) + if _, err = w.Write(b); err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "getMetadata endpoint: failed to write bytes to response"), logData) http.Error(w, err.Error(), http.StatusInternalServerError) } - logInfo(ctx, "getMetadata endpoint: get metadata request successful", logData) + log.InfoCtx(ctx, "getMetadata endpoint: get metadata request successful", logData) } func handleMetadataErr(w http.ResponseWriter, err error) { @@ -116,6 +115,7 @@ func handleMetadataErr(w http.ResponseWriter, err error) { case err == errs.ErrDatasetNotFound: responseStatus = http.StatusNotFound default: + err = errs.ErrInternalServer responseStatus = http.StatusInternalServerError } diff --git a/api/metadata_test.go b/api/metadata_test.go index dccac445..d71422ec 100644 --- a/api/metadata_test.go +++ b/api/metadata_test.go @@ -12,8 +12,9 @@ import ( "github.com/ONSdigital/dp-dataset-api/mocks" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/store/datastoretest" + "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/audit/audit_mock" "github.com/ONSdigital/go-ns/common" - "github.com/pkg/errors" . "github.com/smartystreets/goconvey/convey" ) @@ -39,20 +40,21 @@ func TestGetMetadataReturnsOk(t *testing.T) { }, } - auditor := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - calls := auditor.RecordCalls() ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getMetadataAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getMetadataAction, actionSuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Successful, Params: ap}, + ) bytes, err := ioutil.ReadAll(w.Body) if err != nil { @@ -102,20 +104,21 @@ func TestGetMetadataReturnsOk(t *testing.T) { }, } - auditor := getMockAuditor() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - calls := auditor.RecordCalls() ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getMetadataAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getMetadataAction, actionSuccessful, ap) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Successful, Params: ap}, + ) bytes, err := ioutil.ReadAll(w.Body) if err != nil { @@ -151,18 +154,26 @@ func TestGetMetadataReturnsError(t *testing.T) { mockedDataStore := &storetest.StorerMock{ GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return nil, errInternal + return nil, errs.ErrInternalServer }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, "internal error\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the dataset document cannot be found for version return status not found", t, func() { @@ -176,14 +187,22 @@ func TestGetMetadataReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldEqual, "Dataset not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the dataset document has no current sub document return status not found", t, func() { @@ -203,14 +222,22 @@ func TestGetMetadataReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldEqual, "Dataset not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the edition document cannot be found for version return status not found", t, func() { @@ -229,15 +256,23 @@ func TestGetMetadataReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldEqual, "Edition not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrEditionNotFound.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the version document cannot be found return status not found", t, func() { @@ -259,15 +294,23 @@ func TestGetMetadataReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldEqual, "Version not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrVersionNotFound.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the version document state is invalid return an internal server error", t, func() { @@ -289,15 +332,21 @@ func TestGetMetadataReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, "Incorrect resource state\n") + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Unsuccessful, Params: ap}, + ) }) } @@ -305,43 +354,36 @@ func TestGetMetadataAuditingErrors(t *testing.T) { ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} Convey("given auditing action attempted returns an error", t, func() { - auditor := getMockAuditorFunc(func(a string, r string) error { - if a == getMetadataAction && r == actionAttempted { - return errors.New("audit error") - } - return nil - }) + auditMock := audit_mock.NewErroring(getMetadataAction, audit.Attempted) Convey("when get metadata is called", func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/metadata", nil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{} - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { So(w.Code, ShouldEqual, http.StatusInternalServerError) - - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 1) - verifyAuditRecordCalls(calls[0], getMetadataAction, actionAttempted, ap) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{ + Action: getMetadataAction, + Result: audit.Attempted, + Params: ap, + }, + ) }) }) }) Convey("given auditing action unsuccessful returns an error", t, func() { - auditor := getMockAuditorFunc(func(a string, r string) error { - if a == getMetadataAction && r == actionUnsuccessful { - return errors.New("audit error") - } - return nil - }) + auditMock := audit_mock.NewErroring(getMetadataAction, audit.Unsuccessful) Convey("when datastore getDataset returns dataset not found error", func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/metadata", nil) @@ -352,21 +394,21 @@ func TestGetMetadataAuditingErrors(t *testing.T) { return nil, errs.ErrDatasetNotFound }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - - Convey("then a 404 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusNotFound) - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getMetadataAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getMetadataAction, actionUnsuccessful, ap) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + Convey("then a 500 status is returned", func() { + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) @@ -379,21 +421,21 @@ func TestGetMetadataAuditingErrors(t *testing.T) { return &models.DatasetUpdate{}, nil }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - Convey("then a 404 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusNotFound) - - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getMetadataAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getMetadataAction, actionUnsuccessful, ap) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + Convey("then a 500 status is returned", func() { + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) @@ -409,21 +451,21 @@ func TestGetMetadataAuditingErrors(t *testing.T) { return errs.ErrEditionNotFound }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) - - Convey("then a 404 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusNotFound) - - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getMetadataAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getMetadataAction, actionUnsuccessful, ap) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + Convey("then a 500 status is returned", func() { + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) @@ -442,21 +484,21 @@ func TestGetMetadataAuditingErrors(t *testing.T) { return nil, errs.ErrVersionNotFound }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusNotFound) - - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getMetadataAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getMetadataAction, actionUnsuccessful, ap) - + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) @@ -477,32 +519,27 @@ func TestGetMetadataAuditingErrors(t *testing.T) { return versionDoc, nil }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { - So(w.Code, ShouldEqual, http.StatusInternalServerError) - - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getMetadataAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getMetadataAction, actionUnsuccessful, ap) - + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) }) Convey("given auditing action successful returns an error", t, func() { - auditor := getMockAuditorFunc(func(a string, r string) error { - if a == getMetadataAction && r == actionSuccessful { - return errors.New("audit error") - } - return nil - }) + auditMock := audit_mock.NewErroring(getMetadataAction, audit.Successful) Convey("when get metadata is called", func() { r := httptest.NewRequest("GET", "http://localhost:22000/datasets/123/editions/2017/versions/1/metadata", nil) @@ -519,26 +556,24 @@ func TestGetMetadataAuditingErrors(t *testing.T) { return createVersionDoc(), nil }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a 500 status is returned", func() { So(w.Code, ShouldEqual, http.StatusInternalServerError) - - calls := auditor.RecordCalls() - So(len(calls), ShouldEqual, 2) - verifyAuditRecordCalls(calls[0], getMetadataAction, actionAttempted, ap) - verifyAuditRecordCalls(calls[1], getMetadataAction, actionSuccessful, ap) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getMetadataAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getMetadataAction, Result: audit.Successful, Params: ap}, + ) }) }) - }) - } // createDatasetDoc returns a datasetUpdate doc containing minimal fields but diff --git a/api/observation.go b/api/observation.go index 0b35b428..ddfdb8af 100644 --- a/api/observation.go +++ b/api/observation.go @@ -1,6 +1,7 @@ package api import ( + "context" "encoding/csv" "encoding/json" "fmt" @@ -13,9 +14,11 @@ import ( errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-filter/observation" + "github.com/ONSdigital/go-ns/audit" "github.com/ONSdigital/go-ns/common" "github.com/ONSdigital/go-ns/log" "github.com/gorilla/mux" + "github.com/pkg/errors" ) //go:generate moq -out ../mocks/observation_store.go -pkg mocks . ObservationStore @@ -29,16 +32,43 @@ const ( getObservationsAction = "getObservations" ) +var ( + observationNotFound = map[error]bool{ + errs.ErrDatasetNotFound: true, + errs.ErrEditionNotFound: true, + errs.ErrVersionNotFound: true, + errs.ErrObservationsNotFound: true, + } + + observationBadRequest = map[error]bool{ + errs.ErrTooManyWildcards: true, + } +) + +type observationQueryError struct { + message string +} + +func (e observationQueryError) Error() string { + return e.message +} + func errorIncorrectQueryParameters(params []string) error { - return fmt.Errorf("Incorrect selection of query parameters: %v, these dimensions do not exist for this version of the dataset", params) + return observationQueryError{ + message: fmt.Sprintf("incorrect selection of query parameters: %v, these dimensions do not exist for this version of the dataset", params), + } } func errorMissingQueryParameters(params []string) error { - return fmt.Errorf("Missing query parameters for the following dimensions: %v", params) + return observationQueryError{ + message: fmt.Sprintf("missing query parameters for the following dimensions: %v", params), + } } func errorMultivaluedQueryParameters(params []string) error { - return fmt.Errorf("Multi-valued query parameters for the following dimensions: %v", params) + return observationQueryError{ + message: fmt.Sprintf("multi-valued query parameters for the following dimensions: %v", params), + } } // ObservationStore provides filtered observation data in CSV rows. @@ -47,145 +77,111 @@ type ObservationStore interface { } func (api *DatasetAPI) getObservations(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) datasetID := vars["id"] edition := vars["edition"] version := vars["version"] - logData := log.Data{"dataset_id": datasetID, "edition": edition, "version": version} auditParams := common.Params{"dataset_id": datasetID, "edition": edition, "version": version} + logData := audit.ToLogData(auditParams) - if auditErr := api.auditor.Record(r.Context(), getObservationsAction, actionAttempted, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getObservationsAction, audit.Attempted, auditParams); auditErr != nil { + handleObservationsErrorType(ctx, w, auditErr, logData) return } - // get dataset document - datasetDoc, err := api.dataStore.Backend.GetDataset(datasetID) - if err != nil { - log.Error(err, logData) - if auditErr := api.auditor.Record(r.Context(), getObservationsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + observationsDoc, err := func() (*models.ObservationsDoc, error) { + // get dataset document + datasetDoc, err := api.dataStore.Backend.GetDataset(datasetID) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "get observations: datastore.GetDataset returned an error"), logData) + return nil, err } - handleObservationsErrorType(w, err) - return - } - authorised, logData := api.authenticate(r, logData) - - var ( - state string - dataset *models.Dataset - ) - - // if request is not authenticated then only access resources of state published - if !authorised { - // Check for current sub document - if datasetDoc.Current == nil || datasetDoc.Current.State != models.PublishedState { - logData["dataset_doc"] = datasetDoc.Current - log.ErrorC("found no published dataset", errs.ErrDatasetNotFound, logData) - if auditErr := api.auditor.Record(r.Context(), getObservationsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + authorised, logData := api.authenticate(r, logData) + + var ( + state string + dataset *models.Dataset + ) + + // if request is not authenticated then only access resources of state published + if !authorised { + // Check for current sub document + if datasetDoc.Current == nil || datasetDoc.Current.State != models.PublishedState { + logData["dataset_doc"] = datasetDoc.Current + log.ErrorCtx(ctx, errors.WithMessage(errs.ErrDatasetNotFound, "get observations: found no published dataset"), logData) + return nil, errs.ErrDatasetNotFound } - http.Error(w, errs.ErrDatasetNotFound.Error(), http.StatusNotFound) - return - } - dataset = datasetDoc.Current - state = dataset.State - } else { - dataset = datasetDoc.Next - } + dataset = datasetDoc.Current + state = dataset.State + } else { + dataset = datasetDoc.Next + } - if err = api.dataStore.Backend.CheckEditionExists(datasetID, edition, state); err != nil { - log.ErrorC("failed to find edition for dataset", err, logData) - if auditErr := api.auditor.Record(r.Context(), getObservationsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if err = api.dataStore.Backend.CheckEditionExists(datasetID, edition, state); err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "get observations: failed to find edition for dataset"), logData) + return nil, err } - handleObservationsErrorType(w, err) - return - } - versionDoc, err := api.dataStore.Backend.GetVersion(datasetID, edition, version, state) - if err != nil { - log.ErrorC("failed to find version for dataset edition", err, logData) - if auditErr := api.auditor.Record(r.Context(), getObservationsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + versionDoc, err := api.dataStore.Backend.GetVersion(datasetID, edition, version, state) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "get observations: failed to find version for dataset edition"), logData) + return nil, err } - handleObservationsErrorType(w, err) - return - } - if err = models.CheckState("version", versionDoc.State); err != nil { - logData["state"] = versionDoc.State - log.ErrorC("unpublished version has an invalid state", err, logData) - if auditErr := api.auditor.Record(r.Context(), getObservationsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if err = models.CheckState("version", versionDoc.State); err != nil { + logData["state"] = versionDoc.State + log.ErrorCtx(ctx, errors.WithMessage(err, "get observations: unpublished version has an invalid state"), logData) + return nil, err } - handleObservationsErrorType(w, err) - return - } - if versionDoc.Headers == nil || versionDoc.Dimensions == nil { - logData["version_doc"] = versionDoc - log.Error(errs.ErrMissingVersionHeadersOrDimensions, logData) - if auditErr := api.auditor.Record(r.Context(), getObservationsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if versionDoc.Headers == nil || versionDoc.Dimensions == nil { + logData["version_doc"] = versionDoc + log.ErrorCtx(ctx, errors.WithMessage(errs.ErrMissingVersionHeadersOrDimensions, "get observations"), logData) + return nil, errs.ErrMissingVersionHeadersOrDimensions } - http.Error(w, "", http.StatusInternalServerError) - return - } - // loop through version dimensions to retrieve list of dimension names - validDimensionNames := getListOfValidDimensionNames(versionDoc.Dimensions) - logData["version_dimensions"] = validDimensionNames + // loop through version dimensions to retrieve list of dimension names + validDimensionNames := getListOfValidDimensionNames(versionDoc.Dimensions) + logData["version_dimensions"] = validDimensionNames - dimensionOffset, err := getDimensionOffsetInHeaderRow(versionDoc.Headers) - if err != nil { - log.ErrorC("unable to distinguish headers from version document", err, logData) - if auditErr := api.auditor.Record(r.Context(), getObservationsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + dimensionOffset, err := getDimensionOffsetInHeaderRow(versionDoc.Headers) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "get observations: unable to distinguish headers from version document"), logData) + return nil, err } - handleObservationsErrorType(w, err) - return - } - // check query parameters match the version headers - queryParameters, err := extractQueryParameters(r.URL.Query(), validDimensionNames) - if err != nil { - log.Error(err, logData) - if auditErr := api.auditor.Record(r.Context(), getObservationsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + // check query parameters match the version headers + queryParameters, err := extractQueryParameters(r.URL.Query(), validDimensionNames) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "get observations: error extracting query parameters"), logData) + return nil, err } - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - logData["query_parameters"] = queryParameters + logData["query_parameters"] = queryParameters + + // retrieve observations + observations, err := api.getObservationList(versionDoc, queryParameters, defaultObservationLimit, dimensionOffset, logData) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "get observations: unable to retrieve observations"), logData) + return nil, err + } + + return models.CreateObservationsDoc(r.URL.RawQuery, versionDoc, dataset, observations, queryParameters, defaultOffset, defaultObservationLimit), nil + }() - // retrieve observations - observations, err := api.getObservationList(versionDoc, queryParameters, defaultObservationLimit, dimensionOffset, logData) if err != nil { - log.ErrorC("unable to retrieve observations", err, logData) - if auditErr := api.auditor.Record(r.Context(), getObservationsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if auditErr := api.auditor.Record(ctx, getObservationsAction, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr } - handleObservationsErrorType(w, err) + handleObservationsErrorType(ctx, w, err, logData) return } - observationsDoc := models.CreateObservationsDoc(r.URL.RawQuery, versionDoc, dataset, observations, queryParameters, defaultOffset, defaultObservationLimit) - - if auditErr := api.auditor.Record(r.Context(), getObservationsAction, actionSuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getObservationsAction, audit.Successful, auditParams); auditErr != nil { + handleObservationsErrorType(ctx, w, auditErr, logData) return } @@ -198,12 +194,11 @@ func (api *DatasetAPI) getObservations(w http.ResponseWriter, r *http.Request) { enc.SetEscapeHTML(false) if err = enc.Encode(observationsDoc); err != nil { - log.ErrorC("failed to marshal metadata resource into bytes", err, logData) - handleObservationsErrorType(w, err) + handleObservationsErrorType(ctx, w, errors.WithMessage(err, "failed to marshal metadata resource into bytes"), logData) return } - log.Info("successfully retrieved observations relative to a selected set of dimension options for a version", logData) + log.InfoCtx(ctx, "get observations endpoint: successfully retrieved observations relative to a selected set of dimension options for a version", logData) } func getDimensionOffsetInHeaderRow(headerRow []string) (int, error) { @@ -399,21 +394,27 @@ func (api *DatasetAPI) getObservationList(versionDoc *models.Version, queryParam return observations, nil } -func handleObservationsErrorType(w http.ResponseWriter, err error) { - log.Error(err, nil) - - switch err { - case errs.ErrDatasetNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case errs.ErrEditionNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case errs.ErrVersionNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case errs.ErrObservationsNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - case errs.ErrTooManyWildcards: - http.Error(w, err.Error(), http.StatusBadRequest) +func handleObservationsErrorType(ctx context.Context, w http.ResponseWriter, err error, data log.Data) { + _, isObservationErr := err.(observationQueryError) + var status int + + switch { + case isObservationErr: + status = http.StatusBadRequest + case observationNotFound[err]: + status = http.StatusNotFound + case observationBadRequest[err]: + status = http.StatusBadRequest default: - http.Error(w, err.Error(), http.StatusInternalServerError) + err = errs.ErrInternalServer + status = http.StatusInternalServerError } + + if data == nil { + data = log.Data{} + } + + data["responseStatus"] = status + log.ErrorCtx(ctx, errors.WithMessage(err, "get observation endpoint: request unsuccessful"), data) + http.Error(w, err.Error(), status) } diff --git a/api/observation_test.go b/api/observation_test.go index f93e5e67..734eb1e8 100644 --- a/api/observation_test.go +++ b/api/observation_test.go @@ -13,8 +13,11 @@ import ( errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/mocks" "github.com/ONSdigital/dp-dataset-api/models" - storetest "github.com/ONSdigital/dp-dataset-api/store/datastoretest" + "github.com/ONSdigital/dp-dataset-api/store/datastoretest" "github.com/ONSdigital/dp-filter/observation" + "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/audit/audit_mock" + "github.com/ONSdigital/go-ns/common" "github.com/ONSdigital/go-ns/log" . "github.com/smartystreets/goconvey/convey" ) @@ -91,13 +94,14 @@ func TestGetObservationsReturnsOK(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), mockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, mockedObservationStore) Convey("When request contains query parameters where the dimension name is in lower casing", func() { r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) w := httptest.NewRecorder() + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(w.Body.String(), ShouldContainSubstring, getTestData("expectedDocWithSingleObservation")) @@ -106,13 +110,20 @@ func TestGetObservationsReturnsOK(t *testing.T) { So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedObservationStore.GetCSVRowsCalls()), ShouldEqual, 1) So(len(mockRowReader.ReadCalls()), ShouldEqual, 3) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Successful, Params: ap}, + ) }) Convey("When request contains query parameters where the dimension name is in upper casing", func() { r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&AggregaTe=cpi1dim1S40403&GEOGRAPHY=K02000001", nil) w := httptest.NewRecorder() + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(w.Body.String(), ShouldContainSubstring, getTestData("expectedSecondDocWithSingleObservation")) @@ -121,6 +132,13 @@ func TestGetObservationsReturnsOK(t *testing.T) { So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedObservationStore.GetCSVRowsCalls()), ShouldEqual, 1) So(len(mockRowReader.ReadCalls()), ShouldEqual, 3) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Successful, Params: ap}, + ) }) }) @@ -191,8 +209,10 @@ func TestGetObservationsReturnsOK(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), mockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, mockedObservationStore) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusOK) So(w.Body.String(), ShouldContainSubstring, getTestData("expectedDocWithMultipleObservations")) @@ -201,6 +221,13 @@ func TestGetObservationsReturnsOK(t *testing.T) { So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedObservationStore.GetCSVRowsCalls()), ShouldEqual, 1) So(len(mockRowReader.ReadCalls()), ShouldEqual, 4) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Successful, Params: ap}, + ) }) } @@ -211,17 +238,26 @@ func TestGetObservationsReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return nil, errInternal + return nil, errs.ErrInternalServer }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusInternalServerError) So(w.Body.String(), ShouldResemble, "internal error\n") So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the dataset does not exist return status not found", t, func() { @@ -233,13 +269,22 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Dataset not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the dataset exists but is unpublished return status not found", t, func() { @@ -251,13 +296,22 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Dataset not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the edition of a dataset does not exist return status not found", t, func() { @@ -272,14 +326,23 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Edition not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrEditionNotFound.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When version does not exist for an edition of a dataset returns status not found", t, func() { @@ -297,14 +360,22 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Version not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrVersionNotFound.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When an unpublished version has an incorrect state for an edition of a dataset return an internal error", t, func() { @@ -322,14 +393,21 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "Incorrect resource state\n") + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When a version document has not got a headers field return an internal server error", t, func() { @@ -350,13 +428,22 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When a version document has not got any dimensions field return an internal server error", t, func() { @@ -377,13 +464,22 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the first header in array does not describe the header row correctly return internal error", t, func() { @@ -405,14 +501,21 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "index out of range\n") + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + assertInternalServerErr(w) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When an invalid query parameter is set in request return 400 bad request with an error message containing a list of incorrect query parameters", t, func() { @@ -434,14 +537,23 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "Incorrect selection of query parameters: [geography], these dimensions do not exist for this version of the dataset\n") + So(w.Body.String(), ShouldResemble, "incorrect selection of query parameters: [geography], these dimensions do not exist for this version of the dataset\n") So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When there is a missing query parameter that is expected to be set in request return 400 bad request with an error message containing a list of missing query parameters", t, func() { @@ -463,14 +575,23 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "Missing query parameters for the following dimensions: [age]\n") + So(w.Body.String(), ShouldResemble, "missing query parameters for the following dimensions: [age]\n") So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When there are too many query parameters that are set to wildcard (*) value request returns 400 bad request", t, func() { @@ -492,14 +613,23 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusBadRequest) So(w.Body.String(), ShouldResemble, "only one wildcard (*) is allowed as a value in selected query parameters\n") So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When requested query does not find a unique observation return no observations found", t, func() { @@ -528,15 +658,24 @@ func TestGetObservationsReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), mockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, mockedObservationStore) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "No observations found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrObservationsNotFound.Error()) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedObservationStore.GetCSVRowsCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When requested query has a multi-valued dimension return bad request", t, func() { @@ -583,16 +722,24 @@ func TestGetObservationsReturnsError(t *testing.T) { } mockedObservationStore := &mocks.ObservationStoreMock{} + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, mockedObservationStore) + api.Router.ServeHTTP(w, r) - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), mockedObservationStore) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "Multi-valued query parameters for the following dimensions: [geography]\n") + So(w.Body.String(), ShouldResemble, "multi-valued query parameters for the following dimensions: [geography]\n") So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedObservationStore.GetCSVRowsCalls()), ShouldEqual, 0) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) }) } @@ -792,6 +939,276 @@ func TestExtractQueryParameters(t *testing.T) { }) } +func TestGetObservationAuditAttemptedError(t *testing.T) { + Convey("given audit action attempted returns an error", t, func() { + auditMock := audit_mock.NewErroring(getObservationsAction, audit.Attempted) + + mockedDataStore := &storetest.StorerMock{} + mockedObservationStore := &mocks.ObservationStoreMock{} + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, mockedObservationStore) + + Convey("when get observation is called", func() { + r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) + w := httptest.NewRecorder() + + api.Router.ServeHTTP(w, r) + + Convey("then a 500 response status is returned", func() { + assertInternalServerErr(w) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) + So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) + So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{ + Action: getObservationsAction, + Result: audit.Attempted, + Params: common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"}}, + ) + }) + }) + }) +} + +func TestGetObservationAuditUnsuccessfulError(t *testing.T) { + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + + Convey("given audit action unsuccessful returns an error", t, func() { + + Convey("when datastore.getDataset returns an error", func() { + auditMock := audit_mock.NewErroring(getObservationsAction, audit.Unsuccessful) + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound + }, + } + + mockedObservationStore := &mocks.ObservationStoreMock{} + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, mockedObservationStore) + r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) + w := httptest.NewRecorder() + + api.Router.ServeHTTP(w, r) + + Convey("then a 500 response status is returned", func() { + assertInternalServerErr(w) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + So(len(mockedObservationStore.GetCSVRowsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) + }) + + Convey("when datastore.getEdition returns an error", func() { + auditMock := audit_mock.NewErroring(getObservationsAction, audit.Unsuccessful) + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil + }, + CheckEditionExistsFunc: func(ID string, editionID string, state string) error { + return errs.ErrEditionNotFound + }, + } + + mockedObservationStore := &mocks.ObservationStoreMock{} + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, mockedObservationStore) + r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) + w := httptest.NewRecorder() + + api.Router.ServeHTTP(w, r) + + Convey("then a 500 response status is returned", func() { + assertInternalServerErr(w) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + So(len(mockedObservationStore.GetCSVRowsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) + }) + + Convey("when datastore.getVersion returns an error", func() { + auditMock := audit_mock.NewErroring(getObservationsAction, audit.Unsuccessful) + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil + }, + CheckEditionExistsFunc: func(ID string, editionID string, state string) error { + return nil + }, + GetVersionFunc: func(datasetID string, editionID string, version string, state string) (*models.Version, error) { + return nil, errs.ErrVersionNotFound + }, + } + + mockedObservationStore := &mocks.ObservationStoreMock{} + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, mockedObservationStore) + r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) + w := httptest.NewRecorder() + + api.Router.ServeHTTP(w, r) + + Convey("then a 500 response status is returned", func() { + assertInternalServerErr(w) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedObservationStore.GetCSVRowsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) + }) + + Convey("when the version does not have no header data", func() { + auditMock := audit_mock.NewErroring(getObservationsAction, audit.Unsuccessful) + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil + }, + CheckEditionExistsFunc: func(ID string, editionID string, state string) error { + return nil + }, + GetVersionFunc: func(datasetID string, editionID string, version string, state string) (*models.Version, error) { + return &models.Version{}, nil + }, + } + + mockedObservationStore := &mocks.ObservationStoreMock{} + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, mockedObservationStore) + r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) + w := httptest.NewRecorder() + + api.Router.ServeHTTP(w, r) + + Convey("then a 500 response status is returned", func() { + assertInternalServerErr(w) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedObservationStore.GetCSVRowsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) + }) + }) +} + +func TestGetObservationAuditSuccessfulError(t *testing.T) { + Convey("given audit action successful returns an error", t, func() { + auditMock := audit_mock.NewErroring(getObservationsAction, audit.Successful) + + Convey("when get observations is called with a valid request", func() { + + dimensions := []models.CodeList{ + models.CodeList{ + Name: "aggregate", + HRef: "http://localhost:8081/code-lists/cpih1dim1aggid", + }, + models.CodeList{ + Name: "geography", + HRef: "http://localhost:8081/code-lists/uk-only", + }, + models.CodeList{ + Name: "time", + HRef: "http://localhost:8081/code-lists/time", + }, + } + usagesNotes := &[]models.UsageNote{models.UsageNote{Title: "data_marking", Note: "this marks the obsevation with a special character"}} + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil + }, + CheckEditionExistsFunc: func(datasetID, editionID, state string) error { + return nil + }, + GetVersionFunc: func(string, string, string, string) (*models.Version, error) { + return &models.Version{ + Dimensions: dimensions, + Headers: []string{"v4_2", "data_marking", "confidence_interval", "aggregate_code", "aggregate", "geography_code", "geography", "time", "time"}, + Links: &models.VersionLinks{ + Version: &models.LinkObject{ + HRef: "http://localhost:8080/datasets/cpih012/editions/2017/versions/1", + ID: "1", + }, + }, + State: models.PublishedState, + UsageNotes: usagesNotes, + }, nil + }, + } + + count := 0 + mockRowReader := &mocks.CSVRowReaderMock{ + ReadFunc: func() (string, error) { + count++ + if count == 1 { + return "v4_2,data_marking,confidence_interval,time,time,geography_code,geography,aggregate_code,aggregate", nil + } else if count == 2 { + return "146.3,p,2,Month,Aug-16,K02000001,,cpi1dim1G10100,01.1 Food", nil + } + return "", io.EOF + }, + CloseFunc: func() error { + return nil + }, + } + + mockedObservationStore := &mocks.ObservationStoreMock{ + GetCSVRowsFunc: func(*observation.Filter, *int) (observation.CSVRowReader, error) { + return mockRowReader, nil + }, + } + + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, mockedObservationStore) + r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) + w := httptest.NewRecorder() + + api.Router.ServeHTTP(w, r) + + Convey("then a 500 status response is returned", func() { + assertInternalServerErr(w) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + So(len(mockedObservationStore.GetCSVRowsCalls()), ShouldEqual, 1) + + ap := common.Params{"dataset_id": "cpih012", "edition": "2017", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getObservationsAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: getObservationsAction, Result: audit.Successful, Params: ap}, + ) + }) + + }) + }) +} + func getTestData(filename string) string { jsonBytes, err := ioutil.ReadFile("./observation_test_data/" + filename + ".json") if err != nil { diff --git a/api/versions.go b/api/versions.go index 8ba44f1e..3b8b2776 100644 --- a/api/versions.go +++ b/api/versions.go @@ -1,336 +1,444 @@ package api import ( + "context" "encoding/json" + "io" "net/http" + "strings" + errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" + "github.com/ONSdigital/go-ns/audit" "github.com/ONSdigital/go-ns/common" "github.com/ONSdigital/go-ns/log" "github.com/gorilla/mux" "github.com/pkg/errors" ) +var ( + // errors that map to a HTTP 404 response + notFound = map[error]bool{ + errs.ErrDatasetNotFound: true, + errs.ErrEditionNotFound: true, + errs.ErrVersionNotFound: true, + } + + // errors that map to a HTTP 400 response + badRequest = map[error]bool{ + errs.ErrVersionBadRequest: true, + models.ErrPublishedVersionCollectionIDInvalid: true, + models.ErrAssociatedVersionCollectionIDInvalid: true, + models.ErrVersionStateInvalid: true, + } + + // HTTP 500 responses with a specific message + internalServerErrWithMessage = map[error]bool{ + errs.ErrResourceState: true, + } +) + +// VersionDetails contains the details that uniquely identify a version resource +type VersionDetails struct { + datasetID string + edition string + version string +} + +func (v VersionDetails) baseAuditParams() common.Params { + return common.Params{"dataset_id": v.datasetID, "edition": v.edition, "version": v.version} +} + func (api *DatasetAPI) getVersions(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) id := vars["id"] editionID := vars["edition"] - logData := log.Data{"dataset_id": id, "edition": editionID} auditParams := common.Params{"dataset_id": id, "edition": editionID} + logData := audit.ToLogData(auditParams) - if auditErr := api.auditor.Record(r.Context(), getVersionsAction, actionAttempted, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getVersionsAction, audit.Attempted, auditParams); auditErr != nil { + handleVersionAPIErr(ctx, errs.ErrInternalServer, w, logData) return } - authorised, logData := api.authenticate(r, logData) + b, err := func() ([]byte, error) { + authorised, logData := api.authenticate(r, logData) - var state string - if !authorised { - state = models.PublishedState - } - - if err := api.dataStore.Backend.CheckDatasetExists(id, state); err != nil { - log.ErrorC("failed to find dataset for list of versions", err, logData) - if auditErr := api.auditor.Record(r.Context(), getVersionsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + var state string + if !authorised { + state = models.PublishedState } - handleErrorType(versionDocType, err, w) - return - } - if err := api.dataStore.Backend.CheckEditionExists(id, editionID, state); err != nil { - log.ErrorC("failed to find edition for list of versions", err, logData) - if auditErr := api.auditor.Record(r.Context(), getVersionsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if err := api.dataStore.Backend.CheckDatasetExists(id, state); err != nil { + audit.LogError(ctx, errors.WithMessage(err, "failed to find dataset for list of versions"), logData) + return nil, err } - handleErrorType(versionDocType, err, w) - return - } - results, err := api.dataStore.Backend.GetVersions(id, editionID, state) - if err != nil { - log.ErrorC("failed to find any versions for dataset edition", err, logData) - if auditErr := api.auditor.Record(r.Context(), getVersionsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if err := api.dataStore.Backend.CheckEditionExists(id, editionID, state); err != nil { + audit.LogError(ctx, errors.WithMessage(err, "failed to find edition for list of versions"), logData) + return nil, err } - handleErrorType(versionDocType, err, w) - return - } - var hasInvalidState bool - for _, item := range results.Items { - if err = models.CheckState("version", item.State); err != nil { - hasInvalidState = true - log.ErrorC("unpublished version has an invalid state", err, log.Data{"state": item.State}) + results, err := api.dataStore.Backend.GetVersions(id, editionID, state) + if err != nil { + audit.LogError(ctx, errors.WithMessage(err, "failed to find any versions for dataset edition"), logData) + return nil, err } - // Only the download service should have access to the - // public/private download fields - if r.Header.Get(downloadServiceToken) != api.downloadServiceToken { - if item.Downloads != nil { - if item.Downloads.CSV != nil { - item.Downloads.CSV.Private = "" - item.Downloads.CSV.Public = "" - } - if item.Downloads.XLS != nil { - item.Downloads.XLS.Private = "" - item.Downloads.XLS.Public = "" + var hasInvalidState bool + for _, item := range results.Items { + if err = models.CheckState("version", item.State); err != nil { + hasInvalidState = true + audit.LogError(ctx, errors.WithMessage(err, "unpublished version has an invalid state"), log.Data{"state": item.State}) + } + + // Only the download service should have access to the + // public/private download fields + if r.Header.Get(downloadServiceToken) != api.downloadServiceToken { + if item.Downloads != nil { + if item.Downloads.CSV != nil { + item.Downloads.CSV.Private = "" + item.Downloads.CSV.Public = "" + } + if item.Downloads.XLS != nil { + item.Downloads.XLS.Private = "" + item.Downloads.XLS.Public = "" + } } } } - } - if hasInvalidState { - if auditErr := api.auditor.Record(r.Context(), getVersionsAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if hasInvalidState { + return nil, err } - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - b, err := json.Marshal(results) + b, err := json.Marshal(results) + if err != nil { + audit.LogError(ctx, errors.WithMessage(err, "failed to marshal list of version resources into bytes"), logData) + return nil, err + } + return b, nil + }() + if err != nil { - log.ErrorC("failed to marshal list of version resources into bytes", err, logData) - http.Error(w, err.Error(), http.StatusInternalServerError) + if auditErr := api.auditor.Record(ctx, getVersionsAction, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr + } + handleVersionAPIErr(ctx, err, w, logData) return } - if auditErr := api.auditor.Record(r.Context(), getVersionsAction, actionSuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getVersionsAction, audit.Successful, auditParams); auditErr != nil { + handleVersionAPIErr(ctx, auditErr, w, logData) return } setJSONContentType(w) _, err = w.Write(b) if err != nil { - log.Error(err, log.Data{"dataset_id": id, "edition": editionID}) - http.Error(w, err.Error(), http.StatusInternalServerError) + audit.LogError(ctx, errors.WithMessage(err, "error writing bytes to response"), logData) + handleVersionAPIErr(ctx, err, w, logData) } - log.Debug("get all versions", logData) + audit.LogInfo(ctx, "getVersions endpoint: request successful", logData) } func (api *DatasetAPI) getVersion(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) id := vars["id"] editionID := vars["edition"] version := vars["version"] - logData := log.Data{"dataset_id": id, "edition": editionID, "version": version} auditParams := common.Params{"dataset_id": id, "edition": editionID, "version": version} + logData := audit.ToLogData(auditParams) - if auditErr := api.auditor.Record(r.Context(), getVersionAction, actionAttempted, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getVersionAction, audit.Attempted, auditParams); auditErr != nil { + handleVersionAPIErr(ctx, auditErr, w, logData) return } - authorised, logData := api.authenticate(r, logData) + b, getVersionErr := func() ([]byte, error) { + authorised, logData := api.authenticate(r, logData) - var state string - if !authorised { - state = models.PublishedState - } + var state string + if !authorised { + state = models.PublishedState + } - if err := api.dataStore.Backend.CheckDatasetExists(id, state); err != nil { - log.ErrorC("failed to find dataset", err, logData) - if auditErr := api.auditor.Record(r.Context(), getVersionAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if err := api.dataStore.Backend.CheckDatasetExists(id, state); err != nil { + audit.LogError(ctx, errors.WithMessage(err, "failed to find dataset"), logData) + return nil, err } - handleErrorType(versionDocType, err, w) - return - } - if err := api.dataStore.Backend.CheckEditionExists(id, editionID, state); err != nil { - log.ErrorC("failed to find edition for dataset", err, logData) - if auditErr := api.auditor.Record(r.Context(), getVersionAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if err := api.dataStore.Backend.CheckEditionExists(id, editionID, state); err != nil { + checkEditionErr := errors.WithMessage(err, "failed to find edition for dataset") + audit.LogError(ctx, checkEditionErr, logData) + return nil, err } - handleErrorType(versionDocType, err, w) - return - } - results, err := api.dataStore.Backend.GetVersion(id, editionID, version, state) - if err != nil { - log.ErrorC("failed to find version for dataset edition", err, logData) - if auditErr := api.auditor.Record(r.Context(), getVersionAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + results, err := api.dataStore.Backend.GetVersion(id, editionID, version, state) + if err != nil { + audit.LogError(ctx, errors.WithMessage(err, "failed to find version for dataset edition"), logData) + return nil, err } - handleErrorType(versionDocType, err, w) - return - } - results.Links.Self.HRef = results.Links.Version.HRef + results.Links.Self.HRef = results.Links.Version.HRef - if err = models.CheckState("version", results.State); err != nil { - log.ErrorC("unpublished version has an invalid state", err, log.Data{"state": results.State}) - if auditErr := api.auditor.Record(r.Context(), getVersionAction, actionUnsuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) - return + if err = models.CheckState("version", results.State); err != nil { + audit.LogError(ctx, errors.WithMessage(err, "unpublished version has an invalid state"), log.Data{"state": results.State}) + return nil, errs.ErrResourceState } - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - // Only the download service should not have access to the public/private download - // fields - if r.Header.Get(downloadServiceToken) != api.downloadServiceToken { - if results.Downloads != nil { - if results.Downloads.CSV != nil { - results.Downloads.CSV.Private = "" - results.Downloads.CSV.Public = "" - } - if results.Downloads.XLS != nil { - results.Downloads.XLS.Private = "" - results.Downloads.XLS.Public = "" + // Only the download service should not have access to the public/private download + // fields + if r.Header.Get(downloadServiceToken) != api.downloadServiceToken { + if results.Downloads != nil { + if results.Downloads.CSV != nil { + results.Downloads.CSV.Private = "" + results.Downloads.CSV.Public = "" + } + if results.Downloads.XLS != nil { + results.Downloads.XLS.Private = "" + results.Downloads.XLS.Public = "" + } } } - } - b, err := json.Marshal(results) - if err != nil { - log.ErrorC("failed to marshal version resource into bytes", err, logData) - http.Error(w, err.Error(), http.StatusInternalServerError) + b, err := json.Marshal(results) + if err != nil { + audit.LogError(ctx, errors.WithMessage(err, "failed to marshal version resource into bytes"), logData) + return nil, err + } + return b, nil + }() + + if getVersionErr != nil { + if auditErr := api.auditor.Record(ctx, getVersionAction, audit.Unsuccessful, auditParams); auditErr != nil { + getVersionErr = auditErr + } + handleVersionAPIErr(ctx, getVersionErr, w, logData) return } - if auditErr := api.auditor.Record(r.Context(), getVersionAction, actionSuccessful, auditParams); auditErr != nil { - handleAuditingFailure(w, auditErr, logData) + if auditErr := api.auditor.Record(ctx, getVersionAction, audit.Successful, auditParams); auditErr != nil { + handleVersionAPIErr(ctx, auditErr, w, logData) return } setJSONContentType(w) - _, err = w.Write(b) + _, err := w.Write(b) if err != nil { - log.Error(err, logData) - http.Error(w, err.Error(), http.StatusInternalServerError) + audit.LogError(ctx, errors.WithMessage(err, "failed writing bytes to response"), logData) + handleVersionAPIErr(ctx, err, w, logData) } - log.Debug("get version", logData) + audit.LogInfo(ctx, "getVersion endpoint: request successful", logData) } func (api *DatasetAPI) putVersion(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) - datasetID := vars["id"] - edition := vars["edition"] - version := vars["version"] - - versionDoc, err := models.CreateVersion(r.Body) - defer r.Body.Close() - if err != nil { - log.ErrorC("failed to model version resource based on request", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) - http.Error(w, err.Error(), http.StatusBadRequest) - return + versionDetails := VersionDetails{ + datasetID: vars["id"], + edition: vars["edition"], + version: vars["version"], + } + data := log.Data{ + "datasetID": vars["id"], + "edition": vars["edition"], + "version": vars["version"], } - currentDataset, err := api.dataStore.Backend.GetDataset(datasetID) + currentDataset, currentVersion, versionDoc, err := api.updateVersion(ctx, r.Body, versionDetails) if err != nil { - log.ErrorC("failed to find dataset", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) - handleErrorType(versionDocType, err, w) + handleVersionAPIErr(ctx, err, w, data) return } - if err = api.dataStore.Backend.CheckEditionExists(datasetID, edition, ""); err != nil { - log.ErrorC("failed to find edition of dataset", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) - handleErrorType(versionDocType, err, w) - return + // If update was to add downloads do not try to publish/associate version + if vars[hasDownloads] != trueStringified { + if versionDoc.State == models.PublishedState { + if err := api.publishVersion(ctx, currentDataset, currentVersion, versionDoc, versionDetails); err != nil { + handleVersionAPIErr(ctx, err, w, data) + return + } + } + + if versionDoc.State == models.AssociatedState && currentVersion.State != models.AssociatedState { + if err := api.associateVersion(ctx, currentVersion, versionDoc, versionDetails); err != nil { + handleVersionAPIErr(ctx, err, w, data) + return + } + } } - currentVersion, err := api.dataStore.Backend.GetVersion(datasetID, edition, version, "") + setJSONContentType(w) + w.WriteHeader(http.StatusOK) + audit.LogInfo(ctx, "putVersion endpoint: request successful", data) +} + +func (api *DatasetAPI) updateVersion(ctx context.Context, body io.ReadCloser, versionDetails VersionDetails) (*models.DatasetUpdate, *models.Version, *models.Version, error) { + ap := versionDetails.baseAuditParams() + data := audit.ToLogData(ap) + + // attempt to update the version + currentDataset, currentVersion, versionUpdate, err := func() (*models.DatasetUpdate, *models.Version, *models.Version, error) { + defer body.Close() + versionUpdate, err := models.CreateVersion(body) + if err != nil { + audit.LogError(ctx, errors.WithMessage(err, "putVersion endpoint: failed to model version resource based on request"), data) + return nil, nil, nil, errs.ErrVersionBadRequest + } + + currentDataset, err := api.dataStore.Backend.GetDataset(versionDetails.datasetID) + if err != nil { + audit.LogError(ctx, errors.WithMessage(err, "putVersion endpoint: datastore.getDataset returned an error"), data) + return nil, nil, nil, err + } + + if err = api.dataStore.Backend.CheckEditionExists(versionDetails.datasetID, versionDetails.edition, ""); err != nil { + audit.LogError(ctx, errors.WithMessage(err, "putVersion endpoint: failed to find edition of dataset"), data) + return nil, nil, nil, err + } + + currentVersion, err := api.dataStore.Backend.GetVersion(versionDetails.datasetID, versionDetails.edition, versionDetails.version, "") + if err != nil { + audit.LogError(ctx, errors.WithMessage(err, "putVersion endpoint: datastore.GetVersion returned an error"), data) + return nil, nil, nil, err + } + + // Combine update version document to existing version document + populateNewVersionDoc(currentVersion, versionUpdate) + data["updated_version"] = versionUpdate + audit.LogInfo(ctx, "putVersion endpoint: combined current version document with update request", data) + + if err = models.ValidateVersion(versionUpdate); err != nil { + audit.LogError(ctx, errors.Wrap(err, "putVersion endpoint: failed validation check for version update"), nil) + return nil, nil, nil, err + } + + if err := api.dataStore.Backend.UpdateVersion(versionUpdate.ID, versionUpdate); err != nil { + audit.LogError(ctx, errors.WithMessage(err, "putVersion endpoint: failed to update version document"), data) + return nil, nil, nil, err + } + return currentDataset, currentVersion, versionUpdate, nil + }() + + // audit update unsuccessful if error if err != nil { - log.ErrorC("failed to find version of dataset edition", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) - handleErrorType(versionDocType, err, w) - return + if auditErr := api.auditor.Record(ctx, updateVersionAction, audit.Unsuccessful, ap); auditErr != nil { + audit.LogActionFailure(ctx, updateVersionAction, audit.Unsuccessful, auditErr, data) + } + return nil, nil, nil, err } - // Combine update version document to existing version document - populateNewVersionDoc(currentVersion, versionDoc) - log.Debug("combined current version document with update request", log.Data{"dataset_id": datasetID, "edition": edition, "version": version, "updated_version": versionDoc}) - - if err = models.ValidateVersion(versionDoc); err != nil { - log.ErrorC("failed validation check for version update", err, nil) - http.Error(w, err.Error(), http.StatusBadRequest) - return + if auditErr := api.auditor.Record(ctx, updateVersionAction, audit.Successful, ap); auditErr != nil { + audit.LogActionFailure(ctx, updateVersionAction, audit.Successful, auditErr, data) } - if err := api.dataStore.Backend.UpdateVersion(versionDoc.ID, versionDoc); err != nil { - log.ErrorC("failed to update version document", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) - handleErrorType(versionDocType, err, w) - return + audit.LogInfo(ctx, "update version completed successfully", data) + return currentDataset, currentVersion, versionUpdate, nil +} + +func (api *DatasetAPI) publishVersion(ctx context.Context, currentDataset *models.DatasetUpdate, currentVersion *models.Version, versionDoc *models.Version, versionDetails VersionDetails) error { + ap := versionDetails.baseAuditParams() + data := audit.ToLogData(ap) + + if auditErr := api.auditor.Record(ctx, publishVersionAction, audit.Attempted, ap); auditErr != nil { + audit.LogActionFailure(ctx, publishVersionAction, audit.Attempted, auditErr, data) + return auditErr } - if versionDoc.State == models.PublishedState { + audit.LogInfo(ctx, "attempting to publish version", data) - editionDoc, err := api.dataStore.Backend.GetEdition(datasetID, edition, "") + err := func() error { + editionDoc, err := api.dataStore.Backend.GetEdition(versionDetails.datasetID, versionDetails.edition, "") if err != nil { - log.ErrorC("failed to find the edition we're trying to update", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) - handleErrorType(versionDocType, err, w) - return + audit.LogError(ctx, errors.WithMessage(err, "putVersion endpoint: failed to find the edition we're trying to update"), data) + return err } editionDoc.Next.State = models.PublishedState editionDoc.Current = editionDoc.Next - if err := api.dataStore.Backend.UpsertEdition(datasetID, edition, editionDoc); err != nil { - log.ErrorC("failed to update edition during publishing", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) - handleErrorType(versionDocType, err, w) - return + if err := api.dataStore.Backend.UpsertEdition(versionDetails.datasetID, versionDetails.edition, editionDoc); err != nil { + audit.LogError(ctx, errors.WithMessage(err, "putVersion endpoint: failed to update edition during publishing"), data) + return err } // Pass in newVersion variable to include relevant data needed for update on dataset API (e.g. links) - if err := api.publishDataset(currentDataset, versionDoc); err != nil { - log.ErrorC("failed to update dataset document once version state changes to publish", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) - handleErrorType(versionDocType, err, w) - return + if err := api.publishDataset(ctx, currentDataset, versionDoc); err != nil { + audit.LogError(ctx, errors.WithMessage(err, "putVersion endpoint: failed to update dataset document once version state changes to publish"), data) + return err } // Only want to generate downloads again if there is no public link available if currentVersion.Downloads != nil && currentVersion.Downloads.CSV != nil && currentVersion.Downloads.CSV.Public == "" { - if err := api.downloadGenerator.Generate(datasetID, versionDoc.ID, edition, version); err != nil { - err = errors.Wrap(err, "error while attempting to generate full dataset version downloads on version publish") - log.Error(err, log.Data{ - "dataset_id": datasetID, - "instance_id": versionDoc.ID, - "edition": edition, - "version": version, - "state": versionDoc.State, - }) + if err := api.downloadGenerator.Generate(versionDetails.datasetID, versionDoc.ID, versionDetails.edition, versionDetails.version); err != nil { + data["instance_id"] = versionDoc.ID + data["state"] = versionDoc.State + audit.LogError(ctx, errors.WithMessage(err, "putVersion endpoint: error while attempting to generate full dataset version downloads on version publish"), data) // TODO - TECH DEBT - need to add an error event for this. - handleErrorType(versionDocType, err, w) + return err } } + return nil + }() + + if err != nil { + if auditErr := api.auditor.Record(ctx, publishVersionAction, audit.Unsuccessful, ap); auditErr != nil { + audit.LogActionFailure(ctx, publishVersionAction, audit.Unsuccessful, auditErr, data) + } + return err + } + + if auditErr := api.auditor.Record(ctx, publishVersionAction, audit.Successful, ap); auditErr != nil { + audit.LogActionFailure(ctx, publishVersionAction, audit.Successful, auditErr, data) } - if versionDoc.State == models.AssociatedState && currentVersion.State != models.AssociatedState { - if err := api.dataStore.Backend.UpdateDatasetWithAssociation(datasetID, versionDoc.State, versionDoc); err != nil { - log.ErrorC("failed to update dataset document after a version of a dataset has been associated with a collection", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) - handleErrorType(versionDocType, err, w) - return + audit.LogInfo(ctx, "publish version completed successfully", data) + return nil +} + +func (api *DatasetAPI) associateVersion(ctx context.Context, currentVersion *models.Version, versionDoc *models.Version, versionDetails VersionDetails) error { + ap := versionDetails.baseAuditParams() + data := audit.ToLogData(ap) + + if auditErr := api.auditor.Record(ctx, associateVersionAction, audit.Attempted, ap); auditErr != nil { + audit.LogActionFailure(ctx, associateVersionAction, audit.Attempted, auditErr, data) + return auditErr + } + + associateVersionErr := func() error { + if err := api.dataStore.Backend.UpdateDatasetWithAssociation(versionDetails.datasetID, versionDoc.State, versionDoc); err != nil { + audit.LogError(ctx, errors.WithMessage(err, "putVersion endpoint: failed to update dataset document after a version of a dataset has been associated with a collection"), data) + return err } - log.Info("generating full dataset version downloads", log.Data{"dataset_id": datasetID, "edition": edition, "version": version}) + audit.LogInfo(ctx, "putVersion endpoint: generating full dataset version downloads", data) + + if err := api.downloadGenerator.Generate(versionDetails.datasetID, versionDoc.ID, versionDetails.edition, versionDetails.version); err != nil { + data["instance_id"] = versionDoc.ID + data["state"] = versionDoc.State + err = errors.WithMessage(err, "putVersion endpoint: error while attempting to generate full dataset version downloads on version association") + audit.LogError(ctx, err, data) + return err + } + return nil + }() - if err := api.downloadGenerator.Generate(datasetID, versionDoc.ID, edition, version); err != nil { - err = errors.Wrap(err, "error while attempting to generate full dataset version downloads on version association") - log.Error(err, log.Data{ - "dataset_id": datasetID, - "instance_id": versionDoc.ID, - "edition": edition, - "version": version, - "state": versionDoc.State, - }) - // TODO - TECH DEBT - need to add an error event for this. - handleErrorType(versionDocType, err, w) + if associateVersionErr != nil { + if auditErr := api.auditor.Record(ctx, associateVersionAction, audit.Unsuccessful, ap); auditErr != nil { + audit.LogActionFailure(ctx, associateVersionAction, audit.Unsuccessful, auditErr, data) } + return associateVersionErr } - setJSONContentType(w) - w.WriteHeader(http.StatusOK) - log.Debug("update dataset", log.Data{"dataset_id": datasetID}) + if auditErr := api.auditor.Record(ctx, associateVersionAction, audit.Successful, ap); auditErr != nil { + audit.LogActionFailure(ctx, associateVersionAction, audit.Successful, auditErr, data) + } + + audit.LogInfo(ctx, "associate version completed successfully", data) + return associateVersionErr } func populateNewVersionDoc(currentVersion *models.Version, version *models.Version) *models.Version { @@ -345,7 +453,6 @@ func populateNewVersionDoc(currentVersion *models.Version, version *models.Versi } if version.Alerts != nil { - // loop through new alerts and add each alert to array for _, newAlert := range *version.Alerts { alerts = append(alerts, newAlert) @@ -448,3 +555,29 @@ func populateNewVersionDoc(currentVersion *models.Version, version *models.Versi return version } + +func handleVersionAPIErr(ctx context.Context, err error, w http.ResponseWriter, data log.Data) { + var status int + switch { + case notFound[err]: + status = http.StatusNotFound + case badRequest[err]: + status = http.StatusBadRequest + case internalServerErrWithMessage[err]: + status = http.StatusInternalServerError + case strings.HasPrefix(err.Error(), "missing mandatory fields:"): + status = http.StatusBadRequest + case strings.HasPrefix(err.Error(), "invalid fields:"): + status = http.StatusBadRequest + default: + err = errs.ErrInternalServer + status = http.StatusInternalServerError + } + + if data == nil { + data = log.Data{} + } + + audit.LogError(ctx, errors.WithMessage(err, "request unsuccessful"), data) + http.Error(w, err.Error(), status) +} diff --git a/api/versions_test.go b/api/versions_test.go index 8b94485e..e7806f3e 100644 --- a/api/versions_test.go +++ b/api/versions_test.go @@ -4,9 +4,9 @@ import ( "bytes" "context" "encoding/json" - "errors" "net/http" "net/http/httptest" + "strings" "testing" "time" @@ -16,9 +16,12 @@ import ( "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/store" "github.com/ONSdigital/dp-dataset-api/store/datastoretest" + "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/audit/audit_mock" "github.com/ONSdigital/go-ns/common" "github.com/ONSdigital/go-ns/log" "github.com/gorilla/mux" + "github.com/pkg/errors" . "github.com/smartystreets/goconvey/convey" ) @@ -45,21 +48,21 @@ func TestGetVersionsReturnsOK(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - - p := common.Params{"dataset_id": "123-456", "edition": "678"} - recCalls := auditMock.RecordCalls() + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionSuccessful, p) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 1) + + p := common.Params{"dataset_id": "123-456", "edition": "678"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Successful, Params: p}, + ) }) } @@ -72,24 +75,24 @@ func TestGetVersionsReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(datasetID, state string) error { - return errInternal + return errs.ErrInternalServer }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionUnsuccessful, p) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 0) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When the dataset does not exist return status not found", t, func() { @@ -101,19 +104,22 @@ func TestGetVersionsReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Dataset not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionUnsuccessful, p) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When the edition of a dataset does not exist return status not found", t, func() { @@ -128,22 +134,22 @@ func TestGetVersionsReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Edition not found\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionUnsuccessful, p) + So(w.Body.String(), ShouldContainSubstring, errs.ErrEditionNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When version does not exist for an edition of a dataset returns status not found", t, func() { @@ -162,22 +168,22 @@ func TestGetVersionsReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Version not found\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionUnsuccessful, p) + So(w.Body.String(), ShouldContainSubstring, errs.ErrVersionNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When version is not published against an edition of a dataset return status not found", t, func() { @@ -195,22 +201,22 @@ func TestGetVersionsReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Version not found\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionUnsuccessful, p) + So(w.Body.String(), ShouldContainSubstring, errs.ErrVersionNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When a published version has an incorrect state for an edition of a dataset return an internal error", t, func() { @@ -231,22 +237,22 @@ func TestGetVersionsReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "Incorrect resource state\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionUnsuccessful, p) + So(w.Body.String(), ShouldContainSubstring, errs.ErrResourceState.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Unsuccessful, Params: p}, + ) }) } @@ -260,23 +266,23 @@ func TestGetVersionsAuditError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{} - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { return err } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - recCalls := auditMock.RecordCalls() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - So(len(recCalls), ShouldEqual, 1) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + ) }) Convey("when auditing check dataset exists error returns an error then a 500 status is returned", t, func() { @@ -288,27 +294,27 @@ func TestGetVersionsAuditError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getVersionsAction && result == actionUnsuccessful { + if action == getVersionsAction && result == audit.Unsuccessful { return errors.New("error") } return nil } api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() + api.Router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionUnsuccessful, p) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("when auditing check edition exists error returns an error then a 500 status is returned", t, func() { @@ -323,27 +329,27 @@ func TestGetVersionsAuditError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getVersionsAction && result == actionUnsuccessful { + if action == getVersionsAction && result == audit.Unsuccessful { return errors.New("error") } return nil } api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() + api.Router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionUnsuccessful, p) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("when auditing get versions error returns an error then a 500 status is returned", t, func() { @@ -361,27 +367,20 @@ func TestGetVersionsAuditError(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getVersionsAction && result == actionUnsuccessful { - return errors.New("error") - } - return nil - } + auditMock := audit_mock.NewErroring(getVersionAction, audit.Unsuccessful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionUnsuccessful, p) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("when auditing invalid state returns an error then a 500 status is returned", t, func() { @@ -401,27 +400,27 @@ func TestGetVersionsAuditError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getVersionsAction && result == actionUnsuccessful { + if action == getVersionsAction && result == audit.Unsuccessful { return errors.New("error") } return nil } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) - - recCalls := auditMock.RecordCalls() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionUnsuccessful, p) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("when auditing get versions successful event errors then an 500 status is returned", t, func() { @@ -439,27 +438,28 @@ func TestGetVersionsAuditError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getVersionsAction && result == actionSuccessful { + if action == getVersionsAction && result == audit.Successful { return errors.New("error") } return nil } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) - p := common.Params{"dataset_id": "123-456", "edition": "678"} - recCalls := auditMock.RecordCalls() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionsAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionsAction, actionSuccessful, p) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 1) + + p := common.Params{"dataset_id": "123-456", "edition": "678"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionsAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionsAction, Result: audit.Successful, Params: p}, + ) }) } @@ -489,20 +489,21 @@ func TestGetVersionReturnsOK(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - p := common.Params{"dataset_id": "123-456", "edition": "678", "version": "1"} - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionSuccessful, p) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + p := common.Params{"dataset_id": "123-456", "edition": "678", "version": "1"} + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Successful, Params: p}, + ) }) } @@ -514,23 +515,22 @@ func TestGetVersionReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(datasetID, state string) error { - return errInternal + return errs.ErrInternalServer }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionUnsuccessful, p) - + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When the dataset does not exist for return status not found", t, func() { @@ -543,21 +543,22 @@ func TestGetVersionReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Dataset not found\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionUnsuccessful, p) + So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When the edition of a dataset does not exist return status not found", t, func() { @@ -573,21 +574,22 @@ func TestGetVersionReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Edition not found\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionUnsuccessful, p) + So(w.Body.String(), ShouldContainSubstring, errs.ErrEditionNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When version does not exist for an edition of a dataset return status not found", t, func() { @@ -606,21 +608,22 @@ func TestGetVersionReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Version not found\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionUnsuccessful, p) + So(w.Body.String(), ShouldContainSubstring, errs.ErrVersionNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When version is not published for an edition of a dataset return status not found", t, func() { @@ -638,21 +641,22 @@ func TestGetVersionReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldResemble, "Version not found\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionUnsuccessful, p) + So(w.Body.String(), ShouldContainSubstring, errs.ErrVersionNotFound.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When an unpublished version has an incorrect state for an edition of a dataset return an internal error", t, func() { @@ -680,21 +684,22 @@ func TestGetVersionReturnsError(t *testing.T) { }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "Incorrect resource state\n") - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionUnsuccessful, p) + So(w.Body.String(), ShouldContainSubstring, errs.ErrResourceState.Error()) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Unsuccessful, Params: p}, + ) }) } @@ -706,28 +711,28 @@ func TestGetVersionAuditErrors(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(datasetID, state string) error { - return errInternal + return errs.ErrInternalServer }, } - auditMock := getMockAuditor() + auditMock := audit_mock.New() auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getVersionAction && result == actionAttempted { + if action == getVersionAction && result == audit.Attempted { return errors.New("error") } return nil } - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, internalServerErr) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 1) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + ) }) Convey("When the dataset does not exist and audit errors then return a 500 status", t, func() { @@ -740,27 +745,20 @@ func TestGetVersionAuditErrors(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getVersionAction && result == actionUnsuccessful { - return errors.New("error") - } - return nil - } + auditMock := audit_mock.NewErroring(getVersionAction, audit.Unsuccessful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, internalServerErr) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionUnsuccessful, p) - + assertInternalServerErr(w) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When the edition does not exist for a dataset and auditing errors then a 500 status", t, func() { @@ -776,28 +774,20 @@ func TestGetVersionAuditErrors(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getVersionAction && result == actionUnsuccessful { - return errors.New("error") - } - return nil - } + auditMock := audit_mock.NewErroring(getVersionAction, audit.Unsuccessful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionUnsuccessful, p) - + assertInternalServerErr(w) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When version does not exist for an edition of a dataset and auditing errors then a 500 status", t, func() { @@ -816,28 +806,20 @@ func TestGetVersionAuditErrors(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getVersionAction && result == actionUnsuccessful { - return errors.New("error") - } - return nil - } + auditMock := audit_mock.NewErroring(getVersionAction, audit.Unsuccessful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionUnsuccessful, p) - + assertInternalServerErr(w) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("When version does not exist for an edition of a dataset and auditing errors then a 500 status", t, func() { @@ -862,28 +844,20 @@ func TestGetVersionAuditErrors(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getVersionAction && result == actionUnsuccessful { - return errors.New("error") - } - return nil - } + auditMock := audit_mock.NewErroring(getVersionAction, audit.Unsuccessful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionUnsuccessful, p) - + assertInternalServerErr(w) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Unsuccessful, Params: p}, + ) }) Convey("when auditing a successful request to get a version errors then return a 500 status", t, func() { @@ -910,30 +884,25 @@ func TestGetVersionAuditErrors(t *testing.T) { }, } - auditMock := getMockAuditor() - auditMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { - if action == getVersionAction && result == actionSuccessful { - return errors.New("error") - } - return nil - } + auditMock := audit_mock.NewErroring(getVersionAction, audit.Successful) api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, internalServerErr) - recCalls := auditMock.RecordCalls() - So(len(recCalls), ShouldEqual, 2) - verifyAuditRecordCalls(recCalls[0], getVersionAction, actionAttempted, p) - verifyAuditRecordCalls(recCalls[1], getVersionAction, actionSuccessful, p) + assertInternalServerErr(w) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckDatasetExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: getVersionAction, Result: audit.Attempted, Params: p}, + audit_mock.Expected{Action: getVersionAction, Result: audit.Successful, Params: p}, + ) }) } func TestPutVersionReturnsSuccessfully(t *testing.T) { + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} t.Parallel() Convey("When state is unchanged", t, func() { generatorMock := &mocks.DownloadsGeneratorMock{ @@ -983,22 +952,26 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { return nil }, } - mockedDataStore.GetVersion("123", "2017", "1", "") - mockedDataStore.UpdateVersion("a1b2c3", &models.Version{}) - - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 3) - So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 2) + So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateEditionCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateDatasetWithAssociationCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Successful, Params: ap}, + ) }) Convey("When state is set to associated", t, func() { @@ -1034,23 +1007,26 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { return nil }, } - mockedDataStore.GetVersion("123", "2017", "1", "") - mockedDataStore.UpdateVersion("a1b2c3", &models.Version{}) - mockedDataStore.UpdateDatasetWithAssociation("123", models.AssociatedState, &models.Version{}) - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 3) - So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 2) - So(len(mockedDataStore.UpdateDatasetWithAssociationCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateDatasetWithAssociationCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateEditionCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Successful, Params: ap}, + ) }) Convey("When state is set to edition-confirmed", t, func() { @@ -1090,9 +1066,9 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, getMockAuditor(), genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) select { case <-downloadsGenerated: @@ -1104,14 +1080,22 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { } So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) - So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetWithAssociationCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateEditionCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 4) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Successful, Params: ap}, + audit_mock.Expected{Action: associateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: associateVersionAction, Result: audit.Successful, Params: ap}, + ) }) Convey("When state is set to published", t, func() { @@ -1192,30 +1176,142 @@ func TestPutVersionReturnsSuccessfully(t *testing.T) { }, } - mockedDataStore.GetVersion("789", "2017", "1", "") - mockedDataStore.GetEdition("123", "2017", "") - mockedDataStore.UpdateVersion("a1b2c3", &models.Version{}) - mockedDataStore.GetDataset("123") - mockedDataStore.UpsertDataset("123", &models.DatasetUpdate{Next: &models.Dataset{}}) - - api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 3) - So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 2) + So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpsertEditionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 2) - So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 2) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDatasetWithAssociationCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 4) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Successful, Params: ap}, + audit_mock.Expected{Action: publishVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: publishVersionAction, Result: audit.Successful, Params: ap}, + ) }) + + Convey("When version is already published and update includes downloads object only", t, func() { + Convey("And downloads object contains only a csv object", func() { + var b string + b = `{"downloads": { "csv": { "public": "http://cmd-dev/test-site/cpih01", "size": "12", "href": "http://localhost:8080/cpih01"}}}` + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123/editions/2017/versions/1", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + updateVersionDownloadTest(r, ap) + }) + + Convey("And downloads object contains only a xls object", func() { + var b string + b = `{"downloads": { "xls": { "public": "http://cmd-dev/test-site/cpih01", "size": "12", "href": "http://localhost:8080/cpih01"}}}` + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123/editions/2017/versions/1", bytes.NewBufferString(b)) + So(err, ShouldBeNil) + + updateVersionDownloadTest(r, ap) + }) + }) +} + +func updateVersionDownloadTest(r *http.Request, ap common.Params) { + w := httptest.NewRecorder() + + generatorMock := &mocks.DownloadsGeneratorMock{ + GenerateFunc: func(string, string, string, string) error { + return nil + }, + } + + mockedDataStore := &storetest.StorerMock{ + GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{ + ID: "123", + Next: &models.Dataset{Links: &models.DatasetLinks{}}, + Current: &models.Dataset{Links: &models.DatasetLinks{}}, + }, nil + }, + CheckEditionExistsFunc: func(string, string, string) error { + return nil + }, + GetVersionFunc: func(string, string, string, string) (*models.Version, error) { + return &models.Version{ + ID: "789", + Links: &models.VersionLinks{ + Dataset: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123", + ID: "123", + }, + Dimensions: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions", + }, + Edition: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017", + ID: "2017", + }, + Self: &models.LinkObject{ + HRef: "http://localhost:22000/instances/765", + }, + Version: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017/versions/1", + }, + }, + ReleaseDate: "2017-12-12", + Downloads: &models.DownloadList{ + CSV: &models.DownloadObject{ + Private: "s3://csv-exported/myfile.csv", + HRef: "http://localhost:23600/datasets/123/editions/2017/versions/1.csv", + Size: "1234", + }, + }, + State: models.PublishedState, + }, nil + }, + UpdateVersionFunc: func(string, *models.Version) error { + return nil + }, + GetEditionFunc: func(string, string, string) (*models.EditionUpdate, error) { + return &models.EditionUpdate{ + ID: "123", + Next: &models.Edition{ + State: models.PublishedState, + }, + Current: &models.Edition{}, + }, nil + }, + } + + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) + So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) + So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 1) + // Check updates to edition and dataset resources were not called + So(len(mockedDataStore.UpsertEditionCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpsertDatasetCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateDatasetWithAssociationCalls()), ShouldEqual, 0) + So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Successful, Params: ap}, + ) } func TestPutVersionGenerateDownloadsError(t *testing.T) { Convey("given download generator returns an error", t, func() { - + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} mockedErr := errors.New("spectacular explosion") var v models.Version json.Unmarshal([]byte(versionAssociatedPayload), &v) @@ -1254,9 +1350,9 @@ func TestPutVersionGenerateDownloadsError(t *testing.T) { So(err, ShouldBeNil) cfg.EnablePrivateEnpoints = true - api := routes(*cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockDownloadGenerator, nil, genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := Routes(*cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockDownloadGenerator, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then an internal server error response is returned", func() { So(w.Code, ShouldEqual, http.StatusInternalServerError) @@ -1283,12 +1379,21 @@ func TestPutVersionGenerateDownloadsError(t *testing.T) { So(genCalls[0].DatasetID, ShouldEqual, "123") So(genCalls[0].Edition, ShouldEqual, "2017") So(genCalls[0].Version, ShouldEqual, "1") + + So(len(auditMock.RecordCalls()), ShouldEqual, 4) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Successful, Params: ap}, + audit_mock.Expected{Action: associateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: associateVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) }) }) }) } func TestPutEmptyVersion(t *testing.T) { + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} var v models.Version json.Unmarshal([]byte(versionAssociatedPayload), &v) v.State = models.AssociatedState @@ -1316,9 +1421,9 @@ func TestPutEmptyVersion(t *testing.T) { w := httptest.NewRecorder() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a http status ok is returned", func() { So(w.Code, ShouldEqual, http.StatusOK) @@ -1328,6 +1433,12 @@ func TestPutEmptyVersion(t *testing.T) { So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 1) So(mockedDataStore.UpdateVersionCalls()[0].Version.Downloads, ShouldBeNil) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Successful, Params: ap}, + ) }) }) }) @@ -1356,9 +1467,9 @@ func TestPutEmptyVersion(t *testing.T) { So(err, ShouldBeNil) w := httptest.NewRecorder() - api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) - - api.router.ServeHTTP(w, r) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) Convey("then a http status ok is returned", func() { So(w.Code, ShouldEqual, http.StatusOK) @@ -1387,12 +1498,445 @@ func TestPutEmptyVersion(t *testing.T) { So(len(mockedDataStore.UpdateEditionCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateDatasetWithAssociationCalls()), ShouldEqual, 0) So(len(mockDownloadGenerator.GenerateCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Successful, Params: ap}, + ) + }) + }) + }) +} + +func TestUpdateVersionAuditErrors(t *testing.T) { + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} + + t.Parallel() + Convey("given audit action attempted returns an error", t, func() { + auditMock := audit_mock.NewErroring(updateVersionAction, audit.Attempted) + + Convey("when updateVersion is called with a valid request", func() { + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123/editions/2017/versions/1", bytes.NewBufferString(versionPayload)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + store := &storetest.StorerMock{} + api := GetAPIWithMockedDatastore(store, nil, auditMock, nil) + + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + Convey("then an error is returned and updateVersion fails", func() { + // Check no calls have been made to the datastore + So(len(store.GetDatasetCalls()), ShouldEqual, 0) + So(len(store.CheckEditionExistsCalls()), ShouldEqual, 0) + So(len(store.GetVersionCalls()), ShouldEqual, 0) + So(len(store.UpdateVersionCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + ) + }) + }) + }) + + currentVersion := &models.Version{ + ID: "789", + Links: &models.VersionLinks{ + Dataset: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123", + ID: "123", + }, + Dimensions: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions", + }, + Edition: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017", + ID: "456", + }, + Self: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017/versions/1", + }, + }, + ReleaseDate: "2017", + State: models.EditionConfirmedState, + } + + Convey("given audit action successful returns an error", t, func() { + auditMock := audit_mock.NewErroring(updateVersionAction, audit.Successful) + + Convey("when updateVersion is called with a valid request", func() { + + store := &storetest.StorerMock{ + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return &models.DatasetUpdate{}, nil + }, + CheckEditionExistsFunc: func(string, string, string) error { + return nil + }, + GetVersionFunc: func(string, string, string, string) (*models.Version, error) { + return currentVersion, nil + }, + UpdateVersionFunc: func(string, *models.Version) error { + return nil + }, + } + + var expectedUpdateVersion models.Version + err := json.Unmarshal([]byte(versionPayload), &expectedUpdateVersion) + So(err, ShouldBeNil) + expectedUpdateVersion.Downloads = currentVersion.Downloads + expectedUpdateVersion.Links = currentVersion.Links + expectedUpdateVersion.ID = currentVersion.ID + expectedUpdateVersion.State = models.EditionConfirmedState + + w := httptest.NewRecorder() + + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123/editions/2017/versions/1", bytes.NewBufferString(versionPayload)) + So(err, ShouldBeNil) + + api := GetAPIWithMockedDatastore(store, nil, auditMock, nil) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusOK) + + Convey("then the expected audit events are recorded and the expected error is returned", func() { + + So(len(store.GetDatasetCalls()), ShouldEqual, 1) + So(len(store.CheckEditionExistsCalls()), ShouldEqual, 1) + So(len(store.GetVersionCalls()), ShouldEqual, 2) + So(len(store.UpdateVersionCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Successful, Params: ap}, + ) + }) + }) + }) + + Convey("given audit action unsuccessful returns an error", t, func() { + auditMock := audit_mock.NewErroring(updateVersionAction, audit.Unsuccessful) + + Convey("when update version is unsuccessful", func() { + store := &storetest.StorerMock{ + GetVersionFunc: func(string, string, string, string) (*models.Version, error) { + return nil, errs.ErrVersionNotFound + }, + GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { + return nil, errs.ErrDatasetNotFound + }, + } + r, err := createRequestWithAuth("PUT", "http://localhost:22000/datasets/123/editions/2017/versions/1", bytes.NewBufferString(versionPayload)) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + api := GetAPIWithMockedDatastore(store, nil, auditMock, nil) + api.Router.ServeHTTP(w, r) + So(w.Code, ShouldEqual, http.StatusNotFound) + So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) + + Convey("then the expected audit events are recorded and the expected error is returned", func() { + So(len(store.GetVersionCalls()), ShouldEqual, 1) + So(len(store.GetDatasetCalls()), ShouldEqual, 1) + So(len(store.CheckEditionExistsCalls()), ShouldEqual, 0) + So(len(store.UpdateVersionCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) + }) + }) +} + +func TestPublishVersionAuditErrors(t *testing.T) { + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} + versionDetails := VersionDetails{ + datasetID: "123", + edition: "2017", + version: "1", + } + + Convey("given audit action attempted returns an error", t, func() { + auditMock := audit_mock.NewErroring(publishVersionAction, audit.Attempted) + + Convey("when publish version is called", func() { + store := &storetest.StorerMock{} + api := GetAPIWithMockedDatastore(store, nil, auditMock, nil) + + err := api.publishVersion(context.Background(), nil, nil, nil, versionDetails) + So(err, ShouldNotBeNil) + + Convey("then the expected audit events are recorded and an error is returned", func() { + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: publishVersionAction, Result: audit.Attempted, Params: ap}, + ) + }) + }) + }) + + Convey("given audit action unsuccessful returns an error", t, func() { + auditMock := audit_mock.NewErroring(publishVersionAction, audit.Unsuccessful) + + Convey("when publish version returns an error", func() { + store := &storetest.StorerMock{ + GetEditionFunc: func(ID, editionID, state string) (*models.EditionUpdate, error) { + return nil, errs.ErrEditionNotFound + }, + } + + api := GetAPIWithMockedDatastore(store, nil, auditMock, nil) + err := api.publishVersion(context.Background(), nil, nil, nil, versionDetails) + So(err, ShouldNotBeNil) + + Convey("then the expected audit events are recorded and the expected error is returned", func() { + So(len(store.GetEditionCalls()), ShouldEqual, 1) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: publishVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: publishVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) + }) + }) + + Convey("given audit action successful returns an error", t, func() { + auditMock := audit_mock.NewErroring(publishVersionAction, audit.Successful) + + Convey("when publish version returns an error", func() { + store := &storetest.StorerMock{ + GetEditionFunc: func(string, string, string) (*models.EditionUpdate, error) { + return &models.EditionUpdate{ + ID: "123", + Next: &models.Edition{ + State: models.PublishedState, + }, + Current: &models.Edition{}, + }, nil + }, + UpsertEditionFunc: func(datasetID string, edition string, editionDoc *models.EditionUpdate) error { + return nil + }, + UpsertDatasetFunc: func(ID string, datasetDoc *models.DatasetUpdate) error { + return nil + }, + } + + currentDataset := &models.DatasetUpdate{ + ID: "123", + Next: &models.Dataset{Links: &models.DatasetLinks{}}, + Current: &models.Dataset{Links: &models.DatasetLinks{}}, + } + + currentVersion := &models.Version{ + ID: "789", + Links: &models.VersionLinks{ + Dataset: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123", + ID: "123", + }, + Dimensions: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions", + }, + Edition: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017", + ID: "456", + }, + Self: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017/versions/1", + }, + Version: &models.LinkObject{ + HRef: "", + }, + }, + ReleaseDate: "2017-12-12", + State: models.EditionConfirmedState, + } + + var updateVersion models.Version + err := json.Unmarshal([]byte(versionPublishedPayload), &updateVersion) + So(err, ShouldBeNil) + updateVersion.Links = currentVersion.Links + + api := GetAPIWithMockedDatastore(store, nil, auditMock, nil) + + err = api.publishVersion(context.Background(), currentDataset, currentVersion, &updateVersion, versionDetails) + So(err, ShouldBeNil) + + Convey("then the expected audit events are recorded and the expected error is returned", func() { + So(len(store.GetEditionCalls()), ShouldEqual, 1) + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: publishVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: publishVersionAction, Result: audit.Successful, Params: ap}, + ) + }) + }) + }) +} + +func TestAssociateVersionAuditErrors(t *testing.T) { + ap := common.Params{"dataset_id": "123", "edition": "2018", "version": "1"} + currentVersion := &models.Version{ + ID: "789", + Links: &models.VersionLinks{ + Dataset: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123", + ID: "123", + }, + Dimensions: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017/versions/1/dimensions", + }, + Edition: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017", + ID: "456", + }, + Self: &models.LinkObject{ + HRef: "http://localhost:22000/datasets/123/editions/2017/versions/1", + }, + Version: &models.LinkObject{ + HRef: "", + }, + }, + ReleaseDate: "2017-12-12", + State: models.EditionConfirmedState, + } + + var versionDoc models.Version + json.Unmarshal([]byte(versionAssociatedPayload), &versionDoc) + + versionDetails := VersionDetails{ + datasetID: "123", + edition: "2018", + version: "1", + } + + expectedErr := errors.New("err") + + Convey("given audit action attempted returns an error", t, func() { + auditMock := audit_mock.NewErroring(associateVersionAction, audit.Attempted) + + Convey("when associate version is called", func() { + + store := &storetest.StorerMock{} + gen := &mocks.DownloadsGeneratorMock{} + api := GetAPIWithMockedDatastore(store, gen, auditMock, genericMockedObservationStore) + + err := api.associateVersion(context.Background(), currentVersion, &versionDoc, versionDetails) + So(err, ShouldEqual, audit_mock.ErrAudit) + + Convey("then the expected audit event is captured and the expected error is returned", func() { + So(len(store.UpdateDatasetWithAssociationCalls()), ShouldEqual, 0) + So(len(gen.GenerateCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 1) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: associateVersionAction, Result: audit.Attempted, Params: ap}, + ) + }) + }) + }) + + Convey("given audit action unsuccessful returns an error", t, func() { + auditMock := audit_mock.NewErroring(associateVersionAction, audit.Unsuccessful) + + Convey("when datastore.UpdateDatasetWithAssociation returns an error", func() { + store := &storetest.StorerMock{ + UpdateDatasetWithAssociationFunc: func(ID string, state string, version *models.Version) error { + return expectedErr + }, + } + gen := &mocks.DownloadsGeneratorMock{} + api := GetAPIWithMockedDatastore(store, gen, auditMock, genericMockedObservationStore) + + err := api.associateVersion(context.Background(), currentVersion, &versionDoc, versionDetails) + So(err, ShouldEqual, expectedErr) + + Convey("then the expected audit event is captured and the expected error is returned", func() { + So(len(store.UpdateDatasetWithAssociationCalls()), ShouldEqual, 1) + So(len(gen.GenerateCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: associateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: associateVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) + }) + + Convey("when generating downloads returns an error", func() { + store := &storetest.StorerMock{ + UpdateDatasetWithAssociationFunc: func(ID string, state string, version *models.Version) error { + return nil + }, + } + gen := &mocks.DownloadsGeneratorMock{ + GenerateFunc: func(datasetID string, instanceID string, edition string, version string) error { + return expectedErr + }, + } + api := GetAPIWithMockedDatastore(store, gen, auditMock, genericMockedObservationStore) + + err := api.associateVersion(context.Background(), currentVersion, &versionDoc, versionDetails) + + Convey("then the expected audit event is captured and the expected error is returned", func() { + So(expectedErr.Error(), ShouldEqual, errors.Cause(err).Error()) + So(len(store.UpdateDatasetWithAssociationCalls()), ShouldEqual, 1) + So(len(gen.GenerateCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: associateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: associateVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) + }) + }) + }) + + Convey("given audit action successful returns an error", t, func() { + auditMock := audit_mock.NewErroring(associateVersionAction, audit.Successful) + + Convey("when associateVersion is called", func() { + store := &storetest.StorerMock{ + UpdateDatasetWithAssociationFunc: func(ID string, state string, version *models.Version) error { + return nil + }, + } + gen := &mocks.DownloadsGeneratorMock{ + GenerateFunc: func(datasetID string, instanceID string, edition string, version string) error { + return nil + }, + } + + api := GetAPIWithMockedDatastore(store, gen, auditMock, genericMockedObservationStore) + err := api.associateVersion(context.Background(), currentVersion, &versionDoc, versionDetails) + So(err, ShouldBeNil) + + Convey("then the expected audit event is captured and the expected error is returned", func() { + So(len(store.UpdateDatasetWithAssociationCalls()), ShouldEqual, 1) + So(len(gen.GenerateCalls()), ShouldEqual, 1) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: associateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: associateVersionAction, Result: audit.Successful, Params: ap}, + ) }) }) }) } func TestPutVersionReturnsError(t *testing.T) { + ap := common.Params{"dataset_id": "123", "edition": "2017", "version": "1"} t.Parallel() Convey("When the request contain malformed json a bad request status is returned", t, func() { generatorMock := &mocks.DownloadsGeneratorMock{ @@ -1416,15 +1960,22 @@ func TestPutVersionReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldEqual, "Failed to parse json body\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrUnableToParseJSON.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the api cannot connect to datastore return an internal server error", t, func() { @@ -1442,22 +1993,29 @@ func TestPutVersionReturnsError(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetVersionFunc: func(string, string, string, string) (*models.Version, error) { - return nil, errInternal + return nil, errs.ErrInternalServer }, GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { return &models.DatasetUpdate{}, nil }, } - api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldEqual, "internal error\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the dataset document cannot be found for version return status not found", t, func() { @@ -1485,16 +2043,23 @@ func TestPutVersionReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldEqual, "Dataset not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the edition document cannot be found for version return status not found", t, func() { @@ -1522,16 +2087,23 @@ func TestPutVersionReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldEqual, "Edition not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrEditionNotFound.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the version document cannot be found return status not found", t, func() { @@ -1562,17 +2134,24 @@ func TestPutVersionReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldEqual, "Version not found\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrVersionNotFound.Error()) So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the request is not authorised to update version then response returns status not found", t, func() { @@ -1595,14 +2174,19 @@ func TestPutVersionReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusUnauthorized) So(w.Body.String(), ShouldEqual, "unauthenticated request\n") So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) + So(len(auditMock.RecordCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 0) + auditMock.AssertRecordCalls() }) Convey("When the version document has already been published return status forbidden", t, func() { @@ -1629,14 +2213,21 @@ func TestPutVersionReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, auditMock, genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusForbidden) So(w.Body.String(), ShouldEqual, "unable to update version as it has been published\n") So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) }) Convey("When the request body is invalid return status bad request", t, func() { @@ -1667,17 +2258,24 @@ func TestPutVersionReturnsError(t *testing.T) { }, } - api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, getMockAuditor(), genericMockedObservationStore) + auditMock := audit_mock.New() + api := GetAPIWithMockedDatastore(mockedDataStore, generatorMock, auditMock, genericMockedObservationStore) + api.Router.ServeHTTP(w, r) - api.router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldEqual, "Missing collection_id for association between version and a collection\n") + So(w.Body.String(), ShouldEqual, "missing collection_id for association between version and a collection\n") So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 2) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateVersionCalls()), ShouldEqual, 0) So(len(generatorMock.GenerateCalls()), ShouldEqual, 0) + + So(len(auditMock.RecordCalls()), ShouldEqual, 2) + auditMock.AssertRecordCalls( + audit_mock.Expected{Action: updateVersionAction, Result: audit.Attempted, Params: ap}, + audit_mock.Expected{Action: updateVersionAction, Result: audit.Unsuccessful, Params: ap}, + ) }) } @@ -1796,3 +2394,8 @@ func TestCreateNewVersionDoc(t *testing.T) { So(version.Links.Spatial, ShouldBeNil) }) } + +func assertInternalServerErr(w *httptest.ResponseRecorder) { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(strings.TrimSpace(w.Body.String()), ShouldEqual, errs.ErrInternalServer.Error()) +} diff --git a/api/webendpoints_test.go b/api/webendpoints_test.go index d84985a8..3b14f5d4 100644 --- a/api/webendpoints_test.go +++ b/api/webendpoints_test.go @@ -7,8 +7,8 @@ import ( "net/http/httptest" "testing" - "github.com/ONSdigital/go-ns/audit" - "gopkg.in/mgo.v2/bson" + "github.com/ONSdigital/go-ns/audit/audit_mock" + "github.com/gedge/mgo/bson" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/mocks" @@ -23,15 +23,13 @@ import ( // published datasets are returned, even if the secret token is set. func TestWebSubnetDatasetsEndpoint(t *testing.T) { - t.Parallel() - - current := &models.Dataset{ID: "1234", Title: "current"} - next := &models.Dataset{ID: "4321", Title: "next"} - Convey("When the API is started with private endpoints disabled", t, func() { r, err := createRequestWithAuth("GET", "http://localhost:22000/datasets", nil) So(err, ShouldBeNil) + current := &models.Dataset{ID: "1234", Title: "current"} + next := &models.Dataset{ID: "4321", Title: "next"} + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetsFunc: func() ([]models.DatasetUpdate, error) { @@ -43,9 +41,9 @@ func TestWebSubnetDatasetsEndpoint(t *testing.T) { } Convey("Calling the datasets endpoint should allow only published items", func() { - api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, audit_mock.New(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) a, _ := ioutil.ReadAll(w.Body) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetDatasetsCalls()), ShouldEqual, 1) @@ -59,15 +57,13 @@ func TestWebSubnetDatasetsEndpoint(t *testing.T) { } func TestWebSubnetDatasetEndpoint(t *testing.T) { - t.Parallel() - - current := &models.Dataset{ID: "1234", Title: "current"} - next := &models.Dataset{ID: "1234", Title: "next"} - Convey("When the API is started with private endpoints disabled", t, func() { r, err := createRequestWithAuth("GET", "http://localhost:22000/datasets/1234", nil) So(err, ShouldBeNil) + current := &models.Dataset{ID: "1234", Title: "current"} + next := &models.Dataset{ID: "1234", Title: "next"} + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetDatasetFunc: func(ID string) (*models.DatasetUpdate, error) { @@ -79,9 +75,9 @@ func TestWebSubnetDatasetEndpoint(t *testing.T) { } Convey("Calling the dataset endpoint should allow only published items", func() { - api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, audit_mock.New(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) a, _ := ioutil.ReadAll(w.Body) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) @@ -93,15 +89,13 @@ func TestWebSubnetDatasetEndpoint(t *testing.T) { } func TestWebSubnetEditionsEndpoint(t *testing.T) { - t.Parallel() - - edition := &models.EditionUpdate{ID: "1234", Current: &models.Edition{State: models.PublishedState}} - var editionSearchState, datasetSearchState string - Convey("When the API is started with private endpoints disabled", t, func() { r, err := createRequestWithAuth("GET", "http://localhost:22000/datasets/1234/editions", nil) So(err, ShouldBeNil) + edition := &models.EditionUpdate{ID: "1234", Current: &models.Edition{State: models.PublishedState}} + var editionSearchState, datasetSearchState string + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(ID, state string) error { @@ -117,9 +111,9 @@ func TestWebSubnetEditionsEndpoint(t *testing.T) { } Convey("Calling the editions endpoint should allow only published items", func() { - api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, audit_mock.New(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(datasetSearchState, ShouldEqual, models.PublishedState) So(editionSearchState, ShouldEqual, models.PublishedState) @@ -128,15 +122,13 @@ func TestWebSubnetEditionsEndpoint(t *testing.T) { } func TestWebSubnetEditionEndpoint(t *testing.T) { - t.Parallel() - - edition := &models.EditionUpdate{ID: "1234", Current: &models.Edition{State: models.PublishedState}} - var editionSearchState, datasetSearchState string - Convey("When the API is started with private endpoints disabled", t, func() { r, err := createRequestWithAuth("GET", "http://localhost:22000/datasets/1234/editions/1234", nil) So(err, ShouldBeNil) + edition := &models.EditionUpdate{ID: "1234", Current: &models.Edition{State: models.PublishedState}} + var editionSearchState, datasetSearchState string + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(ID, state string) error { @@ -150,9 +142,9 @@ func TestWebSubnetEditionEndpoint(t *testing.T) { } Convey("Calling the edition endpoint should allow only published items", func() { - api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, audit_mock.New(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(datasetSearchState, ShouldEqual, models.PublishedState) So(editionSearchState, ShouldEqual, models.PublishedState) @@ -161,14 +153,11 @@ func TestWebSubnetEditionEndpoint(t *testing.T) { } func TestWebSubnetVersionsEndpoint(t *testing.T) { - t.Parallel() - - var versionSearchState, editionSearchState, datasetSearchState string - Convey("When the API is started with private endpoints disabled", t, func() { r, err := createRequestWithAuth("GET", "http://localhost:22000/datasets/1234/editions/1234/versions", nil) So(err, ShouldBeNil) + var versionSearchState, editionSearchState, datasetSearchState string w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(ID, state string) error { @@ -188,9 +177,9 @@ func TestWebSubnetVersionsEndpoint(t *testing.T) { } Convey("Calling the versions endpoint should allow only published items", func() { - api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, audit_mock.New(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(datasetSearchState, ShouldEqual, models.PublishedState) So(editionSearchState, ShouldEqual, models.PublishedState) @@ -200,14 +189,11 @@ func TestWebSubnetVersionsEndpoint(t *testing.T) { } func TestWebSubnetVersionEndpoint(t *testing.T) { - t.Parallel() - - var versionSearchState, editionSearchState, datasetSearchState string - Convey("When the API is started with private endpoints disabled", t, func() { r, err := createRequestWithAuth("GET", "http://localhost:22000/datasets/1234/editions/1234/versions/1234", nil) So(err, ShouldBeNil) + var versionSearchState, editionSearchState, datasetSearchState string w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ CheckDatasetExistsFunc: func(ID, state string) error { @@ -228,9 +214,9 @@ func TestWebSubnetVersionEndpoint(t *testing.T) { } Convey("Calling the version endpoint should allow only published items", func() { - api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, audit_mock.New(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(datasetSearchState, ShouldEqual, models.PublishedState) @@ -241,14 +227,11 @@ func TestWebSubnetVersionEndpoint(t *testing.T) { } func TestWebSubnetDimensionsEndpoint(t *testing.T) { - t.Parallel() - - var versionSearchState string - Convey("When the API is started with private endpoints disabled", t, func() { r, err := createRequestWithAuth("GET", "http://localhost:22000/datasets/1234/editions/1234/versions/1234/dimensions", nil) So(err, ShouldBeNil) + var versionSearchState string w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetVersionFunc: func(id string, editionID, version string, state string) (*models.Version, error) { @@ -264,9 +247,9 @@ func TestWebSubnetDimensionsEndpoint(t *testing.T) { } Convey("Calling dimension endpoint should allow only published items", func() { - api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, audit_mock.New(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(versionSearchState, ShouldEqual, models.PublishedState) }) @@ -274,14 +257,11 @@ func TestWebSubnetDimensionsEndpoint(t *testing.T) { } func TestWebSubnetDimensionOptionsEndpoint(t *testing.T) { - t.Parallel() - - var versionSearchState string - Convey("When the API is started with private endpoints disabled", t, func() { r, err := createRequestWithAuth("GET", "http://localhost:22000/datasets/1234/editions/1234/versions/1234/dimensions/t/options", nil) So(err, ShouldBeNil) + var versionSearchState string w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetVersionFunc: func(id string, editionID, version string, state string) (*models.Version, error) { @@ -298,9 +278,9 @@ func TestWebSubnetDimensionOptionsEndpoint(t *testing.T) { Convey("Calling dimension option endpoint should allow only published items", func() { - api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, audit_mock.New(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(versionSearchState, ShouldEqual, models.PublishedState) }) @@ -308,7 +288,6 @@ func TestWebSubnetDimensionOptionsEndpoint(t *testing.T) { } func TestPublishedSubnetEndpointsAreDisabled(t *testing.T) { - t.Parallel() type testEndpoint struct { Method string @@ -348,16 +327,16 @@ func TestPublishedSubnetEndpointsAreDisabled(t *testing.T) { w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{} - api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, getMockAuditor(), genericMockedObservationStore) + api := GetWebAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, audit_mock.New(), genericMockedObservationStore) - api.router.ServeHTTP(w, r) + api.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) }) } }) } -func GetWebAPIWithMockedDatastore(mockedDataStore store.Storer, mockedGeneratedDownloads DownloadsGenerator, mockAuditor *audit.AuditorServiceMock, mockedObservationStore ObservationStore) *DatasetAPI { +func GetWebAPIWithMockedDatastore(mockedDataStore store.Storer, mockedGeneratedDownloads DownloadsGenerator, mockAuditor Auditor, mockedObservationStore ObservationStore) *DatasetAPI { cfg, err := config.Get() So(err, ShouldBeNil) @@ -366,5 +345,5 @@ func GetWebAPIWithMockedDatastore(mockedDataStore store.Storer, mockedGeneratedD cfg.EnablePrivateEnpoints = false cfg.HealthCheckTimeout = healthTimeout - return routes(*cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedGeneratedDownloads, mockAuditor, mockedObservationStore) + return Routes(*cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedGeneratedDownloads, mockAuditor, mockedObservationStore) } diff --git a/apierrors/errors.go b/apierrors/errors.go index 8a85b532..7d841e28 100644 --- a/apierrors/errors.go +++ b/apierrors/errors.go @@ -2,30 +2,52 @@ package apierrors import "errors" -// Error messages for Dataset API +// A list of error messages for Dataset API var ( - ErrDatasetNotFound = errors.New("Dataset not found") ErrAddDatasetAlreadyExists = errors.New("forbidden - dataset already exists") - ErrAddUpdateDatasetBadRequest = errors.New("Failed to parse json body") - ErrEditionNotFound = errors.New("Edition not found") - ErrVersionNotFound = errors.New("Version not found") - ErrDimensionNodeNotFound = errors.New("Dimension node not found") - ErrDimensionNotFound = errors.New("Dimension not found") - ErrDimensionsNotFound = errors.New("Dimensions not found") - ErrInstanceNotFound = errors.New("Instance not found") - ErrUnauthorised = errors.New("Unauthorised access to API") - ErrNoAuthHeader = errors.New("No authentication header provided") - ErrResourceState = errors.New("Incorrect resource state") - ErrVersionMissingState = errors.New("Missing state from version") - ErrInternalServer = errors.New("internal error") - ErrObservationsNotFound = errors.New("No observations found") + ErrAddUpdateDatasetBadRequest = errors.New("failed to parse json body") + ErrAuditActionAttemptedFailure = errors.New("internal server error") + ErrDatasetNotFound = errors.New("dataset not found") + ErrDeleteDatasetNotFound = errors.New("dataset not found") + ErrDeletePublishedDatasetForbidden = errors.New("a published dataset cannot be deleted") + ErrDimensionNodeNotFound = errors.New("dimension node not found") + ErrDimensionNotFound = errors.New("dimension not found") + ErrDimensionOptionNotFound = errors.New("dimension option not found") + ErrDimensionsNotFound = errors.New("dimensions not found") + ErrEditionNotFound = errors.New("edition not found") ErrIndexOutOfRange = errors.New("index out of range") + ErrInstanceNotFound = errors.New("instance not found") + ErrInternalServer = errors.New("internal error") + ErrMetadataVersionNotFound = errors.New("version not found") + ErrMissingJobProperties = errors.New("missing job properties") + ErrMissingParameters = errors.New("missing properties in JSON") ErrMissingVersionHeadersOrDimensions = errors.New("missing headers or dimensions or both from version doc") + ErrNoAuthHeader = errors.New("no authentication header provided") + ErrObservationsNotFound = errors.New("no observations found") + ErrResourcePublished = errors.New("unable to update resource as it has been published") + ErrResourceState = errors.New("incorrect resource state") ErrTooManyWildcards = errors.New("only one wildcard (*) is allowed as a value in selected query parameters") - ErrDeletePublishedDatasetForbidden = errors.New("a published dataset cannot be deleted") - ErrDeleteDatasetNotFound = errors.New("dataset not found") - ErrAuditActionAttemptedFailure = errors.New("internal server error") + ErrUnableToReadMessage = errors.New("failed to read message body") + ErrUnableToParseJSON = errors.New("failed to parse json body") + ErrUnauthorised = errors.New("unauthorised access to API") + ErrVersionBadRequest = errors.New("failed to parse json body") + ErrVersionMissingState = errors.New("missing state from version") + ErrVersionNotFound = errors.New("version not found") + + NotFoundMap = map[error]bool{ + ErrDatasetNotFound: true, + ErrDimensionNotFound: true, + ErrDimensionsNotFound: true, + ErrDimensionNodeNotFound: true, + ErrDimensionOptionNotFound: true, + ErrEditionNotFound: true, + ErrInstanceNotFound: true, + ErrVersionNotFound: true, + } - // metadata endpoint errors - ErrMetadataVersionNotFound = errors.New("Version not found") + BadRequestMap = map[error]bool{ + ErrMissingParameters: true, + ErrUnableToParseJSON: true, + ErrUnableToReadMessage: true, + } ) diff --git a/ci/build.yml b/ci/build.yml index ddeeae45..59c07f0b 100644 --- a/ci/build.yml +++ b/ci/build.yml @@ -6,7 +6,7 @@ image_resource: type: docker-image source: repository: golang - tag: 1.10.0 + tag: 1.10.2 inputs: - name: dp-dataset-api diff --git a/ci/unit.yml b/ci/unit.yml index d1bb60e8..a86bd9c9 100644 --- a/ci/unit.yml +++ b/ci/unit.yml @@ -6,7 +6,7 @@ image_resource: type: docker-image source: repository: golang - tag: 1.10.0 + tag: 1.10.2 inputs: - name: dp-dataset-api diff --git a/dimension/dimension.go b/dimension/dimension.go index 3677fa31..eb2816ad 100644 --- a/dimension/dimension.go +++ b/dimension/dimension.go @@ -1,204 +1,271 @@ package dimension import ( + "context" "encoding/json" + "fmt" "net/http" - "errors" - "io" - "io/ioutil" - - errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/store" + "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/common" "github.com/ONSdigital/go-ns/log" "github.com/gorilla/mux" + "github.com/pkg/errors" ) -//Store provides a backend for dimensions +// Store provides a backend for dimensions type Store struct { + Auditor audit.AuditorService store.Storer } -//GetNodes list from a specified instance -func (s *Store) GetNodes(w http.ResponseWriter, r *http.Request) { +// List of audit actions for dimensions +const ( + GetDimensions = "getInstanceDimensions" + GetUniqueDimensionAndOptions = "getInstanceUniqueDimensionAndOptions" + PostDimensionsAction = "postDimensions" + PutNodeIDAction = "putNodeID" +) + +func dimensionError(err error, message, action string) error { + return errors.WithMessage(err, fmt.Sprintf("%v endpoint: %v", action, message)) +} + +// GetDimensionsHandler returns a list of all dimensions and their options for an instance resource +func (s *Store) GetDimensionsHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) - id := vars["id"] + instanceID := vars["id"] + logData := log.Data{"instance_id": instanceID} + auditParams := common.Params{"instance_id": instanceID} - // Get instance - instance, err := s.GetInstance(id) + if auditErr := s.Auditor.Record(ctx, GetDimensions, audit.Attempted, auditParams); auditErr != nil { + handleDimensionErr(ctx, w, auditErr, logData) + return + } + + b, err := s.getDimensions(ctx, instanceID, logData) if err != nil { - log.ErrorC("failed to GET instance", err, log.Data{"instance": id}) - handleErrorType(err, w) + if auditErr := s.Auditor.Record(ctx, GetDimensions, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr + } + + handleDimensionErr(ctx, w, err, logData) return } + if auditErr := s.Auditor.Record(ctx, GetDimensions, audit.Successful, auditParams); auditErr != nil { + handleDimensionErr(ctx, w, auditErr, logData) + return + } + + writeBody(ctx, w, b, GetDimensions, logData) + log.InfoCtx(ctx, fmt.Sprintf("%v endpoint: successfully get dimensions for an instance resource", GetDimensions), logData) +} + +func (s *Store) getDimensions(ctx context.Context, instanceID string, logData log.Data) ([]byte, error) { + instance, err := s.GetInstance(instanceID) + if err != nil { + log.ErrorCtx(ctx, dimensionError(err, "failed to get instance", GetDimensions), logData) + return nil, err + } + // Early return if instance state is invalid if err = models.CheckState("instance", instance.State); err != nil { - log.ErrorC("current instance has an invalid state", err, log.Data{"state": instance.State}) - handleErrorType(errs.ErrInternalServer, w) - return + logData["state"] = instance.State + log.ErrorCtx(ctx, dimensionError(err, "current instance has an invalid state", GetDimensions), logData) + return nil, err } - results, err := s.GetDimensionNodesFromInstance(id) + results, err := s.GetDimensionsFromInstance(instanceID) if err != nil { - log.Error(err, nil) - handleErrorType(err, w) - return + log.ErrorCtx(ctx, dimensionError(err, "failed to get dimension options for instance", GetDimensions), logData) + return nil, err } b, err := json.Marshal(results) if err != nil { - internalError(w, err) - return + log.ErrorCtx(ctx, dimensionError(err, "failed to marshal dimension nodes to json", GetDimensions), logData) + return nil, err } - writeBody(w, b) - log.Debug("get dimension nodes", log.Data{"instance": id}) + return b, nil } -//GetUnique dimension values from a specified dimension -func (s *Store) GetUnique(w http.ResponseWriter, r *http.Request) { +// GetUniqueDimensionAndOptionsHandler returns a list of dimension options for a dimension of an instance +func (s *Store) GetUniqueDimensionAndOptionsHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) - id := vars["id"] + instanceID := vars["id"] dimension := vars["dimension"] + logData := log.Data{"instance_id": instanceID, "dimension": dimension} + auditParams := common.Params{"instance_id": instanceID, "dimension": dimension} - // Get instance - instance, err := s.GetInstance(id) + if auditErr := s.Auditor.Record(ctx, GetUniqueDimensionAndOptions, audit.Attempted, auditParams); auditErr != nil { + handleDimensionErr(ctx, w, auditErr, logData) + return + } + + b, err := s.getUniqueDimensionAndOptions(ctx, instanceID, dimension, logData) if err != nil { - log.ErrorC("failed to GET instance", err, log.Data{"instance": id}) - handleErrorType(err, w) + if auditErr := s.Auditor.Record(ctx, GetUniqueDimensionAndOptions, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr + } + + handleDimensionErr(ctx, w, err, logData) return } + if auditErr := s.Auditor.Record(ctx, GetUniqueDimensionAndOptions, audit.Successful, auditParams); auditErr != nil { + handleDimensionErr(ctx, w, auditErr, logData) + return + } + + writeBody(ctx, w, b, GetUniqueDimensionAndOptions, logData) + log.InfoCtx(ctx, fmt.Sprintf("%v endpoint: successfully get unique dimension options for an instance resource", GetUniqueDimensionAndOptions), logData) +} + +func (s *Store) getUniqueDimensionAndOptions(ctx context.Context, instanceID, dimension string, logData log.Data) ([]byte, error) { + instance, err := s.GetInstance(instanceID) + if err != nil { + log.ErrorCtx(ctx, dimensionError(err, "failed to get instance", GetUniqueDimensionAndOptions), logData) + return nil, err + } + // Early return if instance state is invalid if err = models.CheckState("instance", instance.State); err != nil { - log.ErrorC("current instance has an invalid state", err, log.Data{"state": instance.State}) - handleErrorType(errs.ErrInternalServer, w) - return + logData["state"] = instance.State + log.ErrorCtx(ctx, dimensionError(err, "current instance has an invalid state", GetUniqueDimensionAndOptions), logData) + return nil, err } - values, err := s.GetUniqueDimensionValues(id, dimension) + values, err := s.GetUniqueDimensionAndOptions(instanceID, dimension) if err != nil { - log.Error(err, nil) - handleErrorType(err, w) - return + log.ErrorCtx(ctx, dimensionError(err, "failed to get unique dimension values for instance", GetUniqueDimensionAndOptions), logData) + return nil, err } b, err := json.Marshal(values) if err != nil { - internalError(w, err) - return + log.ErrorCtx(ctx, dimensionError(err, "failed to marshal dimension values to json", GetUniqueDimensionAndOptions), logData) + return nil, err } - writeBody(w, b) - log.Debug("get dimension values", log.Data{"instance": id}) + return b, nil } -// Add represents adding a dimension to a specific instance -func (s *Store) Add(w http.ResponseWriter, r *http.Request) { +// AddHandler represents adding a dimension to a specific instance +func (s *Store) AddHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + vars := mux.Vars(r) - id := vars["id"] + instanceID := vars["id"] + logData := log.Data{"instance_id": instanceID} + auditParams := common.Params{"instance_id": instanceID} - // Get instance - instance, err := s.GetInstance(id) + option, err := unmarshalDimensionCache(r.Body) if err != nil { - log.ErrorC("Failed to GET instance", err, log.Data{"instance": id}) - handleErrorType(err, w) + log.ErrorCtx(ctx, dimensionError(err, "failed to unmarshal dimension cache", PostDimensionsAction), logData) + + if auditErr := s.Auditor.Record(ctx, PostDimensionsAction, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr + } + + handleDimensionErr(ctx, w, err, logData) return } - // Early return if instance state is invalid - if err = models.CheckState("instance", instance.State); err != nil { - log.ErrorC("current instance has an invalid state", err, log.Data{"state": instance.State}) - handleErrorType(errs.ErrInternalServer, w) + if err := s.add(ctx, instanceID, option, logData); err != nil { + if auditErr := s.Auditor.Record(ctx, PostDimensionsAction, audit.Unsuccessful, auditParams); auditErr != nil { + err = auditErr + } + + handleDimensionErr(ctx, w, err, logData) return } - option, err := unmarshalDimensionCache(r.Body) + s.Auditor.Record(ctx, PostDimensionsAction, audit.Successful, auditParams) + + log.InfoCtx(ctx, "added dimension to instance resource", logData) +} + +func (s *Store) add(ctx context.Context, instanceID string, option *models.CachedDimensionOption, logData log.Data) error { + // Get instance + instance, err := s.GetInstance(instanceID) if err != nil { - log.Error(err, nil) - http.Error(w, err.Error(), http.StatusBadRequest) - return + log.ErrorCtx(ctx, dimensionError(err, "failed to get instance", PostDimensionsAction), logData) + return err + } + + // Early return if instance state is invalid + if err = models.CheckState("instance", instance.State); err != nil { + logData["state"] = instance.State + log.ErrorCtx(ctx, dimensionError(err, "current instance has an invalid state", PostDimensionsAction), logData) + return err } - option.InstanceID = id + + option.InstanceID = instanceID if err := s.AddDimensionToInstance(option); err != nil { - log.Error(err, nil) - handleErrorType(err, w) + log.ErrorCtx(ctx, dimensionError(err, "failed to upsert dimension for an instance", PostDimensionsAction), logData) + return err } + + return nil } -//AddNodeID against a specific value for dimension -func (s *Store) AddNodeID(w http.ResponseWriter, r *http.Request) { +// AddNodeIDHandler against a specific value for dimension +func (s *Store) AddNodeIDHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) - id := vars["id"] + instanceID := vars["id"] dimensionName := vars["dimension"] value := vars["value"] nodeID := vars["node_id"] + logData := log.Data{"instance_id": instanceID, "dimension_name": dimensionName, "option": value, "node_id": nodeID} + auditParams := common.Params{"instance_id": instanceID, "dimension_name": dimensionName, "option": value, "node_id": nodeID} - // Get instance - instance, err := s.GetInstance(id) - if err != nil { - log.ErrorC("Failed to GET instance when attempting to update a dimension of that instance.", err, log.Data{"instance": id}) - handleErrorType(err, w) - return - } + dim := models.DimensionOption{Name: dimensionName, Option: value, NodeID: nodeID, InstanceID: instanceID} - // Early return if instance state is invalid - if err = models.CheckState("instance", instance.State); err != nil { - log.ErrorC("current instance has an invalid state", err, log.Data{"state": instance.State}) - handleErrorType(errs.ErrInternalServer, w) + if err := s.addNodeID(ctx, dim, logData); err != nil { + s.Auditor.Record(ctx, PutNodeIDAction, audit.Unsuccessful, auditParams) + handleDimensionErr(ctx, w, err, logData) return } - dim := models.DimensionOption{Name: dimensionName, Option: value, NodeID: nodeID, InstanceID: id} - if err := s.UpdateDimensionNodeID(&dim); err != nil { - log.Error(err, nil) - handleErrorType(err, w) - } + s.Auditor.Record(ctx, PutNodeIDAction, audit.Successful, auditParams) + + log.InfoCtx(ctx, "added node id to dimension of an instance resource", logData) } -// CreateDataset manages the creation of a dataset from a reader -func unmarshalDimensionCache(reader io.Reader) (*models.CachedDimensionOption, error) { - b, err := ioutil.ReadAll(reader) +func (s *Store) addNodeID(ctx context.Context, dim models.DimensionOption, logData log.Data) error { + // Get instance + instance, err := s.GetInstance(dim.InstanceID) if err != nil { - return nil, errors.New("Failed to read message body") + log.ErrorCtx(ctx, dimensionError(err, "failed to get instance", PutNodeIDAction), logData) + return err } - var option models.CachedDimensionOption - - err = json.Unmarshal(b, &option) - if err != nil { - return nil, errors.New("Failed to parse json body") - - } - if option.Name == "" || (option.Option == "" && option.CodeList == "") { - return nil, errors.New("Missing properties in JSON") + // Early return if instance state is invalid + if err = models.CheckState("instance", instance.State); err != nil { + logData["state"] = instance.State + log.ErrorCtx(ctx, dimensionError(err, "current instance has an invalid state", PutNodeIDAction), logData) + return err } - return &option, nil -} - -func handleErrorType(err error, w http.ResponseWriter) { - status := http.StatusInternalServerError - - if err == errs.ErrDatasetNotFound || err == errs.ErrEditionNotFound || err == errs.ErrVersionNotFound || err == errs.ErrDimensionNodeNotFound || err == errs.ErrInstanceNotFound { - status = http.StatusNotFound + if err := s.UpdateDimensionNodeID(&dim); err != nil { + log.ErrorCtx(ctx, dimensionError(err, "failed to update a dimension of that instance", PutNodeIDAction), logData) + return err } - http.Error(w, err.Error(), status) - -} - -func internalError(w http.ResponseWriter, err error) { - log.Error(err, nil) - http.Error(w, err.Error(), http.StatusInternalServerError) + return nil } -func writeBody(w http.ResponseWriter, b []byte) { +func writeBody(ctx context.Context, w http.ResponseWriter, b []byte, action string, data log.Data) { w.Header().Set("Content-Type", "application/json") if _, err := w.Write(b); err != nil { - log.Error(err, nil) + log.ErrorCtx(ctx, dimensionError(err, "failed to write response body", action), data) http.Error(w, err.Error(), http.StatusInternalServerError) } } diff --git a/dimension/dimension_test.go b/dimension/dimension_test.go index 0e0baf67..44d1f47e 100644 --- a/dimension/dimension_test.go +++ b/dimension/dimension_test.go @@ -1,31 +1,49 @@ package dimension_test import ( + "context" + "errors" "io" "net/http" "net/http/httptest" "strings" "testing" + "time" + "github.com/ONSdigital/dp-dataset-api/api" errs "github.com/ONSdigital/dp-dataset-api/apierrors" + "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/dimension" + "github.com/ONSdigital/dp-dataset-api/mocks" "github.com/ONSdigital/dp-dataset-api/models" + "github.com/ONSdigital/dp-dataset-api/store" "github.com/ONSdigital/dp-dataset-api/store/datastoretest" + "github.com/ONSdigital/dp-dataset-api/url" + "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/audit/audit_mock" + "github.com/ONSdigital/go-ns/common" + "github.com/gorilla/mux" . "github.com/smartystreets/goconvey/convey" ) -const secretKey = "coffee" +var ( + urlBuilder = url.NewBuilder("localhost:20000") +) -func createRequestWithToken(method, url string, body io.Reader) *http.Request { - r := httptest.NewRequest(method, url, body) - r.Header.Add("internal-token", secretKey) - return r +func createRequestWithToken(method, url string, body io.Reader) (*http.Request, error) { + r, err := http.NewRequest(method, url, body) + ctx := r.Context() + ctx = common.SetCaller(ctx, "someone@ons.gov.uk") + r = r.WithContext(ctx) + return r, err } func TestAddNodeIDToDimensionReturnsOK(t *testing.T) { t.Parallel() Convey("Add node id to a dimension returns ok", t, func() { - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -37,19 +55,38 @@ func TestAddNodeIDToDimensionReturnsOK(t *testing.T) { }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.AddNodeID(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 1) + + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Attempted, + Params: common.Params{"instance_id": "123"}, + }, + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Successful, + Params: common.Params{"dimension_name": "age", "instance_id": "123", "node_id": "11", "option": "55"}, + }, + ) }) } func TestAddNodeIDToDimensionReturnsBadRequest(t *testing.T) { t.Parallel() Convey("Add node id to a dimension returns bad request", t, func() { - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -61,19 +98,38 @@ func TestAddNodeIDToDimensionReturnsBadRequest(t *testing.T) { }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.AddNodeID(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 1) + + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Attempted, + Params: common.Params{"instance_id": "123"}, + }, + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Unsuccessful, + Params: common.Params{"dimension_name": "age", "instance_id": "123", "node_id": "11", "option": "55"}, + }, + ) }) } func TestAddNodeIDToDimensionReturnsInternalError(t *testing.T) { t.Parallel() Convey("Given an internal error is returned from mongo, then response returns an internal error", t, func() { - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -82,16 +138,34 @@ func TestAddNodeIDToDimensionReturnsInternalError(t *testing.T) { }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.AddNodeID(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 0) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Unsuccessful, + Params: p, + }, + ) }) Convey("Given instance state is invalid, then response returns an internal error", t, func() { - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -100,21 +174,231 @@ func TestAddNodeIDToDimensionReturnsInternalError(t *testing.T) { }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.AddNodeID(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) + So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 0) + + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Attempted, + Params: common.Params{"instance_id": "123"}, + }, + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Unsuccessful, + Params: common.Params{"dimension_name": "age", "instance_id": "123", "node_id": "11", "option": "55"}, + }, + ) + }) +} + +func TestAddNodeIDToDimensionReturnsForbidden(t *testing.T) { + t.Parallel() + Convey("Add node id to a dimension of a published instance returns forbidden", t, func() { + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.PublishedState}, nil + }, + } + + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusForbidden) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 0) + + calls := auditorMock.RecordCalls() + So(len(calls), ShouldEqual, 2) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Unsuccessful, + Params: p, + }, + ) + }) +} + +func TestAddNodeIDToDimensionReturnsUnauthorized(t *testing.T) { + t.Parallel() + Convey("Add node id to a dimension of an instance returns unauthorized", t, func() { + r, err := http.NewRequest("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.PublishedState}, nil + }, + } + + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusUnauthorized) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 0) + + auditorMock.AssertRecordCalls() + }) +} + +func TestAddNodeIDToDimensionAuditFailure(t *testing.T) { + t.Parallel() + Convey("When auditing add node id to dimension attempt fails return an error of internal server error", t, func() { + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return nil, nil + }, + } + + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 0) + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Attempted, + Params: common.Params{"instance_id": "123"}, + }) + }) + + Convey("When request to add node id to dimension is forbidden but audit fails returns an error of internal server error", t, func() { + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.PublishedState}, nil + }, + } + + count := 1 + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + if count == 1 { + count++ + return nil + } + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Unsuccessful, + Params: p, + }, + ) + }) + + Convey("When request to add node id to dimension and audit fails to send success message return 200 response", t, func() { + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions/age/options/55/node_id/11", nil) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateDimensionNodeIDFunc: func(event *models.DimensionOption) error { + return nil + }, + } + + count := 1 + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + if count <= 2 { + count++ + return nil + } + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusOK) + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) + So(len(mockedDataStore.UpdateDimensionNodeIDCalls()), ShouldEqual, 1) + + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Attempted, + Params: common.Params{"instance_id": "123"}, + }, + audit_mock.Expected{ + Action: dimension.PutNodeIDAction, + Result: audit.Successful, + Params: common.Params{"dimension_name": "age", "instance_id": "123", "node_id": "11", "option": "55"}, + }, + ) }) } func TestAddDimensionToInstanceReturnsOk(t *testing.T) { t.Parallel() Convey("Add a dimension to an instance returns ok", t, func() { - w := httptest.NewRecorder() json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test"}`) - r := createRequestWithToken("POST", "http://localhost:21800/instances/123/dimensions", json) + r, err := createRequestWithToken("POST", "http://localhost:22000/instances/123/dimensions", json) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(ID string) (*models.Instance, error) { return &models.Instance{State: models.CreatedState}, nil @@ -124,12 +408,33 @@ func TestAddDimensionToInstanceReturnsOk(t *testing.T) { }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.Add(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.AddDimensionToInstanceCalls()), ShouldEqual, 1) + + calls := auditorMock.RecordCalls() + So(len(calls), ShouldEqual, 2) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Successful, + Params: p, + }, + ) }) } @@ -137,7 +442,9 @@ func TestAddDimensionToInstanceReturnsNotFound(t *testing.T) { t.Parallel() Convey("Add a dimension to an instance returns not found", t, func() { json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test"}`) - r := createRequestWithToken("POST", "http://localhost:21800/instances/123/dimensions", json) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/dimensions", json) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -149,12 +456,104 @@ func TestAddDimensionToInstanceReturnsNotFound(t *testing.T) { }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.Add(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(w.Body.String(), ShouldContainSubstring, errs.ErrDimensionNodeNotFound.Error()) + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.AddDimensionToInstanceCalls()), ShouldEqual, 1) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Unsuccessful, + Params: p, + }, + ) + }) +} + +func TestAddDimensionToInstanceReturnsForbidden(t *testing.T) { + t.Parallel() + Convey("Add a dimension to a published instance returns forbidden", t, func() { + json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test"}`) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/dimensions", json) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.PublishedState}, nil + }, + AddDimensionToInstanceFunc: func(event *models.CachedDimensionOption) error { + return nil + }, + } + + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusForbidden) + So(w.Body.String(), ShouldContainSubstring, errs.ErrResourcePublished.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.AddDimensionToInstanceCalls()), ShouldEqual, 0) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Unsuccessful, + Params: p, + }, + ) + }) +} + +func TestAddDimensionToInstanceReturnsUnauthorized(t *testing.T) { + t.Parallel() + Convey("Add a dimension to a instance returns unauthorized", t, func() { + json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test"}`) + r, err := http.NewRequest("POST", "http://localhost:21800/instances/123/dimensions", json) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + } + + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusUnauthorized) + So(w.Body.String(), ShouldContainSubstring, "unauthenticated request") + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 0) + + calls := auditorMock.RecordCalls() + So(len(calls), ShouldEqual, 0) }) } @@ -162,7 +561,9 @@ func TestAddDimensionToInstanceReturnsInternalError(t *testing.T) { t.Parallel() Convey("Given an internal error is returned from mongo, then response returns an internal error", t, func() { json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test"}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions", json) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/dimensions", json) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -174,17 +575,37 @@ func TestAddDimensionToInstanceReturnsInternalError(t *testing.T) { }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.Add(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.AddDimensionToInstanceCalls()), ShouldEqual, 0) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Unsuccessful, + Params: p, + }, + ) }) Convey("Given instance state is invalid, then response returns an internal error", t, func() { json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test"}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/dimensions", json) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/dimensions", json) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -196,197 +617,766 @@ func TestAddDimensionToInstanceReturnsInternalError(t *testing.T) { }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.Add(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.AddDimensionToInstanceCalls()), ShouldEqual, 0) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Unsuccessful, + Params: p, + }, + ) + }) +} + +func TestAddDimensionAuditFailure(t *testing.T) { + t.Parallel() + Convey("When a valid request to add dimension is made but the audit attempt fails returns an error of internal server error", t, func() { + json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test"}`) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/dimensions", json) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return nil, nil + }, + } + + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 0) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls(audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Attempted, + Params: p, + }) + }) + + Convey("When request to add a dimension is forbidden but audit fails returns an error of internal server error", t, func() { + json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test"}`) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/dimensions", json) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.PublishedState}, nil + }, + } + + count := 1 + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + if count == 1 { + count++ + return nil + } + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Unsuccessful, + Params: p, + }, + ) + }) + + Convey("When request to add dimension and audit fails to send success message return 200 response", t, func() { + json := strings.NewReader(`{"value":"24", "code_list":"123-456", "dimension": "test"}`) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/dimensions", json) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + AddDimensionToInstanceFunc: func(event *models.CachedDimensionOption) error { + return nil + }, + } + + count := 1 + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + if count <= 2 { + count++ + return nil + } + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusOK) + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) + So(len(mockedDataStore.AddDimensionToInstanceCalls()), ShouldEqual, 1) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.PostDimensionsAction, + Result: audit.Successful, + Params: p, + }, + ) }) } -func TestGetDimensionNodesReturnsOk(t *testing.T) { +func TestGetDimensionsReturnsOk(t *testing.T) { t.Parallel() - Convey("Get dimension nodes returns ok", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions", nil) + Convey("Get dimensions (and their respective nodes) returns ok", t, func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(ID string) (*models.Instance, error) { return &models.Instance{State: models.CreatedState}, nil }, - GetDimensionNodesFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { + GetDimensionsFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { return &models.DimensionNodeResults{}, nil }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.GetNodes(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetDimensionNodesFromInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDimensionsFromInstanceCalls()), ShouldEqual, 1) + + calls := auditorMock.RecordCalls() + So(len(calls), ShouldEqual, 2) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Successful, + Params: p, + }, + ) }) } -func TestGetDimensionNodesReturnsNotFound(t *testing.T) { +func TestGetDimensionsReturnsNotFound(t *testing.T) { t.Parallel() - Convey("Get dimension nodes returns not found", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions", nil) + Convey("Get dimensions returns not found", t, func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(ID string) (*models.Instance, error) { return &models.Instance{State: models.CreatedState}, nil }, - GetDimensionNodesFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { - return nil, errs.ErrInstanceNotFound + GetDimensionsFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { + return nil, errs.ErrDimensionNodeNotFound }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.GetNodes(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) + So(w.Body.String(), ShouldContainSubstring, errs.ErrDimensionNodeNotFound.Error()) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetDimensionNodesFromInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDimensionsFromInstanceCalls()), ShouldEqual, 1) + + calls := auditorMock.RecordCalls() + So(len(calls), ShouldEqual, 2) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Unsuccessful, + Params: p, + }, + ) }) } -func TestGetDimensionNodesReturnsInternalError(t *testing.T) { +func TestGetDimensionsAndOptionsReturnsInternalError(t *testing.T) { t.Parallel() Convey("Given an internal error is returned from mongo, then response returns an internal error", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(ID string) (*models.Instance, error) { return nil, errs.ErrInternalServer }, - GetDimensionNodesFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { + GetDimensionsFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { return &models.DimensionNodeResults{}, nil }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.GetNodes(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetDimensionNodesFromInstanceCalls()), ShouldEqual, 0) + So(len(mockedDataStore.GetDimensionsFromInstanceCalls()), ShouldEqual, 0) + + calls := auditorMock.RecordCalls() + So(len(calls), ShouldEqual, 2) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Unsuccessful, + Params: p, + }, + ) }) Convey("Given instance state is invalid, then response returns an internal error", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(ID string) (*models.Instance, error) { return &models.Instance{State: "gobbly gook"}, nil }, - GetDimensionNodesFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { + GetDimensionsFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { return &models.DimensionNodeResults{}, nil }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.GetNodes(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetDimensionNodesFromInstanceCalls()), ShouldEqual, 0) + So(len(mockedDataStore.GetDimensionsFromInstanceCalls()), ShouldEqual, 0) + + calls := auditorMock.RecordCalls() + So(len(calls), ShouldEqual, 2) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Unsuccessful, + Params: p, + }, + ) }) } -func TestGetUniqueDimensionValuesReturnsOk(t *testing.T) { +func TestGetDimensionsAndOptionsAuditFailure(t *testing.T) { + t.Parallel() + Convey("When a request to get a list of dimensions is made but the audit attempt fails return internal server error", t, func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions", nil) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{} + + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls(audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Attempted, + Params: p, + }) + }) + + Convey("When a request to get a list of dimensions is unsuccessful and audit fails returns internal server error", t, func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions", nil) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: "gobbly gook"}, nil + }, + } + + count := 1 + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + if count == 1 { + count++ + return nil + } + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Unsuccessful, + Params: p, + }, + ) + }) + + Convey("When a request to get a list of dimensions is made and audit fails to send success message return internal server error", t, func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions", nil) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + GetDimensionsFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { + return &models.DimensionNodeResults{}, nil + }, + } + + count := 1 + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + if count == 1 { + count++ + return nil + } + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetDimensionsFromInstanceCalls()), ShouldEqual, 1) + + p := common.Params{"instance_id": "123"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetDimensions, + Result: audit.Successful, + Params: p, + }, + ) + }) +} + +func TestGetUniqueDimensionAndOptionsReturnsOk(t *testing.T) { t.Parallel() Convey("Get all unique dimensions returns ok", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions/age/options", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions/age/options", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(ID string) (*models.Instance, error) { return &models.Instance{State: models.CreatedState}, nil }, - GetUniqueDimensionValuesFunc: func(id, dimension string) (*models.DimensionValues, error) { + GetUniqueDimensionAndOptionsFunc: func(id, dimension string) (*models.DimensionValues, error) { return &models.DimensionValues{}, nil }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.GetUnique(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetUniqueDimensionValuesCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetUniqueDimensionAndOptionsCalls()), ShouldEqual, 1) + + calls := auditorMock.RecordCalls() + So(len(calls), ShouldEqual, 2) + + p := common.Params{"instance_id": "123", "dimension": "age"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Successful, + Params: p, + }, + ) }) } -func TestGetUniqueDimensionValuesReturnsNotFound(t *testing.T) { +func TestGetUniqueDimensionAndOptionsReturnsNotFound(t *testing.T) { t.Parallel() Convey("Get all unique dimensions returns not found", t, func() { - r, err := http.NewRequest("GET", "http://localhost:21800/instances/123/dimensions/age/options", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions/age/options", nil) So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(ID string) (*models.Instance, error) { return &models.Instance{State: models.CreatedState}, nil }, - GetUniqueDimensionValuesFunc: func(id, dimension string) (*models.DimensionValues, error) { + GetUniqueDimensionAndOptionsFunc: func(id, dimension string) (*models.DimensionValues, error) { return nil, errs.ErrInstanceNotFound }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.GetUnique(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInstanceNotFound.Error()) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetUniqueDimensionValuesCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetUniqueDimensionAndOptionsCalls()), ShouldEqual, 1) + + calls := auditorMock.RecordCalls() + So(len(calls), ShouldEqual, 2) + + p := common.Params{"instance_id": "123", "dimension": "age"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Unsuccessful, + Params: p, + }, + ) }) } -func TestGetUniqueDimensionValuesReturnsInternalError(t *testing.T) { +func TestGetUniqueDimensionAndOptionsReturnsInternalError(t *testing.T) { t.Parallel() Convey("Given an internal error is returned from mongo, then response returns an internal error", t, func() { - r, err := http.NewRequest("GET", "http://localhost:21800/instances/123/dimensions/age/options", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions/age/options", nil) So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(ID string) (*models.Instance, error) { return nil, errs.ErrInternalServer }, - GetDimensionNodesFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { + GetDimensionsFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { return &models.DimensionNodeResults{}, nil }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.GetUnique(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetUniqueDimensionValuesCalls()), ShouldEqual, 0) + So(len(mockedDataStore.GetUniqueDimensionAndOptionsCalls()), ShouldEqual, 0) + + calls := auditorMock.RecordCalls() + So(len(calls), ShouldEqual, 2) + + p := common.Params{"instance_id": "123", "dimension": "age"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Unsuccessful, + Params: p, + }, + ) }) Convey("Given instance state is invalid, then response returns an internal error", t, func() { - r, err := http.NewRequest("GET", "http://localhost:21800/instances/123/dimensions/age/options", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions/age/options", nil) So(err, ShouldBeNil) + w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(ID string) (*models.Instance, error) { return &models.Instance{State: "gobbly gook"}, nil }, - GetDimensionNodesFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { + GetDimensionsFromInstanceFunc: func(id string) (*models.DimensionNodeResults, error) { return &models.DimensionNodeResults{}, nil }, } - dimension := &dimension.Store{Storer: mockedDataStore} - dimension.GetUnique(w, r) + auditorMock := audit_mock.New() + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetUniqueDimensionAndOptionsCalls()), ShouldEqual, 0) + + calls := auditorMock.RecordCalls() + So(len(calls), ShouldEqual, 2) + + p := common.Params{"instance_id": "123", "dimension": "age"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Unsuccessful, + Params: p, + }, + ) + }) +} + +func TestGetUniqueDimensionAndOptionsAuditFailure(t *testing.T) { + t.Parallel() + Convey("When a request to get unique dimension options is made but the audit attempt fails returns internal server error", t, func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions/age/options", nil) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{} + + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + p := common.Params{"instance_id": "123", "dimension": "age"} + auditorMock.AssertRecordCalls(audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Attempted, + Params: p, + }) + }) + + Convey("When a request to get unique dimension options is unsuccessful and audit fails returns internal server error", t, func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions/age/options", nil) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: "gobbly gook"}, nil + }, + } + + count := 1 + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + if count == 1 { + count++ + return nil + } + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + + p := common.Params{"instance_id": "123", "dimension": "age"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Unsuccessful, + Params: p, + }, + ) + }) + + Convey("When a request to get unique dimension options is made and audit fails to send success message return internal server error", t, func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123/dimensions/age/options", nil) + So(err, ShouldBeNil) + + w := httptest.NewRecorder() + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + GetUniqueDimensionAndOptionsFunc: func(id, dimension string) (*models.DimensionValues, error) { + return &models.DimensionValues{}, nil + }, + } + + count := 1 + auditorMock := audit_mock.New() + auditorMock.RecordFunc = func(ctx context.Context, action string, result string, params common.Params) error { + if count == 1 { + count++ + return nil + } + return errors.New("unable to send message to kafka audit topic") + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditorMock, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) + // Gets called twice as there is a check wrapper around this route which + // checks the instance is not published before entering handler So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetUniqueDimensionValuesCalls()), ShouldEqual, 0) + So(len(mockedDataStore.GetUniqueDimensionAndOptionsCalls()), ShouldEqual, 1) + + p := common.Params{"instance_id": "123", "dimension": "age"} + auditorMock.AssertRecordCalls( + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Attempted, + Params: p, + }, + audit_mock.Expected{ + Action: dimension.GetUniqueDimensionAndOptions, + Result: audit.Successful, + Params: p, + }, + ) }) } + +func getAPIWithMockedDatastore(mockedDataStore store.Storer, mockedGeneratedDownloads api.DownloadsGenerator, mockAuditor api.Auditor, mockedObservationStore api.ObservationStore) *api.DatasetAPI { + cfg, err := config.Get() + So(err, ShouldBeNil) + cfg.ServiceAuthToken = "dataset" + cfg.DatasetAPIURL = "http://localhost:22000" + cfg.EnablePrivateEnpoints = true + cfg.HealthCheckTimeout = 2 * time.Second + + return api.Routes(*cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedGeneratedDownloads, mockAuditor, mockedObservationStore) +} diff --git a/dimension/helpers.go b/dimension/helpers.go new file mode 100644 index 00000000..f04d794f --- /dev/null +++ b/dimension/helpers.go @@ -0,0 +1,57 @@ +package dimension + +import ( + "context" + "encoding/json" + "io" + "io/ioutil" + "net/http" + + errs "github.com/ONSdigital/dp-dataset-api/apierrors" + "github.com/ONSdigital/dp-dataset-api/models" + "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/log" + "github.com/pkg/errors" +) + +func unmarshalDimensionCache(reader io.Reader) (*models.CachedDimensionOption, error) { + b, err := ioutil.ReadAll(reader) + if err != nil { + return nil, errs.ErrUnableToReadMessage + } + + var option models.CachedDimensionOption + + err = json.Unmarshal(b, &option) + if err != nil { + return nil, errs.ErrUnableToParseJSON + + } + if option.Name == "" || (option.Option == "" && option.CodeList == "") { + return nil, errs.ErrMissingParameters + } + + return &option, nil +} + +func handleDimensionErr(ctx context.Context, w http.ResponseWriter, err error, data log.Data) { + if data == nil { + data = log.Data{} + } + + var status int + resource := err + switch { + case errs.NotFoundMap[err]: + status = http.StatusNotFound + case errs.BadRequestMap[err]: + status = http.StatusBadRequest + default: + status = http.StatusInternalServerError + resource = errs.ErrInternalServer + } + + data["response_status"] = status + audit.LogError(ctx, errors.WithMessage(err, "request unsuccessful"), data) + http.Error(w, resource.Error(), status) +} diff --git a/dimension/helpers_test.go b/dimension/helpers_test.go new file mode 100644 index 00000000..fd0af970 --- /dev/null +++ b/dimension/helpers_test.go @@ -0,0 +1,100 @@ +package dimension + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + errs "github.com/ONSdigital/dp-dataset-api/apierrors" + "github.com/ONSdigital/go-ns/log" + . "github.com/smartystreets/goconvey/convey" +) + +func TestUnmarshalDimensionCache(t *testing.T) { + t.Parallel() + Convey("Successfully unmarshal dimension cache", t, func() { + json := strings.NewReader(`{"option":"24", "code_list":"123-456", "dimension": "test"}`) + + option, err := unmarshalDimensionCache(json) + So(err, ShouldBeNil) + So(option.CodeList, ShouldEqual, "123-456") + So(option.Name, ShouldEqual, "test") + So(option.Option, ShouldEqual, "24") + }) + + Convey("Fail to unmarshal dimension cache", t, func() { + Convey("When unable to marshal json", func() { + json := strings.NewReader("{") + + option, err := unmarshalDimensionCache(json) + So(err, ShouldNotBeNil) + So(err, ShouldResemble, errs.ErrUnableToParseJSON) + So(option, ShouldBeNil) + }) + + Convey("When options are missing mandatory fields", func() { + json := strings.NewReader("{}") + + option, err := unmarshalDimensionCache(json) + So(err, ShouldNotBeNil) + So(err, ShouldResemble, errs.ErrMissingParameters) + So(option, ShouldBeNil) + }) + }) +} + +type contextKey string + +const requestID = contextKey("request_id") + +func TestHandleDimensionErr(t *testing.T) { + ctx := context.Background() + ctx = context.WithValue(ctx, requestID, "123789") + + t.Parallel() + Convey("Correctly handle dimension not found", t, func() { + w := httptest.NewRecorder() + dimensionError := errs.ErrDimensionNotFound + logData := log.Data{"test": "not found"} + + handleDimensionErr(ctx, w, dimensionError, logData) + + So(w.Code, ShouldEqual, http.StatusNotFound) + So(w.Body.String(), ShouldContainSubstring, dimensionError.Error()) + }) + + Convey("Correctly handle bad request", t, func() { + w := httptest.NewRecorder() + dimensionError := errs.ErrUnableToParseJSON + logData := log.Data{"test": "bad request"} + + handleDimensionErr(ctx, w, dimensionError, logData) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, dimensionError.Error()) + }) + + Convey("Correctly handle internal error", t, func() { + w := httptest.NewRecorder() + dimensionError := errs.ErrInternalServer + logData := log.Data{"test": "internal error"} + + handleDimensionErr(ctx, w, dimensionError, logData) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, dimensionError.Error()) + }) + + Convey("Correctly handle failure to audit", t, func() { + w := httptest.NewRecorder() + dimensionError := errs.ErrAuditActionAttemptedFailure + logData := log.Data{"test": "audit failure"} + + handleDimensionErr(ctx, w, dimensionError, logData) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + }) +} diff --git a/instance/event.go b/instance/event.go index bb5b76e5..cc48da81 100644 --- a/instance/event.go +++ b/instance/event.go @@ -15,12 +15,12 @@ import ( func unmarshalEvent(reader io.Reader) (*models.Event, error) { b, err := ioutil.ReadAll(reader) if err != nil { - return nil, errors.New("Failed to read message body") + return nil, errors.New("failed to read message body") } var event models.Event err = json.Unmarshal(b, &event) if err != nil { - return nil, errors.New("Failed to parse json body") + return nil, errors.New("failed to parse json body") } return &event, err } @@ -46,7 +46,7 @@ func (s *Store) AddEvent(w http.ResponseWriter, r *http.Request) { if err = s.AddEventToInstance(id, event); err != nil { log.Error(err, nil) - handleErrorType(err, w) + handleInstanceErr(r.Context(), err, w, nil) return } diff --git a/instance/event_external_test.go b/instance/event_external_test.go index 2158f9f9..028d39d6 100644 --- a/instance/event_external_test.go +++ b/instance/event_external_test.go @@ -18,7 +18,8 @@ func TestAddEventReturnsOk(t *testing.T) { t.Parallel() Convey("Add an event to an instance returns ok", t, func() { body := strings.NewReader(`{"message": "321", "type": "error", "message_offset":"00", "time":"2017-08-25T15:09:11.829+01:00" }`) - r := createRequestWithToken("POST", "http://localhost:21800/instances/123/events", body) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/events", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -39,7 +40,8 @@ func TestAddEventToInstanceReturnsBadRequest(t *testing.T) { t.Parallel() Convey("Add an event to an instance returns bad request", t, func() { body := strings.NewReader(`{"message": "321", "type": "error", "message_offset":"00" }`) - r := createRequestWithToken("POST", "http://localhost:21800/instances/123/events", body) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/events", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{} @@ -48,9 +50,11 @@ func TestAddEventToInstanceReturnsBadRequest(t *testing.T) { So(w.Code, ShouldEqual, http.StatusBadRequest) }) + Convey("Add an event to an instance returns bad request", t, func() { body := strings.NewReader(`{`) - r := createRequestWithToken("POST", "http://localhost:21800/instances/123/events", body) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/events", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{} @@ -65,7 +69,8 @@ func TestAddEventToInstanceReturnsInternalError(t *testing.T) { t.Parallel() Convey("Add an event to an instance returns internal error", t, func() { body := strings.NewReader(`{"message": "321", "type": "error", "message_offset":"00", "time":"2017-08-25T15:09:11.829+01:00" }`) - r := createRequestWithToken("POST", "http://localhost:21800/instances/123/events", body) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances/123/events", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ diff --git a/instance/instance.go b/instance/instance.go index 27d617e4..5aff3e3c 100644 --- a/instance/instance.go +++ b/instance/instance.go @@ -1,8 +1,8 @@ package instance import ( + "context" "encoding/json" - "errors" "fmt" "io" "io/ioutil" @@ -13,134 +13,227 @@ import ( errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/store" + "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/common" "github.com/ONSdigital/go-ns/log" "github.com/gorilla/mux" + "github.com/pkg/errors" "github.com/satori/go.uuid" ) //Store provides a backend for instances type Store struct { - Host string store.Storer + Host string + Auditor audit.AuditorService } +type taskError struct { + error error + status int +} + +func (e taskError) Error() string { + if e.error != nil { + return e.error.Error() + } + return "" +} + +// List of audit actions for instances +const ( + GetInstanceAction = "getInstance" + GetInstancesAction = "getInstances" + UpdateInstanceAction = "updateInstance" + UpdateDimensionAction = "updateDimension" + UpdateInsertedObservationsAction = "updateInsertedObservations" + UpdateImportTasksAction = "updateImportTasks" +) + //GetList a list of all instances func (s *Store) GetList(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + data := log.Data{} stateFilterQuery := r.URL.Query().Get("state") + var ap common.Params var stateFilterList []string + if stateFilterQuery != "" { + data["query"] = stateFilterQuery + ap = common.Params{"query": stateFilterQuery} stateFilterList = strings.Split(stateFilterQuery, ",") - if err := models.ValidateStateFilter(stateFilterList); err != nil { - log.Error(err, nil) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } } - results, err := s.GetInstances(stateFilterList) - if err != nil { - log.Error(err, nil) - handleErrorType(err, w) + if err := s.Auditor.Record(ctx, GetInstancesAction, audit.Attempted, ap); err != nil { + handleInstanceErr(ctx, err, w, nil) return } - b, err := json.Marshal(results) + b, err := func() ([]byte, error) { + if len(stateFilterList) > 0 { + if err := models.ValidateStateFilter(stateFilterList); err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "get instances: filter state invalid"), data) + return nil, taskError{error: err, status: http.StatusBadRequest} + } + } + + results, err := s.GetInstances(stateFilterList) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "get instances: store.GetInstances returned and error"), nil) + return nil, err + } + + b, err := json.Marshal(results) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "get instances: failed to marshal results to json"), nil) + return nil, err + } + return b, nil + }() + if err != nil { - internalError(w, err) + if auditErr := s.Auditor.Record(ctx, GetInstancesAction, audit.Unsuccessful, ap); auditErr != nil { + err = auditErr + } + handleInstanceErr(ctx, err, w, data) + return + } + + if auditErr := s.Auditor.Record(ctx, GetInstancesAction, audit.Successful, ap); auditErr != nil { + handleInstanceErr(ctx, auditErr, w, data) return } - writeBody(w, b) - log.Debug("get all instances", log.Data{"query": stateFilterQuery}) + writeBody(ctx, w, b) + log.InfoCtx(ctx, "get instances: request successful", data) } //Get a single instance by id func (s *Store) Get(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) id := vars["id"] - instance, err := s.GetInstance(id) - if err != nil { - log.Error(err, nil) - handleErrorType(err, w) + data := log.Data{"instance_id": id} + ap := common.Params{"instance_id": id} + + data["req_path"] = r.URL.Path + log.InfoCtx(ctx, "instance get: handler called, auditing attempt", data) + + if err := s.Auditor.Record(ctx, GetInstanceAction, audit.Attempted, ap); err != nil { + handleInstanceErr(ctx, err, w, nil) return } - // Early return if instance state is invalid - if err = models.CheckState("instance", instance.State); err != nil { - log.ErrorC("instance has an invalid state", err, log.Data{"state": instance.State}) - internalError(w, err) + log.InfoCtx(ctx, "instance get: getting instance", data) + + b, err := func() ([]byte, error) { + instance, err := s.GetInstance(id) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "get instance: failed to retrieve instance"), data) + return nil, err + } + + log.InfoCtx(ctx, "instance get: checking instance state", data) + // Early return if instance state is invalid + if err = models.CheckState("instance", instance.State); err != nil { + data["state"] = instance.State + log.ErrorCtx(ctx, errors.WithMessage(err, "get instance: instance has an invalid state"), data) + return nil, err + } + + log.InfoCtx(ctx, "instance get: marshalling instance json", data) + b, err := json.Marshal(instance) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "get instance: failed to marshal instance to json"), data) + return nil, err + } + + return b, nil + }() + + log.InfoCtx(ctx, "instance get: auditing outcome", data) + if err != nil { + if auditErr := s.Auditor.Record(ctx, GetInstanceAction, audit.Unsuccessful, ap); auditErr != nil { + err = auditErr + } + handleInstanceErr(ctx, err, w, data) return } - b, err := json.Marshal(instance) - if err != nil { - internalError(w, err) + if auditErr := s.Auditor.Record(ctx, GetInstanceAction, audit.Successful, ap); auditErr != nil { + handleInstanceErr(ctx, auditErr, w, data) return } - writeBody(w, b) - log.Debug("get instance", log.Data{"instance_id": id}) + writeBody(ctx, w, b) + log.InfoCtx(ctx, "instance get: request successful", data) } //Add an instance func (s *Store) Add(w http.ResponseWriter, r *http.Request) { defer r.Body.Close() + ctx := r.Context() instance, err := unmarshalInstance(r.Body, true) if err != nil { - log.Error(err, nil) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance add: failed to unmarshal json to model"), nil) http.Error(w, err.Error(), http.StatusBadRequest) return } instance.InstanceID = uuid.NewV4().String() + data := log.Data{"instance_id": instance.InstanceID} + instance.Links.Self = &models.IDLink{ HRef: fmt.Sprintf("%s/instances/%s", s.Host, instance.InstanceID), } instance, err = s.AddInstance(instance) if err != nil { - internalError(w, err) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance add: store.AddInstance returned an error"), data) + internalError(ctx, w, err) return } b, err := json.Marshal(instance) if err != nil { - internalError(w, err) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance add: failed to marshal instance to json"), data) + internalError(ctx, w, err) return } w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusCreated) - writeBody(w, b) - log.Debug("add instance", log.Data{"instance": instance}) + writeBody(ctx, w, b) + log.InfoCtx(ctx, "instance add: request successful", data) } // UpdateDimension updates label and/or description for a specific dimension within an instance func (s *Store) UpdateDimension(w http.ResponseWriter, r *http.Request) { - + ctx := r.Context() vars := mux.Vars(r) id := vars["id"] dimension := vars["dimension"] + data := log.Data{"instance_id": id, "dimension": dimension} // Get instance instance, err := s.GetInstance(id) if err != nil { - log.ErrorC("Failed to GET instance when attempting to update a dimension of that instance.", err, log.Data{"instance": id}) - handleErrorType(err, w) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update dimension: Failed to GET instance"), data) + handleInstanceErr(ctx, err, w, data) return } // Early return if instance state is invalid if err = models.CheckState("instance", instance.State); err != nil { - log.ErrorC("current instance has an invalid state", err, log.Data{"state": instance.State}) - handleErrorType(errs.ErrInternalServer, w) + data["state"] = instance.State + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update dimension: current instance has an invalid state"), data) + handleInstanceErr(ctx, err, w, data) return } // Early return if instance is already published if instance.State == models.PublishedState { - log.Debug("unable to update instance/version, already published", nil) + log.InfoCtx(ctx, "instance update dimension: unable to update instance/version, already published", data) w.WriteHeader(http.StatusForbidden) return } @@ -148,7 +241,7 @@ func (s *Store) UpdateDimension(w http.ResponseWriter, r *http.Request) { // Read and unmarshal request body b, err := ioutil.ReadAll(r.Body) if err != nil { - log.ErrorC("Error reading response.body.", err, nil) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update dimension: error reading request.body"), data) http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -157,7 +250,7 @@ func (s *Store) UpdateDimension(w http.ResponseWriter, r *http.Request) { err = json.Unmarshal(b, &dim) if err != nil { - log.ErrorC("Failing to model models.Codelist resource based on request", err, log.Data{"instance": id, "dimension": dimension}) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update dimension: failing to model models.Codelist resource based on request"), data) http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -178,35 +271,35 @@ func (s *Store) UpdateDimension(w http.ResponseWriter, r *http.Request) { } break } - } if notFound { - log.ErrorC("dimension not found", errs.ErrDimensionNotFound, log.Data{"instance": id, "dimension": dimension}) - handleErrorType(errs.ErrDimensionNotFound, w) + log.ErrorCtx(ctx, errors.WithMessage(errs.ErrDimensionNotFound, "instance update dimension: dimension not found"), data) + handleInstanceErr(ctx, errs.ErrDimensionNotFound, w, data) return } // Update instance if err = s.UpdateInstance(id, instance); err != nil { - log.ErrorC("Failed to update instance with new dimension label/description.", err, log.Data{"instance": id, "dimension": dimension}) - handleErrorType(err, w) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update dimension: failed to update instance with new dimension label/description"), data) + handleInstanceErr(ctx, err, w, data) return } - log.Debug("updated dimension", log.Data{"instance": id, "dimension": dimension}) - + log.InfoCtx(ctx, "instance updated dimension: request successful", data) } //Update a specific instance func (s *Store) Update(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) id := vars["id"] + data := log.Data{"instance_id": id} defer r.Body.Close() instance, err := unmarshalInstance(r.Body, false) if err != nil { - log.Error(err, nil) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update: failed unmarshalling json to model"), data) http.Error(w, err.Error(), http.StatusBadRequest) return } @@ -214,15 +307,16 @@ func (s *Store) Update(w http.ResponseWriter, r *http.Request) { // Get the current document currentInstance, err := s.GetInstance(id) if err != nil { - log.Error(err, nil) - handleErrorType(err, w) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update: store.GetInstance returned error"), data) + handleInstanceErr(ctx, err, w, data) return } // Early return if instance state is invalid if err = models.CheckState("instance", currentInstance.State); err != nil { - log.ErrorC("current instance has an invalid state", err, log.Data{"state": currentInstance.State}) - handleErrorType(errs.ErrInternalServer, w) + data["state"] = currentInstance.State + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update: current instance has an invalid state"), data) + handleInstanceErr(ctx, err, w, data) return } @@ -230,37 +324,15 @@ func (s *Store) Update(w http.ResponseWriter, r *http.Request) { instance.Links = updateLinks(instance, currentInstance) logData := log.Data{"instance_id": id, "current_state": currentInstance.State, "requested_state": instance.State} - if instance.State != currentInstance.State { - switch instance.State { - case models.CompletedState: - if err = validateInstanceUpdate(models.SubmittedState, currentInstance, instance); err != nil { - log.Error(err, logData) - http.Error(w, err.Error(), http.StatusForbidden) - return - } - case models.EditionConfirmedState: - if err = validateInstanceUpdate(models.CompletedState, currentInstance, instance); err != nil { - log.Error(err, logData) - http.Error(w, err.Error(), http.StatusForbidden) - return - } - case models.AssociatedState: - if err = validateInstanceUpdate(models.EditionConfirmedState, currentInstance, instance); err != nil { - log.Error(err, logData) - http.Error(w, err.Error(), http.StatusForbidden) - return - } + if instance.State != "" && instance.State != currentInstance.State { - // TODO Update dataset.next state to associated and add collection id - case models.PublishedState: - if err = validateInstanceUpdate(models.AssociatedState, currentInstance, instance); err != nil { - log.Error(err, logData) - http.Error(w, err.Error(), http.StatusForbidden) - return - } - - // TODO Update both edition and dataset states to published + status, err := validateInstanceStateUpdate(instance, currentInstance) + if err != nil { + log.ErrorCtx(ctx, errors.Errorf("instance update: instance state invalid"), logData) + http.Error(w, err.Error(), status) + return } + } if instance.State == models.EditionConfirmedState { @@ -273,22 +345,23 @@ func (s *Store) Update(w http.ResponseWriter, r *http.Request) { edition := instance.Edition // Only create edition if it doesn't already exist - editionDoc, err := s.getEdition(datasetID, edition, id) + editionDoc, err := s.getEdition(ctx, datasetID, edition, id) if err != nil { - log.ErrorR(r, err, nil) - handleErrorType(err, w) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update: store.getEdition returned an error"), data) + handleInstanceErr(ctx, err, w, data) return } // Update with any edition.next changes editionDoc.Next.State = instance.State if err = s.UpsertEdition(datasetID, edition, editionDoc); err != nil { - log.ErrorR(r, err, nil) - handleErrorType(err, w) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update: store.UpsertEdition returned an error"), data) + handleInstanceErr(ctx, err, w, data) return } - log.Debug("created edition", log.Data{"instance": id, "edition": edition}) + data["edition"] = edition + log.InfoCtx(ctx, "instance update: created edition", data) // Check whether instance has a version if currentInstance.Version < 1 { @@ -296,8 +369,8 @@ func (s *Store) Update(w http.ResponseWriter, r *http.Request) { // instance and append by 1 to set the version of this instance document version, err := s.GetNextVersion(datasetID, edition) if err != nil { - log.ErrorR(r, err, nil) - handleErrorType(err, w) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update: store.GetNextVersion returned an error"), data) + handleInstanceErr(ctx, err, w, data) return } @@ -306,15 +379,59 @@ func (s *Store) Update(w http.ResponseWriter, r *http.Request) { links := s.defineInstanceLinks(instance, editionDoc) instance.Links = links } + + if err := s.AddVersionDetailsToInstance(ctx, currentInstance.InstanceID, datasetID, edition, instance.Version); err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update: datastore.AddVersionDetailsToInstance returned an error"), data) + handleInstanceErr(ctx, err, w, data) + return + } } if err = s.UpdateInstance(id, instance); err != nil { - log.Error(err, nil) - handleErrorType(err, w) + log.ErrorCtx(ctx, errors.WithMessage(err, "instance update: store.UpdateInstance returned an error"), data) + handleInstanceErr(ctx, err, w, data) return } - log.Debug("updated instance", log.Data{"instance": id}) + log.InfoCtx(ctx, "instance update: request successful", data) +} + +func validateInstanceStateUpdate(instance, currentInstance *models.Instance) (state int, err error) { + if instance.State != "" && instance.State != currentInstance.State { + switch instance.State { + case models.SubmittedState: + if currentInstance.State != models.CreatedState { + return http.StatusForbidden, errors.New("Unable to update resource, expected resource to have a state of " + models.CreatedState) + } + break + case models.CompletedState: + if currentInstance.State != models.SubmittedState { + return http.StatusForbidden, errors.New("Unable to update resource, expected resource to have a state of " + models.SubmittedState) + } + break + case models.EditionConfirmedState: + if currentInstance.State != models.CompletedState { + return http.StatusForbidden, errors.New("Unable to update resource, expected resource to have a state of " + models.CompletedState) + } + break + case models.AssociatedState: + if currentInstance.State != models.EditionConfirmedState { + return http.StatusForbidden, errors.New("Unable to update resource, expected resource to have a state of " + models.EditionConfirmedState) + } + break + case models.PublishedState: + if currentInstance.State != models.AssociatedState { + return http.StatusForbidden, errors.New("Unable to update resource, expected resource to have a state of " + models.AssociatedState) + } + break + default: + err = errors.New("instance resource has an invalid state") + return http.StatusInternalServerError, err + } + + } + + return http.StatusOK, nil } func updateLinks(instance, currentInstance *models.Instance) *models.InstanceLinks { @@ -337,12 +454,12 @@ func updateLinks(instance, currentInstance *models.Instance) *models.InstanceLin return links } -func (s *Store) getEdition(datasetID, edition, instanceID string) (*models.EditionUpdate, error) { - +func (s *Store) getEdition(ctx context.Context, datasetID, edition, instanceID string) (*models.EditionUpdate, error) { + data := log.Data{"dataset_id": datasetID, "instance_id": instanceID, "edition": edition} editionDoc, err := s.GetEdition(datasetID, edition, "") if err != nil { if err != errs.ErrEditionNotFound { - log.Error(err, nil) + log.ErrorCtx(ctx, err, data) return nil, err } // create unique id for edition @@ -376,7 +493,8 @@ func (s *Store) getEdition(datasetID, edition, instanceID string) (*models.Editi // Update the latest version for the dataset edition version, err := strconv.Atoi(editionDoc.Next.Links.LatestVersion.ID) if err != nil { - log.ErrorC("unable to retrieve latest version", err, log.Data{"instance": instanceID, "edition": edition, "version": editionDoc.Next.Links.LatestVersion.ID}) + data["version"] = editionDoc.Next.Links.LatestVersion.ID + log.ErrorCtx(ctx, errors.WithMessage(err, "unable to retrieve latest version"), data) return nil, err } @@ -389,19 +507,6 @@ func (s *Store) getEdition(datasetID, edition, instanceID string) (*models.Editi return editionDoc, nil } -func validateInstanceUpdate(expectedState string, currentInstance, instance *models.Instance) error { - var err error - if currentInstance.State == models.PublishedState { - err = fmt.Errorf("Unable to update resource state, as the version has been published") - } else if currentInstance.State != expectedState { - err = fmt.Errorf("Unable to update resource, expected resource to have a state of %s", expectedState) - } else if instance.State == models.EditionConfirmedState && currentInstance.Edition == "" && instance.Edition == "" { - err = errors.New("Unable to update resource, missing a value for the edition") - } - - return err -} - func (s *Store) defineInstanceLinks(instance *models.Instance, editionDoc *models.EditionUpdate) *models.InstanceLinks { stringifiedVersion := strconv.Itoa(instance.Version) @@ -443,88 +548,135 @@ func (s *Store) defineInstanceLinks(instance *models.Instance, editionDoc *model // UpdateObservations increments the count of inserted_observations against // an instance func (s *Store) UpdateObservations(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() vars := mux.Vars(r) id := vars["id"] insert := vars["inserted_observations"] observations, err := strconv.ParseInt(insert, 10, 64) if err != nil { - log.Error(err, nil) + log.ErrorCtx(ctx, errors.WithMessage(err, "update observations: failed to parse inserted_observations string to int"), log.Data{"stringValue": insert}) http.Error(w, err.Error(), http.StatusBadRequest) return } if err = s.UpdateObservationInserted(id, observations); err != nil { - log.Error(err, nil) - handleErrorType(err, w) + log.ErrorCtx(ctx, errors.WithMessage(err, "update observations: store.UpdateObservationInserted returned an error"), log.Data{"id": id}) + handleInstanceErr(ctx, err, w, nil) } } +// UpdateImportTask updates any task in the request body against an instance func (s *Store) UpdateImportTask(w http.ResponseWriter, r *http.Request) { - + ctx := r.Context() vars := mux.Vars(r) - id := vars["id"] - + instanceID := vars["id"] + ap := common.Params{"instance_id": instanceID} + data := audit.ToLogData(ap) defer r.Body.Close() - tasks, err := unmarshalImportTasks(r.Body) - if err != nil { - log.Error(err, nil) - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - - validationErrs := make([]error, 0) + updateErr := func() *taskError { + tasks, err := unmarshalImportTasks(r.Body) + if err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "failed to unmarshal request body to UpdateImportTasks model"), data) + return &taskError{err, http.StatusBadRequest} + } - if tasks.ImportObservations != nil { - if tasks.ImportObservations.State != "" { - if tasks.ImportObservations.State != models.CompletedState { - validationErrs = append(validationErrs, fmt.Errorf("bad request - invalid task state value for import observations: %v", tasks.ImportObservations.State)) - } else if err := s.UpdateImportObservationsTaskState(id, tasks.ImportObservations.State); err != nil { - log.Error(err, nil) - http.Error(w, "Failed to update import observations task state", http.StatusInternalServerError) - return + validationErrs := make([]error, 0) + var hasImportTasks bool + + if tasks.ImportObservations != nil { + hasImportTasks = true + if tasks.ImportObservations.State != "" { + if tasks.ImportObservations.State != models.CompletedState { + validationErrs = append(validationErrs, fmt.Errorf("bad request - invalid task state value for import observations: %v", tasks.ImportObservations.State)) + } else { + if err := s.UpdateImportObservationsTaskState(instanceID, tasks.ImportObservations.State); err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "Failed to update import observations task state"), data) + return &taskError{err, http.StatusInternalServerError} + } + } + } else { + validationErrs = append(validationErrs, errors.New("bad request - invalid import observation task, must include state")) } } - } - if tasks.BuildHierarchyTasks != nil { - for _, task := range tasks.BuildHierarchyTasks { - if task.State != "" { - if task.State != models.CompletedState { - validationErrs = append(validationErrs, fmt.Errorf("bad request - invalid task state value: %v", task.State)) - } else if err := s.UpdateBuildHierarchyTaskState(id, task.DimensionName, task.State); err != nil { - log.Error(err, nil) - http.Error(w, "Failed to update build hierarchy task state", http.StatusInternalServerError) - return + if tasks.BuildHierarchyTasks != nil { + hasImportTasks = true + var hasHierarchyImportTask bool + for _, task := range tasks.BuildHierarchyTasks { + hasHierarchyImportTask = true + if err := models.ValidateImportTask(task.GenericTaskDetails); err != nil { + validationErrs = append(validationErrs, err) + } else { + if err := s.UpdateBuildHierarchyTaskState(instanceID, task.DimensionName, task.State); err != nil { + if err.Error() == "not found" { + notFoundErr := task.DimensionName + " hierarchy import task does not exist" + log.ErrorCtx(ctx, errors.WithMessage(err, notFoundErr), data) + return &taskError{errors.New(notFoundErr), http.StatusNotFound} + } + log.ErrorCtx(ctx, errors.WithMessage(err, "failed to update build hierarchy task state"), data) + return &taskError{err, http.StatusInternalServerError} + } } } + if !hasHierarchyImportTask { + validationErrs = append(validationErrs, errors.New("bad request - missing hierarchy task")) + } } - } - if tasks.BuildSearchIndexTasks != nil { - for _, task := range tasks.BuildSearchIndexTasks { - if task.State != "" { - if task.State != models.CompletedState { - validationErrs = append(validationErrs, fmt.Errorf("bad request - invalid task state value: %v", task.State)) - } else if err := s.UpdateBuildSearchTaskState(id, task.DimensionName, task.State); err != nil { - log.Error(err, nil) - http.Error(w, "Failed to update build search index task state", http.StatusInternalServerError) - return + if tasks.BuildSearchIndexTasks != nil { + hasImportTasks = true + var hasSearchIndexImportTask bool + for _, task := range tasks.BuildSearchIndexTasks { + hasSearchIndexImportTask = true + if err := models.ValidateImportTask(task.GenericTaskDetails); err != nil { + validationErrs = append(validationErrs, err) + } else { + if err := s.UpdateBuildSearchTaskState(instanceID, task.DimensionName, task.State); err != nil { + if err.Error() == "not found" { + notFoundErr := task.DimensionName + " search index import task does not exist" + log.ErrorCtx(ctx, errors.WithMessage(err, notFoundErr), data) + return &taskError{errors.New(notFoundErr), http.StatusNotFound} + } + log.ErrorCtx(ctx, errors.WithMessage(err, "failed to update build hierarchy task state"), data) + return &taskError{err, http.StatusInternalServerError} + } } } + if !hasSearchIndexImportTask { + validationErrs = append(validationErrs, errors.New("bad request - missing search index task")) + } } - } - if len(validationErrs) > 0 { - for _, err := range validationErrs { - log.Error(err, nil) + if !hasImportTasks { + validationErrs = append(validationErrs, errors.New("bad request - request body does not contain any import tasks")) } - // todo: add all validation errors to the response - http.Error(w, validationErrs[0].Error(), http.StatusBadRequest) + + if len(validationErrs) > 0 { + for _, err := range validationErrs { + log.ErrorCtx(ctx, errors.WithMessage(err, "validation error"), data) + } + // todo: add all validation errors to the response + return &taskError{validationErrs[0], http.StatusBadRequest} + } + return nil + }() + + if updateErr != nil { + if auditErr := s.Auditor.Record(ctx, UpdateImportTasksAction, audit.Unsuccessful, ap); auditErr != nil { + updateErr = &taskError{errs.ErrInternalServer, http.StatusInternalServerError} + } + log.ErrorCtx(ctx, errors.WithMessage(updateErr, "updateImportTask endpoint: request unsuccessful"), data) + http.Error(w, updateErr.Error(), updateErr.status) return } + if auditErr := s.Auditor.Record(ctx, UpdateImportTasksAction, audit.Successful, ap); auditErr != nil { + return + } + + log.InfoCtx(ctx, "updateImportTask endpoint: request successful", data) } func unmarshalImportTasks(reader io.Reader) (*models.InstanceImportTasks, error) { @@ -546,13 +698,13 @@ func unmarshalImportTasks(reader io.Reader) (*models.InstanceImportTasks, error) func unmarshalInstance(reader io.Reader, post bool) (*models.Instance, error) { b, err := ioutil.ReadAll(reader) if err != nil { - return nil, errors.New("Failed to read message body") + return nil, errors.New("failed to read message body") } var instance models.Instance err = json.Unmarshal(b, &instance) if err != nil { - return nil, errors.New("Failed to parse json body: " + err.Error()) + return nil, errors.New("failed to parse json body: " + err.Error()) } if instance.State != "" { @@ -564,12 +716,12 @@ func unmarshalInstance(reader io.Reader, post bool) (*models.Instance, error) { if post { log.Debug("post request on an instance", log.Data{"instance_id": instance.InstanceID}) if instance.Links == nil || instance.Links.Job == nil { - return nil, errors.New("Missing job properties") + return nil, errs.ErrMissingJobProperties } // Need both href and id for job link if instance.Links.Job.HRef == "" || instance.Links.Job.ID == "" { - return nil, errors.New("Missing job properties") + return nil, errs.ErrMissingJobProperties } // TODO May want to check the id and href make sense; or change spec to allow @@ -584,25 +736,15 @@ func unmarshalInstance(reader io.Reader, post bool) (*models.Instance, error) { return &instance, nil } -func handleErrorType(err error, w http.ResponseWriter) { - status := http.StatusInternalServerError - - if err == errs.ErrDatasetNotFound || err == errs.ErrEditionNotFound || err == errs.ErrVersionNotFound || err == errs.ErrDimensionNotFound || err == errs.ErrDimensionNodeNotFound || err == errs.ErrInstanceNotFound { - status = http.StatusNotFound - } - - http.Error(w, err.Error(), status) -} - -func internalError(w http.ResponseWriter, err error) { - log.Error(err, nil) +func internalError(ctx context.Context, w http.ResponseWriter, err error) { + log.ErrorCtx(ctx, err, nil) http.Error(w, err.Error(), http.StatusInternalServerError) } -func writeBody(w http.ResponseWriter, b []byte) { +func writeBody(ctx context.Context, w http.ResponseWriter, b []byte) { w.Header().Set("Content-Type", "application/json") if _, err := w.Write(b); err != nil { - log.Error(err, nil) + log.ErrorCtx(ctx, err, nil) http.Error(w, err.Error(), http.StatusInternalServerError) } } @@ -610,28 +752,74 @@ func writeBody(w http.ResponseWriter, b []byte) { // PublishCheck Checks if an instance has been published type PublishCheck struct { Datastore store.Storer + Auditor audit.AuditorService } // Check wraps a HTTP handle. Checks that the state is not published -func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request)) http.HandlerFunc { +func (d *PublishCheck) Check(handle func(http.ResponseWriter, *http.Request), action string) http.HandlerFunc { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - + ctx := r.Context() vars := mux.Vars(r) - id := vars["id"] - instance, err := d.Datastore.GetInstance(id) - if err != nil { - log.Error(err, nil) - handleErrorType(err, w) + instanceID := vars["id"] + logData := log.Data{"instance_id": instanceID} + auditParams := common.Params{"instance_id": instanceID} + + if err := d.Auditor.Record(ctx, action, audit.Attempted, auditParams); err != nil { + handleInstanceErr(ctx, errs.ErrAuditActionAttemptedFailure, w, logData) return } - if instance.State == models.PublishedState { - err = errors.New("unable to update instance as it has been published") - log.Error(err, log.Data{"instance": instance}) - http.Error(w, err.Error(), http.StatusForbidden) + if err := d.checkState(instanceID); err != nil { + log.ErrorCtx(ctx, errors.WithMessage(err, "errored whilst checking instance state"), logData) + if auditErr := d.Auditor.Record(ctx, action, audit.Unsuccessful, auditParams); auditErr != nil { + handleInstanceErr(ctx, errs.ErrAuditActionAttemptedFailure, w, logData) + return + } + + handleInstanceErr(ctx, err, w, logData) return } handle(w, r) }) } + +func (d *PublishCheck) checkState(instanceID string) error { + instance, err := d.Datastore.GetInstance(instanceID) + if err != nil { + return err + } + + if instance.State == models.PublishedState { + return errs.ErrResourcePublished + } + + return nil +} + +func handleInstanceErr(ctx context.Context, err error, w http.ResponseWriter, data log.Data) { + if data == nil { + data = log.Data{} + } + + taskErr, isTaskErr := err.(taskError) + + var status int + response := err + + switch { + case isTaskErr: + status = taskErr.status + case errs.NotFoundMap[err]: + status = http.StatusNotFound + case err == errs.ErrResourcePublished: + status = http.StatusForbidden + default: + status = http.StatusInternalServerError + response = errs.ErrInternalServer + } + + data["responseStatus"] = status + log.ErrorCtx(ctx, errors.WithMessage(err, "request unsuccessful"), data) + http.Error(w, response.Error(), status) +} diff --git a/instance/instance_external_test.go b/instance/instance_external_test.go index b4511b96..4f2ca93e 100644 --- a/instance/instance_external_test.go +++ b/instance/instance_external_test.go @@ -1,34 +1,48 @@ package instance_test import ( + "context" + "errors" "io" "net/http" "net/http/httptest" "strings" "testing" + "time" + "github.com/ONSdigital/dp-dataset-api/api" + errs "github.com/ONSdigital/dp-dataset-api/apierrors" + "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/instance" + "github.com/ONSdigital/dp-dataset-api/mocks" "github.com/ONSdigital/dp-dataset-api/models" + "github.com/ONSdigital/dp-dataset-api/store" "github.com/ONSdigital/dp-dataset-api/store/datastoretest" + "github.com/ONSdigital/dp-dataset-api/url" + "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/audit/audit_mock" + "github.com/ONSdigital/go-ns/common" "github.com/gorilla/mux" . "github.com/smartystreets/goconvey/convey" - - errs "github.com/ONSdigital/dp-dataset-api/apierrors" ) -const secretKey = "coffee" const host = "http://localhost:8080" -func createRequestWithToken(method, url string, body io.Reader) *http.Request { - r := httptest.NewRequest(method, url, body) - r.Header.Add("internal-token", secretKey) - return r +var errAudit = errors.New("auditing error") + +func createRequestWithToken(method, url string, body io.Reader) (*http.Request, error) { + r, err := http.NewRequest(method, url, body) + ctx := r.Context() + ctx = common.SetCaller(ctx, "someone@ons.gov.uk") + r = r.WithContext(ctx) + return r, err } func TestGetInstancesReturnsOK(t *testing.T) { t.Parallel() Convey("Get instances returns a ok status code", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -37,18 +51,26 @@ func TestGetInstancesReturnsOK(t *testing.T) { }, } - instance := &instance.Store{Host: "http://lochost://8080", Storer: mockedDataStore} - instance.GetList(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetInstancesCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Attempted, nil), + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Successful, nil), + ) }) } func TestGetInstancesFiltersOnState(t *testing.T) { t.Parallel() Convey("Get instances filtered by a single state value returns only instances with that value", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances?state=completed", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances?state=completed", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() var result []string @@ -59,16 +81,25 @@ func TestGetInstancesFiltersOnState(t *testing.T) { }, } - instance := &instance.Store{Host: "http://lochost://8080", Storer: mockedDataStore} - instance.GetList(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.GetInstancesCalls()), ShouldEqual, 1) So(result, ShouldResemble, []string{models.CompletedState}) + So(len(mockedDataStore.GetInstancesCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + expectedParams := common.Params{"query": "completed"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Attempted, expectedParams), + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Successful, expectedParams), + ) }) Convey("Get instances filtered by multiple state values returns only instances with those values", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances?state=completed,edition-confirmed", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances?state=completed,edition-confirmed", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() var result []string @@ -79,19 +110,28 @@ func TestGetInstancesFiltersOnState(t *testing.T) { }, } - instance := &instance.Store{Host: "http://lochost://8080", Storer: mockedDataStore} - instance.GetList(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.GetInstancesCalls()), ShouldEqual, 1) So(result, ShouldResemble, []string{models.CompletedState, models.EditionConfirmedState}) + So(len(mockedDataStore.GetInstancesCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + expectedParams := common.Params{"query": "completed,edition-confirmed"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Attempted, expectedParams), + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Successful, expectedParams), + ) }) } func TestGetInstancesReturnsError(t *testing.T) { t.Parallel() Convey("Get instances returns an internal error", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -100,31 +140,145 @@ func TestGetInstancesReturnsError(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.GetList(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetInstancesCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Attempted, nil), + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Unsuccessful, nil), + ) }) Convey("Get instances returns bad request error", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances?state=foo", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances?state=foo", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{} - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.GetList(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) - So(len(mockedDataStore.GetInstancesCalls()), ShouldEqual, 0) + So(w.Body.String(), ShouldContainSubstring, "bad request - invalid filter state values: [foo]") + + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + expectedParams := common.Params{"query": "foo"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Attempted, expectedParams), + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Unsuccessful, expectedParams), + ) + }) +} + +func TestGetInstancesAuditErrors(t *testing.T) { + t.Parallel() + Convey("given audit action attempted returns an error", t, func() { + + auditor := audit_mock.NewErroring(instance.GetInstancesAction, audit.Attempted) + + Convey("when get instances is called", func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{} + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstancesCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Attempted, nil), + ) + }) + }) + }) + + Convey("given audit action unsuccessful returns an error", t, func() { + auditor := audit_mock.NewErroring(instance.GetInstancesAction, audit.Unsuccessful) + + Convey("when get instances return an error", func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstancesFunc: func([]string) (*models.InstanceResults, error) { + return nil, errs.ErrInternalServer + }, + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstancesCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Attempted, nil), + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Unsuccessful, nil), + ) + }) + }) + }) + + Convey("given audit action successful returns an error", t, func() { + auditor := audit_mock.NewErroring(instance.GetInstancesAction, audit.Successful) + + Convey("when get instances is called", func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstancesFunc: func([]string) (*models.InstanceResults, error) { + return &models.InstanceResults{}, nil + }, + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstancesCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Attempted, nil), + audit_mock.NewExpectation(instance.GetInstancesAction, audit.Successful, nil), + ) + }) + }) }) } func TestGetInstanceReturnsOK(t *testing.T) { t.Parallel() Convey("Get instance returns a ok status code", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances/123", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -133,18 +287,27 @@ func TestGetInstanceReturnsOK(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Get(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.GetInstanceAction, audit.Attempted, auditParams}, + audit_mock.Expected{instance.GetInstanceAction, audit.Successful, auditParams}, + ) }) } func TestGetInstanceReturnsInternalError(t *testing.T) { t.Parallel() Convey("Given an internal error is returned from mongo, then response returns an internal error", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances/123", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -153,17 +316,26 @@ func TestGetInstanceReturnsInternalError(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Get(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.GetInstanceAction, audit.Attempted, auditParams}, + audit_mock.Expected{instance.GetInstanceAction, audit.Unsuccessful, auditParams}, + ) }) Convey("Given instance state is invalid, then response returns an internal error", t, func() { - r := createRequestWithToken("GET", "http://localhost:21800/instances/123", nil) + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -172,12 +344,125 @@ func TestGetInstanceReturnsInternalError(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Get(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "Incorrect resource state\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.GetInstanceAction, audit.Attempted, auditParams}, + audit_mock.Expected{instance.GetInstanceAction, audit.Unsuccessful, auditParams}, + ) + }) +} + +func TestGetInstanceAuditErrors(t *testing.T) { + t.Parallel() + Convey("Given audit action 'attempted' fails", t, func() { + + auditor := audit_mock.NewErroring(instance.GetInstanceAction, audit.Attempted) + + Convey("When a GET request is made to get an instance resource", func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return nil, errs.ErrInternalServer + }, + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("Then response returns internal server error (500)", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.GetInstanceAction, audit.Attempted, auditParams}, + ) + }) + }) + }) + + Convey("Given audit action 'unsuccessful' fails", t, func() { + + auditor := audit_mock.NewErroring(instance.GetInstanceAction, audit.Unsuccessful) + + Convey("When a GET request is made to get an instance resource", func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return nil, errs.ErrInternalServer + }, + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("Then response returns internal server error (500)", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.GetInstanceAction, audit.Attempted, auditParams}, + audit_mock.Expected{instance.GetInstanceAction, audit.Unsuccessful, auditParams}, + ) + }) + }) + }) + + Convey("Given audit action 'successful' fails", t, func() { + + auditor := audit_mock.NewErroring(instance.GetInstanceAction, audit.Successful) + + Convey("When a GET request is made to get an instance resource", func() { + r, err := createRequestWithToken("GET", "http://localhost:21800/instances/123", nil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(ID string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("Then response returns internal server error (500)", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.GetInstanceAction, audit.Attempted, auditParams}, + audit_mock.Expected{instance.GetInstanceAction, audit.Successful, auditParams}, + ) + }) + }) }) } @@ -185,7 +470,8 @@ func TestAddInstancesReturnsCreated(t *testing.T) { t.Parallel() Convey("Add instance returns a created code", t, func() { body := strings.NewReader(`{"links": { "job": { "id":"123-456", "href":"http://localhost:2200/jobs/123-456" } } }`) - r := createRequestWithToken("POST", "http://localhost:21800/instances", body) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -194,11 +480,15 @@ func TestAddInstancesReturnsCreated(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Add(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusCreated) So(len(mockedDataStore.AddInstanceCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 0) + + auditor.AssertRecordCalls() }) } @@ -206,7 +496,8 @@ func TestAddInstancesReturnsBadRequest(t *testing.T) { t.Parallel() Convey("Add instance returns a bad request with invalid json", t, func() { body := strings.NewReader(`{`) - r := createRequestWithToken("POST", "http://localhost:21800/instances", body) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -215,15 +506,21 @@ func TestAddInstancesReturnsBadRequest(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Add(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, errs.ErrUnableToParseJSON.Error()) + So(len(auditor.RecordCalls()), ShouldEqual, 0) + + auditor.AssertRecordCalls() }) Convey("Add instance returns a bad request with a empty json", t, func() { body := strings.NewReader(`{}`) - r := createRequestWithToken("POST", "http://localhost:21800/instances", body) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -232,10 +529,15 @@ func TestAddInstancesReturnsBadRequest(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Add(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, errs.ErrMissingJobProperties.Error()) + So(len(auditor.RecordCalls()), ShouldEqual, 0) + + auditor.AssertRecordCalls() }) } @@ -243,7 +545,8 @@ func TestAddInstancesReturnsInternalError(t *testing.T) { t.Parallel() Convey("Add instance returns an internal error", t, func() { body := strings.NewReader(`{"links": {"job": { "id":"123-456", "href":"http://localhost:2200/jobs/123-456" } } }`) - r := createRequestWithToken("POST", "http://localhost:21800/instances", body) + r, err := createRequestWithToken("POST", "http://localhost:21800/instances", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ AddInstanceFunc: func(*models.Instance) (*models.Instance, error) { @@ -251,11 +554,17 @@ func TestAddInstancesReturnsInternalError(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Add(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.AddInstanceCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 0) + + auditor.AssertRecordCalls() }) } @@ -263,7 +572,8 @@ func TestUpdateInstanceReturnsOk(t *testing.T) { t.Parallel() Convey("when an instance has a state of created", t, func() { body := strings.NewReader(`{"state":"created"}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -275,17 +585,26 @@ func TestUpdateInstanceReturnsOk(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Update(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.AddVersionDetailsToInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInstanceAction, audit.Attempted, auditParams}, + ) }) Convey("when an instance changes its state to edition-confirmed", t, func() { body := strings.NewReader(`{"state":"edition-confirmed", "edition": "2017"}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() currentInstanceTestData := &models.Instance{ @@ -322,17 +641,28 @@ func TestUpdateInstanceReturnsOk(t *testing.T) { UpdateInstanceFunc: func(id string, i *models.Instance) error { return nil }, + AddVersionDetailsToInstanceFunc: func(ctx context.Context, instanceID string, datasetID string, edition string, version int) error { + return nil + }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Update(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.GetEditionCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpsertEditionCalls()), ShouldEqual, 1) So(len(mockedDataStore.GetNextVersionCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.AddVersionDetailsToInstanceCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInstanceAction, audit.Attempted, auditParams}, + ) }) } @@ -340,7 +670,8 @@ func TestUpdateInstanceReturnsInternalError(t *testing.T) { t.Parallel() Convey("Given an internal error is returned from mongo, then response returns an internal error", t, func() { body := strings.NewReader(`{"state":"created"}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -349,18 +680,28 @@ func TestUpdateInstanceReturnsInternalError(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Update(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(mockedDataStore.AddVersionDetailsToInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInstanceAction, audit.Attempted, auditParams}, + audit_mock.Expected{instance.UpdateInstanceAction, audit.Unsuccessful, auditParams}, + ) }) Convey("Given the current instance state is invalid, then response returns an internal error", t, func() { - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123", strings.NewReader(`{"state":"completed", "edition": "2017"}`)) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123", strings.NewReader(`{"state":"completed", "edition": "2017"}`)) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(id string) (*models.Instance, error) { @@ -368,14 +709,22 @@ func TestUpdateInstanceReturnsInternalError(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Update(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(mockedDataStore.AddVersionDetailsToInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInstanceAction, audit.Attempted, auditParams}, + ) }) } @@ -383,20 +732,36 @@ func TestUpdateInstanceFailure(t *testing.T) { t.Parallel() Convey("when the json body is in the incorrect structure return a bad request error", t, func() { body := strings.NewReader(`{"state":`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{} + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: "completed"}, nil + }, + } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Update(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 0) + So(w.Body.String(), ShouldContainSubstring, errs.ErrUnableToParseJSON.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.AddVersionDetailsToInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInstanceAction, audit.Attempted, auditParams}, + ) }) Convey("when the instance does not exist return status not found", t, func() { body := strings.NewReader(`{"edition": "2017"}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(id string) (*models.Instance, error) { @@ -404,12 +769,89 @@ func TestUpdateInstanceFailure(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Update(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInstanceNotFound.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(mockedDataStore.AddVersionDetailsToInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInstanceAction, audit.Attempted, auditParams}, + audit_mock.Expected{instance.UpdateInstanceAction, audit.Unsuccessful, auditParams}, + ) + }) + + Convey("when store.AddVersionDetailsToInstance return an error", t, func() { + body := strings.NewReader(`{"state":"edition-confirmed", "edition": "2017"}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + currentInstanceTestData := &models.Instance{ + Edition: "2017", + Links: &models.InstanceLinks{ + Job: &models.IDLink{ + ID: "7654", + HRef: "job-link", + }, + Dataset: &models.IDLink{ + ID: "4567", + HRef: "dataset-link", + }, + Self: &models.IDLink{ + HRef: "self-link", + }, + }, + State: models.CompletedState, + } + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return currentInstanceTestData, nil + }, + GetEditionFunc: func(datasetID string, edition string, state string) (*models.EditionUpdate, error) { + return nil, errs.ErrEditionNotFound + }, + UpsertEditionFunc: func(datasetID, edition string, editionDoc *models.EditionUpdate) error { + return nil + }, + GetNextVersionFunc: func(string, string) (int, error) { + return 1, nil + }, + UpdateInstanceFunc: func(id string, i *models.Instance) error { + return nil + }, + AddVersionDetailsToInstanceFunc: func(ctx context.Context, instanceID string, datasetID string, edition string, version int) error { + return errors.New("boom") + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) + So(len(mockedDataStore.GetEditionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpsertEditionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetNextVersionCalls()), ShouldEqual, 1) + So(len(mockedDataStore.AddVersionDetailsToInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInstanceAction, audit.Attempted, auditParams}, + ) }) } @@ -417,7 +859,8 @@ func TestUpdatePublishedInstanceToCompletedReturnsForbidden(t *testing.T) { t.Parallel() Convey("Given a 'published' instance, when we update to 'completed' then we get a bad-request error", t, func() { body := strings.NewReader(`{"state":"completed"}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/1235", body) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/1235", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() currentInstanceTestData := &models.Instance{ @@ -439,12 +882,23 @@ func TestUpdatePublishedInstanceToCompletedReturnsForbidden(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Update(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusForbidden) + So(w.Body.String(), ShouldContainSubstring, errs.ErrResourcePublished.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(mockedDataStore.AddVersionDetailsToInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + auditParams := common.Params{"instance_id": "1235"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInstanceAction, audit.Attempted, auditParams}, + audit_mock.Expected{instance.UpdateInstanceAction, audit.Unsuccessful, auditParams}, + ) }) } @@ -452,7 +906,8 @@ func TestUpdateEditionConfirmedInstanceToCompletedReturnsForbidden(t *testing.T) t.Parallel() Convey("update to an instance returns an internal error", t, func() { body := strings.NewReader(`{"state":"completed"}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() currentInstanceTestData := &models.Instance{ @@ -474,71 +929,118 @@ func TestUpdateEditionConfirmedInstanceToCompletedReturnsForbidden(t *testing.T) }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Update(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusForbidden) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(w.Body.String(), ShouldContainSubstring, "Unable to update resource, expected resource to have a state of submitted") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(mockedDataStore.AddVersionDetailsToInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInstanceAction, audit.Attempted, auditParams}, + ) }) } func TestInsertedObservationsReturnsOk(t *testing.T) { t.Parallel() Convey("Updateding the inserted observations returns ok", t, func() { - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/inserted_observations/200", nil) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/inserted_observations/200", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, UpdateObservationInsertedFunc: func(id string, ob int64) error { return nil }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - router := mux.NewRouter() - router.HandleFunc("/instances/{id}/inserted_observations/{inserted_observations}", instance.UpdateObservations).Methods("PUT") - router.ServeHTTP(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateObservationInsertedCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInsertedObservationsAction, audit.Attempted, auditParams}, + ) }) } func TestInsertedObservationsReturnsBadRequest(t *testing.T) { t.Parallel() Convey("Updateding the inserted observations returns bad request", t, func() { - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/inserted_observations/aa12a", nil) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/inserted_observations/aa12a", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{} + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.UpdateObservations(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "invalid syntax") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateObservationInsertedCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInsertedObservationsAction, audit.Attempted, auditParams}, + ) }) } func TestInsertedObservationsReturnsNotFound(t *testing.T) { t.Parallel() Convey("Updating the inserted observations returns not found", t, func() { - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/inserted_observations/200", nil) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/inserted_observations/200", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, UpdateObservationInsertedFunc: func(id string, ob int64) error { return errs.ErrInstanceNotFound }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - - router := mux.NewRouter() - router.HandleFunc("/instances/{id}/inserted_observations/{inserted_observations}", instance.UpdateObservations).Methods("PUT") - router.ServeHTTP(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInstanceNotFound.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateObservationInsertedCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + auditParams := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.Expected{instance.UpdateInsertedObservationsAction, audit.Attempted, auditParams}, + ) }) } @@ -547,149 +1049,544 @@ func TestStore_UpdateImportTask_UpdateImportObservations(t *testing.T) { t.Parallel() Convey("update to an import task returns http 200 response if no errors occur", t, func() { body := strings.NewReader(`{"import_observations":{"state":"completed"}}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, UpdateImportObservationsTaskStateFunc: func(id string, state string) error { return nil }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - - instance.UpdateImportTask(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Successful, ap), + ) }) } -func TestStore_UpdateImportTask_UpdateImportObservations_InvalidState(t *testing.T) { +func TestStore_UpdateImportTask_UpdateImportObservations_Failure(t *testing.T) { t.Parallel() - Convey("update to an import task with an invalid state returns http 400 response", t, func() { - body := strings.NewReader(`{"import_observations":{"state":"notvalid"}}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + Convey("update to an import task with invalid json returns http 400 response", t, func() { + body := strings.NewReader(`{`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, UpdateImportObservationsTaskStateFunc: func(id string, state string) error { return nil }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - - instance.UpdateImportTask(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, errs.ErrUnableToParseJSON.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) - }) -} + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) -func TestStore_UpdateImportTask_UpdateBuildHierarchyTask_InvalidState(t *testing.T) { + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) - t.Parallel() - Convey("update to an import task with an invalid state returns http 400 response", t, func() { - body := strings.NewReader(`{"build_hierarchies":[{"state":"notvalid"}]}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + Convey("update to an import task but missing mandatory field, 'state' returns http 400 response", t, func() { + body := strings.NewReader(`{"import_observations":{}}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + UpdateImportObservationsTaskStateFunc: func(id string, state string) error { return nil }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - - instance.UpdateImportTask(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - invalid import observation task, must include state") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) - }) -} + So(len(auditor.RecordCalls()), ShouldEqual, 2) -func TestStore_UpdateImportTask_UpdateBuildHierarchyTask(t *testing.T) { + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) - t.Parallel() - Convey("update to an import task returns http 200 response if no errors occur", t, func() { - body := strings.NewReader(`{"build_hierarchies":[{"state":"completed"}]}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + Convey("update to an import task with an invalid state returns http 400 response", t, func() { + body := strings.NewReader(`{"import_observations":{"state":"notvalid"}}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + UpdateImportObservationsTaskStateFunc: func(id string, state string) error { return nil }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) - instance.UpdateImportTask(w, r) + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - invalid task state value for import observations: notvalid") - So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) - So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) }) } -func TestStore_UpdateImportTask_ReturnsInternalError(t *testing.T) { +func TestStore_UpdateImportTask_UpdateBuildHierarchyTask_Failure(t *testing.T) { t.Parallel() - Convey("update to an import task returns an internal error", t, func() { - body := strings.NewReader(`{"import_observations":{"state":"completed"}}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + Convey("update to an import task with invalid json returns http 400 response", t, func() { + body := strings.NewReader(`{`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ - UpdateImportObservationsTaskStateFunc: func(id string, state string) error { - return errs.ErrInternalServer + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { + return nil }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) - instance.UpdateImportTask(w, r) + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, errs.ErrUnableToParseJSON.Error()) - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) }) -} -func TestUpdateInstanceReturnsErrorWhenStateIsPublished(t *testing.T) { - t.Parallel() - Convey("when an instance has a state of published, then put request to change to it to completed ", t, func() { - body := strings.NewReader(`{"state":"completed"}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + Convey("update to an import task with an empty request body returns http 400 response", t, func() { + body := strings.NewReader(`{}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ GetInstanceFunc: func(id string) (*models.Instance, error) { - return &models.Instance{State: models.PublishedState}, nil + return &models.Instance{State: models.EditionConfirmedState}, nil }, - UpdateInstanceFunc: func(id string, i *models.Instance) error { + UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { return nil }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.Update(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - request body does not contain any import tasks") - So(w.Code, ShouldEqual, http.StatusForbidden) So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) }) -} -func TestUpdateDimensionReturnsInternalError(t *testing.T) { - t.Parallel() + Convey("update to an import task without specifying a task returns http 400 response", t, func() { + body := strings.NewReader(`{"build_hierarchies":[]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - missing hierarchy task") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task without a 'dimension_name' returns http 400 response", t, func() { + body := strings.NewReader(`{"build_hierarchies":[{"state":"completed"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - missing mandatory fields: [dimension_name]") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task without a 'dimension_name' returns http 400 response", t, func() { + body := strings.NewReader(`{"build_hierarchies":[{"dimension_name":"geography"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - missing mandatory fields: [state]") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task with an invalid state returns http 400 response", t, func() { + body := strings.NewReader(`{"build_hierarchies":[{"state":"notvalid", "dimension_name": "geography"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - invalid task state value: notvalid") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task with an invalid state returns http 400 response", t, func() { + body := strings.NewReader(`{"build_hierarchies":[{"state":"completed", "dimension_name": "geography"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { + return errors.New("not found") + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusNotFound) + So(w.Body.String(), ShouldContainSubstring, "geography hierarchy import task does not exist") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task but lose connection to datastore when updating resource", t, func() { + body := strings.NewReader(`{"build_hierarchies":[{"state":"completed", "dimension_name": "geography"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { + return errors.New("internal error") + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) +} + +func TestStore_UpdateImportTask_UpdateBuildHierarchyTask(t *testing.T) { + + t.Parallel() + Convey("update to an import task returns http 200 response if no errors occur", t, func() { + body := strings.NewReader(`{"build_hierarchies":[{"state":"completed", "dimension_name":"geography"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Successful, ap), + ) + }) +} + +func TestStore_UpdateImportTask_ReturnsInternalError(t *testing.T) { + + t.Parallel() + Convey("update to an import task returns an internal error", t, func() { + body := strings.NewReader(`{"import_observations":{"state":"completed"}}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.EditionConfirmedState}, nil + }, + UpdateImportObservationsTaskStateFunc: func(id string, state string) error { + return errs.ErrInternalServer + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) +} + +func TestUpdateInstanceReturnsErrorWhenStateIsPublished(t *testing.T) { + t.Parallel() + Convey("when an instance has a state of published, then put request to change to it to completed ", t, func() { + body := strings.NewReader(`{"state":"completed"}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.PublishedState}, nil + }, + UpdateInstanceFunc: func(id string, i *models.Instance) error { + return nil + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusForbidden) + So(w.Body.String(), ShouldContainSubstring, errs.ErrResourcePublished.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.AddVersionDetailsToInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateInstanceAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateInstanceAction, audit.Unsuccessful, ap), + ) + }) +} + +func TestUpdateDimensionReturnsInternalError(t *testing.T) { + t.Parallel() Convey("Given an internal error is returned from mongo, then response returns an internal error", t, func() { body := strings.NewReader(`{"label":"ages"}`) - r := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", body) + r, err := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -701,17 +1598,28 @@ func TestUpdateDimensionReturnsInternalError(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.UpdateDimension(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateDimensionAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateDimensionAction, audit.Unsuccessful, ap), + ) }) Convey("Given the instance state is invalid, then response returns an internal error", t, func() { body := strings.NewReader(`{"label":"ages"}`) - r := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", body) + r, err := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -723,19 +1631,29 @@ func TestUpdateDimensionReturnsInternalError(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.UpdateDimension(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateDimensionAction, audit.Attempted, ap), + ) }) } func TestUpdateDimensionReturnsNotFound(t *testing.T) { t.Parallel() Convey("When update dimension return status not found", t, func() { - r := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", nil) + r, err := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -744,19 +1662,30 @@ func TestUpdateDimensionReturnsNotFound(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.UpdateDimension(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInstanceNotFound.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateDimensionAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateDimensionAction, audit.Unsuccessful, ap), + ) }) } func TestUpdateDimensionReturnsForbidden(t *testing.T) { t.Parallel() Convey("When update dimension returns forbidden (for already published) ", t, func() { - r := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", nil) + r, err := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", nil) + So(err, ShouldBeNil) w := httptest.NewRecorder() currentInstanceTestData := &models.Instance{ @@ -769,12 +1698,22 @@ func TestUpdateDimensionReturnsForbidden(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.UpdateDimension(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusForbidden) + So(w.Body.String(), ShouldContainSubstring, errs.ErrResourcePublished.Error()) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateDimensionAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateDimensionAction, audit.Unsuccessful, ap), + ) }) } @@ -782,7 +1721,8 @@ func TestUpdateDimensionReturnsBadRequest(t *testing.T) { t.Parallel() Convey("When update dimension returns bad request", t, func() { body := strings.NewReader("{") - r := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", body) + r, err := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -791,10 +1731,21 @@ func TestUpdateDimensionReturnsBadRequest(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - instance.UpdateDimension(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "unexpected end of JSON input") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) + So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateDimensionAction, audit.Attempted, ap), + ) }) } @@ -802,7 +1753,8 @@ func TestUpdateDimensionReturnsNotFoundWithWrongName(t *testing.T) { t.Parallel() Convey("When update dimension fails to update an instance", t, func() { body := strings.NewReader(`{"label":"notages"}`) - r := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/notage", body) + r, err := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/notage", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -816,16 +1768,21 @@ func TestUpdateDimensionReturnsNotFoundWithWrongName(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - - router := mux.NewRouter() - router.HandleFunc("/instances/{id}/dimensions/{dimension}", instance.UpdateDimension).Methods("PUT") - - router.ServeHTTP(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusNotFound) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(w.Body.String(), ShouldContainSubstring, errs.ErrDimensionNotFound.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateDimensionAction, audit.Attempted, ap), + ) }) } @@ -833,7 +1790,8 @@ func TestUpdateDimensionReturnsOk(t *testing.T) { t.Parallel() Convey("When update dimension fails to update an instance", t, func() { body := strings.NewReader(`{"label":"ages"}`) - r := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", body) + r, err := createRequestWithToken("PUT", "http://localhost:22000/instances/123/dimensions/age", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ @@ -847,40 +1805,303 @@ func TestUpdateDimensionReturnsOk(t *testing.T) { }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - router := mux.NewRouter() - router.HandleFunc("/instances/{id}/dimensions/{dimension}", instance.UpdateDimension).Methods("PUT") - - router.ServeHTTP(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) - So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 2) So(len(mockedDataStore.UpdateInstanceCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateDimensionAction, audit.Attempted, ap), + ) }) } -func TestStore_UpdateImportTask_UpdateBuildSearchIndexTask_InvalidState(t *testing.T) { +func TestStore_UpdateImportTask_UpdateBuildSearchIndexTask_Failure(t *testing.T) { t.Parallel() - Convey("update to an import task with an invalid state returns http 400 response", t, func() { - body := strings.NewReader(`{"build_search_indexes":[{"state":"notvalid"}]}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + Convey("update to an import task with invalid json returns http 400 response", t, func() { + body := strings.NewReader(`{`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, errs.ErrUnableToParseJSON.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task with an empty request body returns http 400 response", t, func() { + body := strings.NewReader(`{}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - request body does not contain any import tasks") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task without specifying a task returns http 400 response", t, func() { + body := strings.NewReader(`{"build_search_indexes":[]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - missing search index task") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task without a 'dimension_name' returns http 400 response", t, func() { + body := strings.NewReader(`{"build_search_indexes":[{"state":"completed"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - missing mandatory fields: [dimension_name]") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task without a 'dimension_name' returns http 400 response", t, func() { + body := strings.NewReader(`{"build_search_indexes":[{"dimension_name":"geography"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { return nil }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - missing mandatory fields: [state]") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task with an invalid state returns http 400 response", t, func() { + body := strings.NewReader(`{"build_search_indexes":[{"state":"notvalid", "dimension_name": "geography"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { + return nil + }, + } - instance.UpdateImportTask(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusBadRequest) + So(w.Body.String(), ShouldContainSubstring, "bad request - invalid task state value: notvalid") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task with a dimension that does not exist returns http 404 response", t, func() { + body := strings.NewReader(`{"build_search_indexes":[{"state":"completed", "dimension_name": "geography"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { + return errors.New("not found") + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusNotFound) + So(w.Body.String(), ShouldContainSubstring, "geography search index import task does not exist") + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + + Convey("update to an import task but lose connection to datastore when updating resource", t, func() { + body := strings.NewReader(`{"build_search_indexes":[{"state":"completed", "dimension_name": "geography"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { + return errors.New("internal error") + }, + } + + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) }) } @@ -888,23 +2109,274 @@ func TestStore_UpdateImportTask_UpdateBuildSearchIndexTask(t *testing.T) { t.Parallel() Convey("update to an import task returns http 200 response if no errors occur", t, func() { - body := strings.NewReader(`{"build_search_indexes":[{"state":"completed"}]}`) - r := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + body := strings.NewReader(`{"build_search_indexes":[{"state":"completed", "dimension_name": "geography"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) w := httptest.NewRecorder() mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { return nil }, } - instance := &instance.Store{Host: host, Storer: mockedDataStore} - - instance.UpdateImportTask(w, r) + auditor := audit_mock.New() + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Successful, ap), + ) }) } + +func TestStore_UpdateImportTask_AuditAttemptedError(t *testing.T) { + t.Parallel() + Convey("given audit action attempted returns an error", t, func() { + auditor := audit_mock.NewErroring(instance.UpdateImportTasksAction, audit.Attempted) + + Convey("when update import task is called", func() { + body := strings.NewReader(`{"build_search_indexes":[{"state":"completed"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{} + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 1) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + ) + }) + }) + }) +} + +func TestStore_UpdateImportTask_AuditUnsuccessfulError(t *testing.T) { + t.Parallel() + Convey("given audit action unsuccessful returns an error", t, func() { + Convey("when the request body fails to marshal into the updateImportTask model", func() { + auditor := audit_mock.NewErroring(instance.UpdateImportTasksAction, audit.Unsuccessful) + body := strings.NewReader(`THIS IS NOT JSON`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + }) + + Convey("when UpdateImportObservationsTaskState returns an error", func() { + auditor := audit_mock.NewErroring(instance.UpdateImportTasksAction, audit.Unsuccessful) + body := strings.NewReader(`{"import_observations":{"state":"completed"}}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateImportObservationsTaskStateFunc: func(id string, state string) error { + return errors.New("error") + }, + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + }) + + Convey("when UpdateBuildHierarchyTaskState returns an error", func() { + auditor := audit_mock.NewErroring(instance.UpdateImportTasksAction, audit.Unsuccessful) + body := strings.NewReader(`{"build_hierarchies":[{"dimension_name": "geography", "state":"completed"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { + return errors.New("error") + }, + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + }) + + Convey("when UpdateBuildSearchTaskState returns an error", func() { + auditor := audit_mock.NewErroring(instance.UpdateImportTasksAction, audit.Unsuccessful) + body := strings.NewReader(`{"build_search_indexes":[{"dimension_name": "geography", "state":"completed"}]}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateBuildSearchTaskStateFunc: func(id string, dimension string, state string) error { + return errors.New("error") + }, + } + + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusInternalServerError) + So(w.Body.String(), ShouldContainSubstring, errs.ErrInternalServer.Error()) + + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 1) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Unsuccessful, ap), + ) + }) + }) + }) +} + +func TestStore_UpdateImportTask_AuditSuccessfulError(t *testing.T) { + t.Parallel() + Convey("given audit action successful returns an error", t, func() { + auditor := audit_mock.NewErroring(instance.UpdateImportTasksAction, audit.Successful) + + Convey("when update import task is called", func() { + body := strings.NewReader(`{"import_observations":{"state":"completed"}}`) + r, err := createRequestWithToken("PUT", "http://localhost:21800/instances/123/import_tasks", body) + So(err, ShouldBeNil) + w := httptest.NewRecorder() + + mockedDataStore := &storetest.StorerMock{ + GetInstanceFunc: func(id string) (*models.Instance, error) { + return &models.Instance{State: models.CreatedState}, nil + }, + UpdateImportObservationsTaskStateFunc: func(id string, state string) error { + return nil + }, + } + datasetAPI := getAPIWithMockedDatastore(mockedDataStore, &mocks.DownloadsGeneratorMock{}, auditor, &mocks.ObservationStoreMock{}) + datasetAPI.Router.ServeHTTP(w, r) + + Convey("then a 500 status is returned", func() { + So(w.Code, ShouldEqual, http.StatusOK) + So(len(mockedDataStore.GetInstanceCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateImportObservationsTaskStateCalls()), ShouldEqual, 1) + So(len(mockedDataStore.UpdateBuildHierarchyTaskStateCalls()), ShouldEqual, 0) + So(len(mockedDataStore.UpdateBuildSearchTaskStateCalls()), ShouldEqual, 0) + So(len(auditor.RecordCalls()), ShouldEqual, 2) + + ap := common.Params{"instance_id": "123"} + + auditor.AssertRecordCalls( + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Attempted, ap), + audit_mock.NewExpectation(instance.UpdateImportTasksAction, audit.Successful, ap), + ) + }) + }) + }) +} + +var urlBuilder = url.NewBuilder("localhost:20000") + +func getAPIWithMockedDatastore(mockedDataStore store.Storer, mockedGeneratedDownloads api.DownloadsGenerator, mockAuditor api.Auditor, mockedObservationStore api.ObservationStore) *api.DatasetAPI { + cfg, err := config.Get() + So(err, ShouldBeNil) + cfg.ServiceAuthToken = "dataset" + cfg.DatasetAPIURL = "http://localhost:22000" + cfg.EnablePrivateEnpoints = true + cfg.HealthCheckTimeout = 2 * time.Second + + return api.Routes(*cfg, mux.NewRouter(), store.DataStore{Backend: mockedDataStore}, urlBuilder, mockedGeneratedDownloads, mockAuditor, mockedObservationStore) +} diff --git a/instance/instance_internal_test.go b/instance/instance_internal_test.go index 71b184ad..0fe617f4 100644 --- a/instance/instance_internal_test.go +++ b/instance/instance_internal_test.go @@ -5,6 +5,7 @@ import ( "strings" "testing" + errs "github.com/ONSdigital/dp-dataset-api/apierrors" . "github.com/smartystreets/goconvey/convey" ) @@ -21,7 +22,7 @@ func TestUnmarshalInstanceWithBadReader(t *testing.T) { Convey("Create an instance with an invalid reader", t, func() { instance, err := unmarshalInstance(Reader{}, true) So(instance, ShouldBeNil) - So(err.Error(), ShouldEqual, "Failed to read message body") + So(err.Error(), ShouldEqual, "failed to read message body") }) } @@ -29,7 +30,7 @@ func TestUnmarshalInstanceWithInvalidJson(t *testing.T) { Convey("Create an instance with invalid json", t, func() { instance, err := unmarshalInstance(strings.NewReader("{ "), true) So(instance, ShouldBeNil) - So(err.Error(), ShouldContainSubstring, "Failed to parse json body") + So(err.Error(), ShouldContainSubstring, errs.ErrUnableToParseJSON.Error()) }) } @@ -37,25 +38,25 @@ func TestUnmarshalInstanceWithEmptyJson(t *testing.T) { Convey("Create an instance with empty json", t, func() { instance, err := unmarshalInstance(strings.NewReader("{ }"), true) So(instance, ShouldBeNil) - So(err.Error(), ShouldEqual, "Missing job properties") + So(err.Error(), ShouldEqual, errs.ErrMissingJobProperties.Error()) }) Convey("Create an instance with empty job link", t, func() { instance, err := unmarshalInstance(strings.NewReader(`{"links":{"job": null}}`), true) So(instance, ShouldBeNil) - So(err.Error(), ShouldEqual, "Missing job properties") + So(err.Error(), ShouldEqual, errs.ErrMissingJobProperties.Error()) }) Convey("Create an instance with empty href in job link", t, func() { instance, err := unmarshalInstance(strings.NewReader(`{"links":{"job":{"id": "456"}}}`), true) So(instance, ShouldBeNil) - So(err.Error(), ShouldEqual, "Missing job properties") + So(err.Error(), ShouldEqual, errs.ErrMissingJobProperties.Error()) }) Convey("Create an instance with empty href in job link", t, func() { instance, err := unmarshalInstance(strings.NewReader(`{"links":{"job":{"href": "http://localhost:21800/jobs/456"}}}`), true) So(instance, ShouldBeNil) - So(err.Error(), ShouldEqual, "Missing job properties") + So(err.Error(), ShouldEqual, errs.ErrMissingJobProperties.Error()) }) Convey("Update an instance with empty json", t, func() { @@ -69,13 +70,13 @@ func TestUnmarshalInstanceWithMissingFields(t *testing.T) { Convey("Create an instance with no id", t, func() { instance, err := unmarshalInstance(strings.NewReader(`{"links": { "job": { "link":"http://localhost:2200/jobs/123-456" } }}`), true) So(instance, ShouldBeNil) - So(err.Error(), ShouldEqual, "Missing job properties") + So(err.Error(), ShouldEqual, errs.ErrMissingJobProperties.Error()) }) Convey("Create an instance with no link", t, func() { instance, err := unmarshalInstance(strings.NewReader(`{"links": { "job": {"id":"123-456"} }}`), true) So(instance, ShouldBeNil) - So(err.Error(), ShouldEqual, "Missing job properties") + So(err.Error(), ShouldEqual, errs.ErrMissingJobProperties.Error()) }) Convey("Update an instance with no id", t, func() { diff --git a/main.go b/main.go index 83acf36a..d31ef5c5 100644 --- a/main.go +++ b/main.go @@ -11,6 +11,7 @@ import ( "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/download" "github.com/ONSdigital/dp-dataset-api/mongo" + "github.com/ONSdigital/dp-dataset-api/neo4j" "github.com/ONSdigital/dp-dataset-api/schema" "github.com/ONSdigital/dp-dataset-api/store" "github.com/ONSdigital/dp-filter/observation" @@ -25,9 +26,18 @@ import ( mongolib "github.com/ONSdigital/go-ns/mongo" "github.com/pkg/errors" - bolt "github.com/ONSdigital/golang-neo4j-bolt-driver" + bolt "github.com/johnnadratowski/golang-neo4j-bolt-driver" ) +// check that DatsetAPIStore satifies the the store.Storer interface +var _ store.Storer = (*DatsetAPIStore)(nil) + +//DatsetAPIStore is a wrapper which embeds Neo4j Mongo stucts which between them satisfy the store.Storer interface. +type DatsetAPIStore struct { + *mongo.Mongo + *neo4j.Neo4j +} + func main() { log.Namespace = "dp-dataset-api" @@ -66,7 +76,7 @@ func main() { auditor = &audit.NopAuditor{} } - mongo := &mongo.Mongo{ + mongodb := &mongo.Mongo{ CodeListURL: cfg.CodeListAPIURL, Collection: cfg.MongoConfig.Collection, Database: cfg.MongoConfig.Database, @@ -74,26 +84,28 @@ func main() { URI: cfg.MongoConfig.BindAddr, } - session, err := mongo.Init() + session, err := mongodb.Init() if err != nil { log.ErrorC("failed to initialise mongo", err, nil) os.Exit(1) } - mongo.Session = session + mongodb.Session = session log.Debug("listening...", log.Data{ "bind_address": cfg.BindAddr, }) - store := store.DataStore{Backend: mongo} - neo4jConnPool, err := bolt.NewClosableDriverPool(cfg.Neo4jBindAddress, cfg.Neo4jPoolSize) if err != nil { log.ErrorC("failed to connect to neo4j connection pool", err, nil) os.Exit(1) } + neoDB := &neo4j.Neo4j{neo4jConnPool} + + store := store.DataStore{Backend: DatsetAPIStore{Mongo: mongodb, Neo4j: neoDB}} + observationStore := observation.NewStore(neo4jConnPool) downloadGenerator := &download.Generator{ @@ -104,7 +116,7 @@ func main() { healthTicker := healthcheck.NewTicker( cfg.HealthCheckInterval, neo4jhealth.NewHealthCheckClient(neo4jConnPool), - mongolib.NewHealthCheckClient(mongo.Session), + mongolib.NewHealthCheckClient(mongodb.Session), ) apiErrors := make(chan error, 1) diff --git a/models/dataset.go b/models/dataset.go index 1d957fc5..553bb83d 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -13,6 +13,13 @@ import ( "github.com/satori/go.uuid" ) +// List of error variables +var ( + ErrPublishedVersionCollectionIDInvalid = errors.New("unexpected collection_id in published version") + ErrAssociatedVersionCollectionIDInvalid = errors.New("missing collection_id for association between version and a collection") + ErrVersionStateInvalid = errors.New("incorrect state, can be one of the following: edition-confirmed, associated or published") +) + // DatasetResults represents a structure for a list of datasets type DatasetResults struct { Items []*Dataset `json:"items"` @@ -198,6 +205,7 @@ type TemporalFrequency struct { StartDate string `bson:"start_date,omitempty" json:"start_date,omitempty"` } +// UsageNote represents a note containing extra information associated to the resource type UsageNote struct { Title string `bson:"title,omitempty" json:"title,omitempty"` Note string `bson:"note,omitempty" json:"note,omitempty"` @@ -242,14 +250,14 @@ func CheckState(docType, state string) error { func CreateDataset(reader io.Reader) (*Dataset, error) { b, err := ioutil.ReadAll(reader) if err != nil { - return nil, errors.New("Failed to read message body") + return nil, errors.New("failed to read message body") } var dataset Dataset err = json.Unmarshal(b, &dataset) if err != nil { - return nil, errors.New("Failed to parse json body") + return nil, errors.New("failed to parse json body") } return &dataset, nil } @@ -258,7 +266,7 @@ func CreateDataset(reader io.Reader) (*Dataset, error) { func CreateVersion(reader io.Reader) (*Version, error) { b, err := ioutil.ReadAll(reader) if err != nil { - return nil, errors.New("Failed to read message body") + return nil, errors.New("failed to read message body") } var version Version @@ -267,7 +275,7 @@ func CreateVersion(reader io.Reader) (*Version, error) { err = json.Unmarshal(b, &version) if err != nil { - return nil, errors.New("Failed to parse json body") + return nil, errors.New("failed to parse json body") } return &version, nil @@ -292,12 +300,12 @@ func CreateDownloadList(reader io.Reader) (*DownloadList, error) { func CreateContact(reader io.Reader) (*Contact, error) { b, err := ioutil.ReadAll(reader) if err != nil { - return nil, errors.New("Failed to read message body") + return nil, errors.New("failed to read message body") } var contact Contact err = json.Unmarshal(b, &contact) if err != nil { - return nil, errors.New("Failed to parse json body") + return nil, errors.New("failed to parse json body") } // Create unique id @@ -315,14 +323,14 @@ func ValidateVersion(version *Version) error { case EditionConfirmedState: case PublishedState: if version.CollectionID != "" { - return errors.New("Unexpected collection_id in published version") + return ErrPublishedVersionCollectionIDInvalid } case AssociatedState: if version.CollectionID == "" { - return errors.New("Missing collection_id for association between version and a collection") + return ErrAssociatedVersionCollectionIDInvalid } default: - return errors.New("Incorrect state, can be one of the following: edition-confirmed, associated or published") + return ErrVersionStateInvalid } var missingFields []string diff --git a/models/dataset_test.go b/models/dataset_test.go index 04431c75..8b68cba6 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -90,7 +90,7 @@ func TestCreateDataset(t *testing.T) { version, err := CreateDataset(r) So(version, ShouldBeNil) So(err, ShouldNotBeNil) - So(err.Error(), ShouldResemble, errors.New("Failed to parse json body").Error()) + So(err.Error(), ShouldResemble, errors.New("failed to parse json body").Error()) }) } @@ -130,7 +130,7 @@ func TestCreateVersion(t *testing.T) { version, err := CreateVersion(r) So(version, ShouldBeNil) So(err, ShouldNotBeNil) - So(err.Error(), ShouldResemble, errors.New("Failed to parse json body").Error()) + So(err.Error(), ShouldResemble, errors.New("failed to parse json body").Error()) }) } @@ -161,14 +161,14 @@ func TestValidateVersion(t *testing.T) { err := ValidateVersion(&Version{State: ""}) So(err, ShouldNotBeNil) - So(err.Error(), ShouldResemble, errors.New("Missing state from version").Error()) + So(err.Error(), ShouldResemble, errors.New("missing state from version").Error()) }) Convey("when the version state is set to an invalid value", func() { err := ValidateVersion(&Version{State: SubmittedState}) So(err, ShouldNotBeNil) - So(err.Error(), ShouldResemble, errors.New("Incorrect state, can be one of the following: edition-confirmed, associated or published").Error()) + So(err.Error(), ShouldResemble, errors.New("incorrect state, can be one of the following: edition-confirmed, associated or published").Error()) }) Convey("when mandatory fields are missing from version document when state is set to created", func() { @@ -187,7 +187,7 @@ func TestValidateVersion(t *testing.T) { err := ValidateVersion(version) So(err, ShouldNotBeNil) - So(err.Error(), ShouldResemble, errors.New("Unexpected collection_id in published version").Error()) + So(err.Error(), ShouldResemble, errors.New("unexpected collection_id in published version").Error()) }) Convey("when version downloads are invalid", func() { diff --git a/models/dimension.go b/models/dimension.go index 92c236b9..55d29a68 100644 --- a/models/dimension.go +++ b/models/dimension.go @@ -31,24 +31,24 @@ type DimensionLink struct { // CachedDimensionOption contains information used to create a dimension option type CachedDimensionOption struct { - Name string `bson:"name,omitempty" json:"dimension"` Code string `bson:"code,omitempty" json:"code"` - NodeID string `bson:"node_id,omitempty" json:"node_id"` - InstanceID string `bson:"instance_id,omitempty" json:"instance_id,omitempty"` CodeList string `bson:"code_list,omitempty" json:"code_list,omitempty"` - Option string `bson:"option,omitempty" json:"option"` + InstanceID string `bson:"instance_id,omitempty" json:"instance_id,omitempty"` Label string `bson:"label,omitempty" json:"label"` + Name string `bson:"name,omitempty" json:"dimension"` + NodeID string `bson:"node_id,omitempty" json:"node_id"` + Option string `bson:"option,omitempty" json:"option"` } // DimensionOption contains unique information and metadata used when processing the data type DimensionOption struct { - Name string `bson:"name,omitempty" json:"dimension"` + InstanceID string `bson:"instance_id,omitempty" json:"instance_id,omitempty"` Label string `bson:"label,omitempty" json:"label"` + LastUpdated time.Time `bson:"last_updated,omitempty" json:"-"` Links DimensionOptionLinks `bson:"links,omitempty" json:"links"` - Option string `bson:"option,omitempty" json:"option"` + Name string `bson:"name,omitempty" json:"dimension"` NodeID string `bson:"node_id,omitempty" json:"node_id"` - InstanceID string `bson:"instance_id,omitempty" json:"instance_id,omitempty"` - LastUpdated time.Time `bson:"last_updated,omitempty" json:"-"` + Option string `bson:"option,omitempty" json:"option"` } // PublicDimensionOption hides values which are only used by interval services diff --git a/models/instance.go b/models/instance.go index 0500e9d7..82334282 100644 --- a/models/instance.go +++ b/models/instance.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" "time" + + "github.com/gedge/mgo/bson" ) // Instance which presents a single dataset being imported @@ -24,6 +26,7 @@ type Instance struct { TotalObservations *int `bson:"total_observations,omitempty" json:"total_observations,omitempty"` Version int `bson:"version,omitempty" json:"version,omitempty"` LastUpdated time.Time `bson:"last_updated,omitempty" json:"last_updated,omitempty"` + UniqueTimestamp bson.MongoTimestamp `bson:"unique_timestamp" json:"-"` ImportTasks *InstanceImportTasks `bson:"import_tasks,omitempty" json:"import_tasks"` } @@ -42,15 +45,18 @@ type ImportObservationsTask struct { // BuildHierarchyTask represents a task of importing a single hierarchy. type BuildHierarchyTask struct { - State string `bson:"state,omitempty" json:"state,omitempty"` + CodeListID string `bson:"code_list_id,omitempty" json:"code_list_id,omitempty"` + GenericTaskDetails `bson:",inline"` +} + +type GenericTaskDetails struct { DimensionName string `bson:"dimension_name,omitempty" json:"dimension_name,omitempty"` - CodeListID string `bson:"code_list_id,omitempty" json:"code_list_id,omitempty"` + State string `bson:"state,omitempty" json:"state,omitempty"` } // BuildSearchIndexTask represents a task of importing a single search index into search. type BuildSearchIndexTask struct { - State string `bson:"state,omitempty" json:"state,omitempty"` - DimensionName string `bson:"dimension_name,omitempty" json:"dimension_name,omitempty"` + GenericTaskDetails `bson:",inline"` } // CodeList for a dimension within an instance @@ -142,3 +148,26 @@ func ValidateInstanceState(state string) error { return nil } + +// ValidateImportTask checks the task contains mandatory fields +func ValidateImportTask(task GenericTaskDetails) error { + var missingFields []string + + if task.DimensionName == "" { + missingFields = append(missingFields, "dimension_name") + } + + if task.State == "" { + missingFields = append(missingFields, "state") + } + + if len(missingFields) > 0 { + return fmt.Errorf("bad request - missing mandatory fields: %v", missingFields) + } + + if task.State != CompletedState { + return fmt.Errorf("bad request - invalid task state value: %v", task.State) + } + + return nil +} diff --git a/models/instance_test.go b/models/instance_test.go index dd250b2f..79de9fae 100644 --- a/models/instance_test.go +++ b/models/instance_test.go @@ -3,6 +3,7 @@ package models import ( "errors" "testing" + "time" . "github.com/smartystreets/goconvey/convey" ) @@ -69,3 +70,185 @@ func TestValidateStateFilter(t *testing.T) { }) }) } + +func TestValidateEvent(t *testing.T) { + currentTime := time.Now().UTC() + + t.Parallel() + Convey("Given an event contains all mandatory fields", t, func() { + Convey("Then successfully return without any errors ", func() { + event := &Event{ + Message: "test message", + MessageOffset: "56", + Time: ¤tTime, + Type: "error", + } + err := event.Validate() + So(err, ShouldBeNil) + }) + }) + + Convey("Given event is missing 'message' field from event", t, func() { + Convey("Then event fails validation and returns an error 'missing properties'", func() { + event := &Event{ + MessageOffset: "56", + Time: ¤tTime, + Type: "error", + } + err := event.Validate() + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "missing properties") + }) + }) + + Convey("Given event is missing 'message_offset' field from event", t, func() { + Convey("Then event fails validation and returns an error 'missing properties'", func() { + event := &Event{ + Message: "test message", + Time: ¤tTime, + Type: "error", + } + err := event.Validate() + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "missing properties") + }) + }) + + Convey("Given event is missing 'time' field from event", t, func() { + Convey("Then event fails validation and returns an error 'missing properties'", func() { + event := &Event{ + Message: "test message", + MessageOffset: "56", + Type: "error", + } + err := event.Validate() + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "missing properties") + }) + }) + + Convey("Given event is missing 'type' field from event", t, func() { + Convey("Then event fails validation and returns an error 'missing properties'", func() { + event := &Event{ + Message: "test message", + MessageOffset: "56", + Time: ¤tTime, + } + err := event.Validate() + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "missing properties") + }) + }) +} + +func TestValidateInstanceState(t *testing.T) { + + t.Parallel() + Convey("Given a state of 'created'", t, func() { + Convey("Then successfully return without any errors", func() { + err := ValidateInstanceState(CreatedState) + So(err, ShouldBeNil) + }) + }) + + Convey("Given a state of 'submitted'", t, func() { + Convey("Then successfully return without any errors ", func() { + err := ValidateInstanceState(SubmittedState) + So(err, ShouldBeNil) + }) + }) + + Convey("Given a state of 'completed'", t, func() { + Convey("Then successfully return without any errors", func() { + err := ValidateInstanceState(CompletedState) + So(err, ShouldBeNil) + }) + }) + + Convey("Given a state of 'edition-confirmed'", t, func() { + Convey("Then successfully return without any errors", func() { + err := ValidateInstanceState(EditionConfirmedState) + So(err, ShouldBeNil) + }) + }) + + Convey("Given a state of 'associated'", t, func() { + Convey("Then successfully return without any errors", func() { + err := ValidateInstanceState(AssociatedState) + So(err, ShouldBeNil) + }) + }) + + Convey("Given a state of 'published'", t, func() { + Convey("Then successfully return without any errors", func() { + err := ValidateInstanceState(PublishedState) + So(err, ShouldBeNil) + }) + }) + + Convey("Given a state of 'gobbledygook'", t, func() { + Convey("Then validation of state fails and returns an error", func() { + err := ValidateInstanceState("gobbledygook") + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "bad request - invalid filter state values: [gobbledygook]") + }) + }) +} + +func TestValidateImportTask(t *testing.T) { + + t.Parallel() + Convey("Given an import task contains all mandatory fields and state is set to 'completed'", t, func() { + Convey("Then successfully return without any errors", func() { + task := GenericTaskDetails{ + DimensionName: "geography", + State: CompletedState, + } + err := ValidateImportTask(task) + So(err, ShouldBeNil) + }) + }) + + Convey("Given an import task is missing mandatory field 'dimension_name'", t, func() { + Convey("Then import task fails validation and returns an error", func() { + task := GenericTaskDetails{ + State: CompletedState, + } + err := ValidateImportTask(task) + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "bad request - missing mandatory fields: [dimension_name]") + }) + }) + + Convey("Given an import task is missing mandatory field 'state'", t, func() { + Convey("Then import task fails validation and returns error", func() { + task := GenericTaskDetails{ + DimensionName: "geography", + } + err := ValidateImportTask(task) + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "bad request - missing mandatory fields: [state]") + }) + }) + + Convey("Given an import task is missing mandatory field 'state' and 'dimension_name'", t, func() { + Convey("Then import task fails validation and returns an error", func() { + task := GenericTaskDetails{} + err := ValidateImportTask(task) + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "bad request - missing mandatory fields: [dimension_name state]") + }) + }) + + Convey("Given an import task contains an invalid state, 'submitted'", t, func() { + Convey("Then import task fails validation and returns an error", func() { + task := GenericTaskDetails{ + DimensionName: "geography", + State: SubmittedState, + } + err := ValidateImportTask(task) + So(err, ShouldNotBeNil) + So(err.Error(), ShouldEqual, "bad request - invalid task state value: submitted") + }) + }) +} diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index 4224c8c4..0376be01 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -7,17 +7,14 @@ import ( "sync" "time" - "github.com/ONSdigital/dp-dataset-api/models" - "github.com/ONSdigital/dp-dataset-api/store" - "github.com/ONSdigital/go-ns/log" + "github.com/gedge/mgo" + "github.com/gedge/mgo/bson" errs "github.com/ONSdigital/dp-dataset-api/apierrors" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "github.com/ONSdigital/dp-dataset-api/models" + "github.com/ONSdigital/go-ns/log" ) -var _ store.Storer = &Mongo{} - // Mongo represents a simplistic MongoDB configuration. type Mongo struct { CodeListURL string diff --git a/mongo/dataset_test.go b/mongo/dataset_test.go index 45372c18..53642c8e 100644 --- a/mongo/dataset_test.go +++ b/mongo/dataset_test.go @@ -3,7 +3,7 @@ package mongo import ( "testing" - "gopkg.in/mgo.v2/bson" + "github.com/gedge/mgo/bson" "github.com/ONSdigital/dp-dataset-api/models" . "github.com/smartystreets/goconvey/convey" diff --git a/mongo/dimension_store.go b/mongo/dimension_store.go index 8ec2b5c3..aa0ce55a 100644 --- a/mongo/dimension_store.go +++ b/mongo/dimension_store.go @@ -6,13 +6,13 @@ import ( errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" - "gopkg.in/mgo.v2/bson" + "github.com/gedge/mgo/bson" ) const dimensionOptions = "dimension.options" -// GetDimensionNodesFromInstance which are stored in a mongodb collection -func (m *Mongo) GetDimensionNodesFromInstance(id string) (*models.DimensionNodeResults, error) { +// GetDimensionsFromInstance returns a list of dimensions and their options for an instance resource +func (m *Mongo) GetDimensionsFromInstance(id string) (*models.DimensionNodeResults, error) { s := m.Session.Copy() defer s.Close() @@ -27,8 +27,8 @@ func (m *Mongo) GetDimensionNodesFromInstance(id string) (*models.DimensionNodeR return &models.DimensionNodeResults{Items: dimensions}, nil } -// GetUniqueDimensionValues which are stored in mongodb collection -func (m *Mongo) GetUniqueDimensionValues(id, dimension string) (*models.DimensionValues, error) { +// GetUniqueDimensionAndOptions returns a list of dimension options for an instance resource +func (m *Mongo) GetUniqueDimensionAndOptions(id, dimension string) (*models.DimensionValues, error) { s := m.Session.Copy() defer s.Close() diff --git a/mongo/instance_store.go b/mongo/instance_store.go index 09e1fc4f..382fe77c 100644 --- a/mongo/instance_store.go +++ b/mongo/instance_store.go @@ -6,8 +6,8 @@ import ( errs "github.com/ONSdigital/dp-dataset-api/apierrors" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/go-ns/log" - "gopkg.in/mgo.v2" - "gopkg.in/mgo.v2/bson" + "github.com/gedge/mgo" + "github.com/gedge/mgo/bson" ) const instanceCollection = "instances" @@ -62,8 +62,11 @@ func (m *Mongo) AddInstance(instance *models.Instance) (*models.Instance, error) defer s.Close() instance.LastUpdated = time.Now().UTC() - err := s.DB(m.Database).C(instanceCollection).Insert(&instance) - if err != nil { + var err error + if instance.UniqueTimestamp, err = bson.NewMongoTimestamp(instance.LastUpdated, 1); err != nil { + return nil, err + } + if err = s.DB(m.Database).C(instanceCollection).Insert(&instance); err != nil { return nil, err } @@ -116,7 +119,7 @@ func (m *Mongo) UpdateDimensionNodeID(dimension *models.DimensionOption) error { err := s.DB(m.Database).C(dimensionOptions).Update(bson.M{"instance_id": dimension.InstanceID, "name": dimension.Name, "option": dimension.Option}, bson.M{"$set": bson.M{"node_id": &dimension.NodeID, "last_updated": time.Now().UTC()}}) if err == mgo.ErrNotFound { - return errs.ErrInstanceNotFound + return errs.ErrDimensionOptionNotFound } if err != nil { diff --git a/neo4j/instance_store.go b/neo4j/instance_store.go new file mode 100644 index 00000000..e7998463 --- /dev/null +++ b/neo4j/instance_store.go @@ -0,0 +1,87 @@ +package neo4j + +import ( + "context" + "fmt" + + "github.com/ONSdigital/go-ns/log" + "github.com/pkg/errors" + + bolt "github.com/johnnadratowski/golang-neo4j-bolt-driver" +) + +const addVersionDetailsToInstance = "MATCH (i:`_%s_Instance`) SET i.dataset_id = {dataset_id}, i.edition = {edition}, i.version = {version} RETURN i" + +//go:generate moq -out ./mocks/bolt.go -pkg mocks . DBPool BoltConn BoltStmt BoltResult + +type BoltConn bolt.Conn +type BoltStmt bolt.Stmt +type BoltResult bolt.Result + +// DBPool provides a pool of database connections +type DBPool interface { + OpenPool() (bolt.Conn, error) + Close() error +} + +type Neo4j struct { + Pool DBPool +} + +func (c *Neo4j) AddVersionDetailsToInstance(ctx context.Context, instanceID string, datasetID string, edition string, version int) error { + data := log.Data{ + "instance_id": instanceID, + "dataset_id": datasetID, + "edition": edition, + "version": version, + } + + conn, err := c.Pool.OpenPool() + if err != nil { + return errors.WithMessage(err, "neoClient AddVersionDetailsToInstance: error opening neo4j connection") + } + + defer conn.Close() + + query := fmt.Sprintf(addVersionDetailsToInstance, instanceID) + stmt, err := conn.PrepareNeo(query) + if err != nil { + return errors.WithMessage(err, "neoClient AddVersionDetailsToInstance: error preparing neo update statement") + } + + defer stmt.Close() + + params := map[string]interface{}{ + "dataset_id": datasetID, + "edition": edition, + "version": version, + } + expectedResult := int64(len(params)) + result, err := stmt.ExecNeo(params) + + if err != nil { + return errors.WithMessage(err, "neoClient AddVersionDetailsToInstance: error executing neo4j update statement") + } + + stats, ok := result.Metadata()["stats"].(map[string]interface{}) + if !ok { + return errors.Errorf("neoClient AddVersionDetailsToInstance: error getting query result stats") + } + + propertiesSet, ok := stats["properties-set"] + if !ok { + return errors.Errorf("neoClient AddVersionDetailsToInstance: error verifying query results") + } + + val, ok := propertiesSet.(int64) + if !ok { + return errors.Errorf("neoClient AddVersionDetailsToInstance: error verifying query results") + } + + if val != expectedResult { + return errors.Errorf("neoClient AddVersionDetailsToInstance: unexpected rows affected expected %d but was %d", expectedResult, val) + } + + log.InfoCtx(ctx, "neoClient AddVersionDetailsToInstance: update successful", data) + return nil +} diff --git a/neo4j/instance_store_test.go b/neo4j/instance_store_test.go new file mode 100644 index 00000000..548f7d8a --- /dev/null +++ b/neo4j/instance_store_test.go @@ -0,0 +1,319 @@ +package neo4j + +import ( + "context" + "fmt" + "testing" + + "github.com/ONSdigital/dp-dataset-api/neo4j/mocks" + "github.com/johnnadratowski/golang-neo4j-bolt-driver" + "github.com/pkg/errors" + . "github.com/smartystreets/goconvey/convey" +) + +const ( + testInstanceID = "666" + testDatasetId = "123" + testEdition = "2018" + testVersion = 1 +) + +var errTest = errors.New("test") +var closeNoErr = func() error { + return nil +} + +func TestNeo4j_AddVersionDetailsToInstanceSuccess(t *testing.T) { + Convey("AddVersionDetailsToInstance completes successfully", t, func() { + res := &mocks.BoltResultMock{ + MetadataFunc: func() map[string]interface{} { + return map[string]interface{}{ + "stats": map[string]interface{}{ + "properties-set": int64(3), + }, + } + }, + } + stmt := &mocks.BoltStmtMock{ + CloseFunc: closeNoErr, + ExecNeoFunc: func(params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) { + return res, nil + }, + } + conn := &mocks.BoltConnMock{ + CloseFunc: closeNoErr, + PrepareNeoFunc: func(query string) (golangNeo4jBoltDriver.Stmt, error) { + return stmt, nil + }, + } + mockPool := &mocks.DBPoolMock{ + OpenPoolFunc: func() (golangNeo4jBoltDriver.Conn, error) { + return conn, nil + }, + } + + store := Neo4j{Pool: mockPool} + + err := store.AddVersionDetailsToInstance(context.Background(), testInstanceID, testDatasetId, testEdition, testVersion) + So(err, ShouldBeNil) + So(len(mockPool.OpenPoolCalls()), ShouldEqual, 1) + + So(len(conn.PrepareNeoCalls()), ShouldEqual, 1) + So(conn.PrepareNeoCalls()[0].Query, ShouldEqual, fmt.Sprintf(addVersionDetailsToInstance, testInstanceID)) + So(len(conn.CloseCalls()), ShouldEqual, 1) + + So(len(stmt.ExecNeoCalls()), ShouldEqual, 1) + So(stmt.ExecNeoCalls()[0].Params, ShouldResemble, map[string]interface{}{ + "dataset_id": testDatasetId, + "edition": testEdition, + "version": testVersion, + }) + So(len(stmt.CloseCalls()), ShouldEqual, 1) + + So(len(res.MetadataCalls()), ShouldEqual, 1) + }) +} + +func TestNeo4j_AddVersionDetailsToInstanceError(t *testing.T) { + Convey("given OpenPool returns an error", t, func() { + mockPool := &mocks.DBPoolMock{ + OpenPoolFunc: func() (golangNeo4jBoltDriver.Conn, error) { + return nil, errTest + }, + } + + store := Neo4j{Pool: mockPool} + + err := store.AddVersionDetailsToInstance(context.Background(), testInstanceID, testDatasetId, testEdition, testVersion) + + Convey("then the expected error is returned", func() { + So(err, ShouldResemble, errors.WithMessage(errTest, "neoClient AddVersionDetailsToInstance: error opening neo4j connection")) + So(len(mockPool.OpenPoolCalls()), ShouldEqual, 1) + }) + }) + + Convey("given conn.PrepareNeo returns an error", t, func() { + conn := &mocks.BoltConnMock{ + PrepareNeoFunc: func(query string) (golangNeo4jBoltDriver.Stmt, error) { + return nil, errTest + }, + CloseFunc: closeNoErr, + } + mockPool := &mocks.DBPoolMock{ + OpenPoolFunc: func() (golangNeo4jBoltDriver.Conn, error) { + return conn, nil + }, + } + + store := Neo4j{Pool: mockPool} + + err := store.AddVersionDetailsToInstance(context.Background(), testInstanceID, testDatasetId, testEdition, testVersion) + + Convey("then the expected error is returned", func() { + So(err, ShouldResemble, errors.WithMessage(errTest, "neoClient AddVersionDetailsToInstance: error preparing neo update statement")) + So(len(mockPool.OpenPoolCalls()), ShouldEqual, 1) + So(len(conn.PrepareNeoCalls()), ShouldEqual, 1) + So(len(conn.CloseCalls()), ShouldEqual, 1) + }) + }) + + Convey("given stmt.ExecNeo returns an error", t, func() { + stmt := &mocks.BoltStmtMock{ + ExecNeoFunc: func(params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) { + return nil, errTest + }, + CloseFunc: closeNoErr, + } + conn := &mocks.BoltConnMock{ + PrepareNeoFunc: func(query string) (golangNeo4jBoltDriver.Stmt, error) { + return stmt, nil + }, + CloseFunc: closeNoErr, + } + mockPool := &mocks.DBPoolMock{ + OpenPoolFunc: func() (golangNeo4jBoltDriver.Conn, error) { + return conn, nil + }, + } + + store := Neo4j{Pool: mockPool} + + err := store.AddVersionDetailsToInstance(context.Background(), testInstanceID, testDatasetId, testEdition, testVersion) + + Convey("then the expected error is returned", func() { + So(err, ShouldResemble, errors.WithMessage(errTest, "neoClient AddVersionDetailsToInstance: error executing neo4j update statement")) + So(len(mockPool.OpenPoolCalls()), ShouldEqual, 1) + So(len(conn.PrepareNeoCalls()), ShouldEqual, 1) + So(len(conn.CloseCalls()), ShouldEqual, 1) + So(len(stmt.ExecNeoCalls()), ShouldEqual, 1) + }) + }) + + Convey("given result.Metadata() stats are not as expected", t, func() { + res := &mocks.BoltResultMock{ + MetadataFunc: func() map[string]interface{} { + return map[string]interface{}{ + "stats": "invalid stats", + } + }, + } + stmt := &mocks.BoltStmtMock{ + ExecNeoFunc: func(params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) { + return res, nil + }, + CloseFunc: closeNoErr, + } + conn := &mocks.BoltConnMock{ + PrepareNeoFunc: func(query string) (golangNeo4jBoltDriver.Stmt, error) { + return stmt, nil + }, + CloseFunc: closeNoErr, + } + mockPool := &mocks.DBPoolMock{ + OpenPoolFunc: func() (golangNeo4jBoltDriver.Conn, error) { + return conn, nil + }, + } + + store := Neo4j{Pool: mockPool} + + err := store.AddVersionDetailsToInstance(context.Background(), testInstanceID, testDatasetId, testEdition, testVersion) + + Convey("then the expected error is returned", func() { + So(err.Error(), ShouldEqual, "neoClient AddVersionDetailsToInstance: error getting query result stats") + So(len(mockPool.OpenPoolCalls()), ShouldEqual, 1) + So(len(conn.PrepareNeoCalls()), ShouldEqual, 1) + So(len(conn.CloseCalls()), ShouldEqual, 1) + So(len(stmt.ExecNeoCalls()), ShouldEqual, 1) + So(len(stmt.CloseCalls()), ShouldEqual, 1) + So(len(res.MetadataCalls()), ShouldEqual, 1) + }) + }) + + Convey("given result stats do not contain 'properties-set'", t, func() { + res := &mocks.BoltResultMock{ + MetadataFunc: func() map[string]interface{} { + return map[string]interface{}{ + "stats": map[string]interface{}{}, + } + }, + } + stmt := &mocks.BoltStmtMock{ + ExecNeoFunc: func(params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) { + return res, nil + }, + CloseFunc: closeNoErr, + } + conn := &mocks.BoltConnMock{ + PrepareNeoFunc: func(query string) (golangNeo4jBoltDriver.Stmt, error) { + return stmt, nil + }, + CloseFunc: closeNoErr, + } + mockPool := &mocks.DBPoolMock{ + OpenPoolFunc: func() (golangNeo4jBoltDriver.Conn, error) { + return conn, nil + }, + } + + store := Neo4j{Pool: mockPool} + + err := store.AddVersionDetailsToInstance(context.Background(), testInstanceID, testDatasetId, testEdition, testVersion) + + Convey("then the expected error is returned", func() { + So(err.Error(), ShouldEqual, "neoClient AddVersionDetailsToInstance: error verifying query results") + So(len(mockPool.OpenPoolCalls()), ShouldEqual, 1) + So(len(conn.PrepareNeoCalls()), ShouldEqual, 1) + So(len(conn.CloseCalls()), ShouldEqual, 1) + So(len(stmt.ExecNeoCalls()), ShouldEqual, 1) + So(len(stmt.CloseCalls()), ShouldEqual, 1) + So(len(res.MetadataCalls()), ShouldEqual, 1) + }) + }) + + Convey("given result stats properties-set is not the expected format", t, func() { + res := &mocks.BoltResultMock{ + MetadataFunc: func() map[string]interface{} { + return map[string]interface{}{ + "stats": map[string]interface{}{ + "properties-set": "3", + }, + } + }, + } + stmt := &mocks.BoltStmtMock{ + ExecNeoFunc: func(params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) { + return res, nil + }, + CloseFunc: closeNoErr, + } + conn := &mocks.BoltConnMock{ + PrepareNeoFunc: func(query string) (golangNeo4jBoltDriver.Stmt, error) { + return stmt, nil + }, + CloseFunc: closeNoErr, + } + mockPool := &mocks.DBPoolMock{ + OpenPoolFunc: func() (golangNeo4jBoltDriver.Conn, error) { + return conn, nil + }, + } + + store := Neo4j{Pool: mockPool} + + err := store.AddVersionDetailsToInstance(context.Background(), testInstanceID, testDatasetId, testEdition, testVersion) + + Convey("then the expected error is returned", func() { + So(err.Error(), ShouldEqual, "neoClient AddVersionDetailsToInstance: error verifying query results") + So(len(mockPool.OpenPoolCalls()), ShouldEqual, 1) + So(len(conn.PrepareNeoCalls()), ShouldEqual, 1) + So(len(conn.CloseCalls()), ShouldEqual, 1) + So(len(stmt.ExecNeoCalls()), ShouldEqual, 1) + So(len(stmt.CloseCalls()), ShouldEqual, 1) + So(len(res.MetadataCalls()), ShouldEqual, 1) + }) + }) + + Convey("given result stats properties-set is not the expected value", t, func() { + res := &mocks.BoltResultMock{ + MetadataFunc: func() map[string]interface{} { + return map[string]interface{}{ + "stats": map[string]interface{}{ + "properties-set": int64(666), + }, + } + }, + } + stmt := &mocks.BoltStmtMock{ + ExecNeoFunc: func(params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) { + return res, nil + }, + CloseFunc: closeNoErr, + } + conn := &mocks.BoltConnMock{ + PrepareNeoFunc: func(query string) (golangNeo4jBoltDriver.Stmt, error) { + return stmt, nil + }, + CloseFunc: closeNoErr, + } + mockPool := &mocks.DBPoolMock{ + OpenPoolFunc: func() (golangNeo4jBoltDriver.Conn, error) { + return conn, nil + }, + } + + store := Neo4j{Pool: mockPool} + + err := store.AddVersionDetailsToInstance(context.Background(), testInstanceID, testDatasetId, testEdition, testVersion) + + Convey("then the expected error is returned", func() { + So(err.Error(), ShouldEqual, "neoClient AddVersionDetailsToInstance: unexpected rows affected expected 3 but was 666") + So(len(mockPool.OpenPoolCalls()), ShouldEqual, 1) + So(len(conn.PrepareNeoCalls()), ShouldEqual, 1) + So(len(conn.CloseCalls()), ShouldEqual, 1) + So(len(stmt.ExecNeoCalls()), ShouldEqual, 1) + So(len(stmt.CloseCalls()), ShouldEqual, 1) + So(len(res.MetadataCalls()), ShouldEqual, 1) + }) + }) +} diff --git a/neo4j/mocks/bolt.go b/neo4j/mocks/bolt.go new file mode 100644 index 00000000..71249ecf --- /dev/null +++ b/neo4j/mocks/bolt.go @@ -0,0 +1,886 @@ +// Code generated by moq; DO NOT EDIT +// github.com/matryer/moq + +package mocks + +import ( + "database/sql/driver" + "github.com/johnnadratowski/golang-neo4j-bolt-driver" + "sync" + "time" +) + +var ( + lockDBPoolMockClose sync.RWMutex + lockDBPoolMockOpenPool sync.RWMutex +) + +// DBPoolMock is a mock implementation of DBPool. +// +// func TestSomethingThatUsesDBPool(t *testing.T) { +// +// // make and configure a mocked DBPool +// mockedDBPool := &DBPoolMock{ +// CloseFunc: func() error { +// panic("TODO: mock out the Close method") +// }, +// OpenPoolFunc: func() (golangNeo4jBoltDriver.Conn, error) { +// panic("TODO: mock out the OpenPool method") +// }, +// } +// +// // TODO: use mockedDBPool in code that requires DBPool +// // and then make assertions. +// +// } +type DBPoolMock struct { + // CloseFunc mocks the Close method. + CloseFunc func() error + + // OpenPoolFunc mocks the OpenPool method. + OpenPoolFunc func() (golangNeo4jBoltDriver.Conn, error) + + // calls tracks calls to the methods. + calls struct { + // Close holds details about calls to the Close method. + Close []struct { + } + // OpenPool holds details about calls to the OpenPool method. + OpenPool []struct { + } + } +} + +// Close calls CloseFunc. +func (mock *DBPoolMock) Close() error { + if mock.CloseFunc == nil { + panic("moq: DBPoolMock.CloseFunc is nil but DBPool.Close was just called") + } + callInfo := struct { + }{} + lockDBPoolMockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + lockDBPoolMockClose.Unlock() + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// len(mockedDBPool.CloseCalls()) +func (mock *DBPoolMock) CloseCalls() []struct { +} { + var calls []struct { + } + lockDBPoolMockClose.RLock() + calls = mock.calls.Close + lockDBPoolMockClose.RUnlock() + return calls +} + +// OpenPool calls OpenPoolFunc. +func (mock *DBPoolMock) OpenPool() (golangNeo4jBoltDriver.Conn, error) { + if mock.OpenPoolFunc == nil { + panic("moq: DBPoolMock.OpenPoolFunc is nil but DBPool.OpenPool was just called") + } + callInfo := struct { + }{} + lockDBPoolMockOpenPool.Lock() + mock.calls.OpenPool = append(mock.calls.OpenPool, callInfo) + lockDBPoolMockOpenPool.Unlock() + return mock.OpenPoolFunc() +} + +// OpenPoolCalls gets all the calls that were made to OpenPool. +// Check the length with: +// len(mockedDBPool.OpenPoolCalls()) +func (mock *DBPoolMock) OpenPoolCalls() []struct { +} { + var calls []struct { + } + lockDBPoolMockOpenPool.RLock() + calls = mock.calls.OpenPool + lockDBPoolMockOpenPool.RUnlock() + return calls +} + +var ( + lockBoltConnMockBegin sync.RWMutex + lockBoltConnMockClose sync.RWMutex + lockBoltConnMockExecNeo sync.RWMutex + lockBoltConnMockExecPipeline sync.RWMutex + lockBoltConnMockPrepareNeo sync.RWMutex + lockBoltConnMockPreparePipeline sync.RWMutex + lockBoltConnMockQueryNeo sync.RWMutex + lockBoltConnMockQueryNeoAll sync.RWMutex + lockBoltConnMockQueryPipeline sync.RWMutex + lockBoltConnMockSetChunkSize sync.RWMutex + lockBoltConnMockSetTimeout sync.RWMutex +) + +// BoltConnMock is a mock implementation of BoltConn. +// +// func TestSomethingThatUsesBoltConn(t *testing.T) { +// +// // make and configure a mocked BoltConn +// mockedBoltConn := &BoltConnMock{ +// BeginFunc: func() (driver.Tx, error) { +// panic("TODO: mock out the Begin method") +// }, +// CloseFunc: func() error { +// panic("TODO: mock out the Close method") +// }, +// ExecNeoFunc: func(query string, params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) { +// panic("TODO: mock out the ExecNeo method") +// }, +// ExecPipelineFunc: func(query []string, params ...map[string]interface{}) ([]golangNeo4jBoltDriver.Result, error) { +// panic("TODO: mock out the ExecPipeline method") +// }, +// PrepareNeoFunc: func(query string) (golangNeo4jBoltDriver.Stmt, error) { +// panic("TODO: mock out the PrepareNeo method") +// }, +// PreparePipelineFunc: func(query ...string) (golangNeo4jBoltDriver.PipelineStmt, error) { +// panic("TODO: mock out the PreparePipeline method") +// }, +// QueryNeoFunc: func(query string, params map[string]interface{}) (golangNeo4jBoltDriver.Rows, error) { +// panic("TODO: mock out the QueryNeo method") +// }, +// QueryNeoAllFunc: func(query string, params map[string]interface{}) ([][]interface{}, map[string]interface{}, map[string]interface{}, error) { +// panic("TODO: mock out the QueryNeoAll method") +// }, +// QueryPipelineFunc: func(query []string, params ...map[string]interface{}) (golangNeo4jBoltDriver.PipelineRows, error) { +// panic("TODO: mock out the QueryPipeline method") +// }, +// SetChunkSizeFunc: func(in1 uint16) { +// panic("TODO: mock out the SetChunkSize method") +// }, +// SetTimeoutFunc: func(in1 time.Duration) { +// panic("TODO: mock out the SetTimeout method") +// }, +// } +// +// // TODO: use mockedBoltConn in code that requires BoltConn +// // and then make assertions. +// +// } +type BoltConnMock struct { + // BeginFunc mocks the Begin method. + BeginFunc func() (driver.Tx, error) + + // CloseFunc mocks the Close method. + CloseFunc func() error + + // ExecNeoFunc mocks the ExecNeo method. + ExecNeoFunc func(query string, params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) + + // ExecPipelineFunc mocks the ExecPipeline method. + ExecPipelineFunc func(query []string, params ...map[string]interface{}) ([]golangNeo4jBoltDriver.Result, error) + + // PrepareNeoFunc mocks the PrepareNeo method. + PrepareNeoFunc func(query string) (golangNeo4jBoltDriver.Stmt, error) + + // PreparePipelineFunc mocks the PreparePipeline method. + PreparePipelineFunc func(query ...string) (golangNeo4jBoltDriver.PipelineStmt, error) + + // QueryNeoFunc mocks the QueryNeo method. + QueryNeoFunc func(query string, params map[string]interface{}) (golangNeo4jBoltDriver.Rows, error) + + // QueryNeoAllFunc mocks the QueryNeoAll method. + QueryNeoAllFunc func(query string, params map[string]interface{}) ([][]interface{}, map[string]interface{}, map[string]interface{}, error) + + // QueryPipelineFunc mocks the QueryPipeline method. + QueryPipelineFunc func(query []string, params ...map[string]interface{}) (golangNeo4jBoltDriver.PipelineRows, error) + + // SetChunkSizeFunc mocks the SetChunkSize method. + SetChunkSizeFunc func(in1 uint16) + + // SetTimeoutFunc mocks the SetTimeout method. + SetTimeoutFunc func(in1 time.Duration) + + // calls tracks calls to the methods. + calls struct { + // Begin holds details about calls to the Begin method. + Begin []struct { + } + // Close holds details about calls to the Close method. + Close []struct { + } + // ExecNeo holds details about calls to the ExecNeo method. + ExecNeo []struct { + // Query is the query argument value. + Query string + // Params is the params argument value. + Params map[string]interface{} + } + // ExecPipeline holds details about calls to the ExecPipeline method. + ExecPipeline []struct { + // Query is the query argument value. + Query []string + // Params is the params argument value. + Params []map[string]interface{} + } + // PrepareNeo holds details about calls to the PrepareNeo method. + PrepareNeo []struct { + // Query is the query argument value. + Query string + } + // PreparePipeline holds details about calls to the PreparePipeline method. + PreparePipeline []struct { + // Query is the query argument value. + Query []string + } + // QueryNeo holds details about calls to the QueryNeo method. + QueryNeo []struct { + // Query is the query argument value. + Query string + // Params is the params argument value. + Params map[string]interface{} + } + // QueryNeoAll holds details about calls to the QueryNeoAll method. + QueryNeoAll []struct { + // Query is the query argument value. + Query string + // Params is the params argument value. + Params map[string]interface{} + } + // QueryPipeline holds details about calls to the QueryPipeline method. + QueryPipeline []struct { + // Query is the query argument value. + Query []string + // Params is the params argument value. + Params []map[string]interface{} + } + // SetChunkSize holds details about calls to the SetChunkSize method. + SetChunkSize []struct { + // In1 is the in1 argument value. + In1 uint16 + } + // SetTimeout holds details about calls to the SetTimeout method. + SetTimeout []struct { + // In1 is the in1 argument value. + In1 time.Duration + } + } +} + +// Begin calls BeginFunc. +func (mock *BoltConnMock) Begin() (driver.Tx, error) { + if mock.BeginFunc == nil { + panic("moq: BoltConnMock.BeginFunc is nil but BoltConn.Begin was just called") + } + callInfo := struct { + }{} + lockBoltConnMockBegin.Lock() + mock.calls.Begin = append(mock.calls.Begin, callInfo) + lockBoltConnMockBegin.Unlock() + return mock.BeginFunc() +} + +// BeginCalls gets all the calls that were made to Begin. +// Check the length with: +// len(mockedBoltConn.BeginCalls()) +func (mock *BoltConnMock) BeginCalls() []struct { +} { + var calls []struct { + } + lockBoltConnMockBegin.RLock() + calls = mock.calls.Begin + lockBoltConnMockBegin.RUnlock() + return calls +} + +// Close calls CloseFunc. +func (mock *BoltConnMock) Close() error { + if mock.CloseFunc == nil { + panic("moq: BoltConnMock.CloseFunc is nil but BoltConn.Close was just called") + } + callInfo := struct { + }{} + lockBoltConnMockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + lockBoltConnMockClose.Unlock() + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// len(mockedBoltConn.CloseCalls()) +func (mock *BoltConnMock) CloseCalls() []struct { +} { + var calls []struct { + } + lockBoltConnMockClose.RLock() + calls = mock.calls.Close + lockBoltConnMockClose.RUnlock() + return calls +} + +// ExecNeo calls ExecNeoFunc. +func (mock *BoltConnMock) ExecNeo(query string, params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) { + if mock.ExecNeoFunc == nil { + panic("moq: BoltConnMock.ExecNeoFunc is nil but BoltConn.ExecNeo was just called") + } + callInfo := struct { + Query string + Params map[string]interface{} + }{ + Query: query, + Params: params, + } + lockBoltConnMockExecNeo.Lock() + mock.calls.ExecNeo = append(mock.calls.ExecNeo, callInfo) + lockBoltConnMockExecNeo.Unlock() + return mock.ExecNeoFunc(query, params) +} + +// ExecNeoCalls gets all the calls that were made to ExecNeo. +// Check the length with: +// len(mockedBoltConn.ExecNeoCalls()) +func (mock *BoltConnMock) ExecNeoCalls() []struct { + Query string + Params map[string]interface{} +} { + var calls []struct { + Query string + Params map[string]interface{} + } + lockBoltConnMockExecNeo.RLock() + calls = mock.calls.ExecNeo + lockBoltConnMockExecNeo.RUnlock() + return calls +} + +// ExecPipeline calls ExecPipelineFunc. +func (mock *BoltConnMock) ExecPipeline(query []string, params ...map[string]interface{}) ([]golangNeo4jBoltDriver.Result, error) { + if mock.ExecPipelineFunc == nil { + panic("moq: BoltConnMock.ExecPipelineFunc is nil but BoltConn.ExecPipeline was just called") + } + callInfo := struct { + Query []string + Params []map[string]interface{} + }{ + Query: query, + Params: params, + } + lockBoltConnMockExecPipeline.Lock() + mock.calls.ExecPipeline = append(mock.calls.ExecPipeline, callInfo) + lockBoltConnMockExecPipeline.Unlock() + return mock.ExecPipelineFunc(query, params...) +} + +// ExecPipelineCalls gets all the calls that were made to ExecPipeline. +// Check the length with: +// len(mockedBoltConn.ExecPipelineCalls()) +func (mock *BoltConnMock) ExecPipelineCalls() []struct { + Query []string + Params []map[string]interface{} +} { + var calls []struct { + Query []string + Params []map[string]interface{} + } + lockBoltConnMockExecPipeline.RLock() + calls = mock.calls.ExecPipeline + lockBoltConnMockExecPipeline.RUnlock() + return calls +} + +// PrepareNeo calls PrepareNeoFunc. +func (mock *BoltConnMock) PrepareNeo(query string) (golangNeo4jBoltDriver.Stmt, error) { + if mock.PrepareNeoFunc == nil { + panic("moq: BoltConnMock.PrepareNeoFunc is nil but BoltConn.PrepareNeo was just called") + } + callInfo := struct { + Query string + }{ + Query: query, + } + lockBoltConnMockPrepareNeo.Lock() + mock.calls.PrepareNeo = append(mock.calls.PrepareNeo, callInfo) + lockBoltConnMockPrepareNeo.Unlock() + return mock.PrepareNeoFunc(query) +} + +// PrepareNeoCalls gets all the calls that were made to PrepareNeo. +// Check the length with: +// len(mockedBoltConn.PrepareNeoCalls()) +func (mock *BoltConnMock) PrepareNeoCalls() []struct { + Query string +} { + var calls []struct { + Query string + } + lockBoltConnMockPrepareNeo.RLock() + calls = mock.calls.PrepareNeo + lockBoltConnMockPrepareNeo.RUnlock() + return calls +} + +// PreparePipeline calls PreparePipelineFunc. +func (mock *BoltConnMock) PreparePipeline(query ...string) (golangNeo4jBoltDriver.PipelineStmt, error) { + if mock.PreparePipelineFunc == nil { + panic("moq: BoltConnMock.PreparePipelineFunc is nil but BoltConn.PreparePipeline was just called") + } + callInfo := struct { + Query []string + }{ + Query: query, + } + lockBoltConnMockPreparePipeline.Lock() + mock.calls.PreparePipeline = append(mock.calls.PreparePipeline, callInfo) + lockBoltConnMockPreparePipeline.Unlock() + return mock.PreparePipelineFunc(query...) +} + +// PreparePipelineCalls gets all the calls that were made to PreparePipeline. +// Check the length with: +// len(mockedBoltConn.PreparePipelineCalls()) +func (mock *BoltConnMock) PreparePipelineCalls() []struct { + Query []string +} { + var calls []struct { + Query []string + } + lockBoltConnMockPreparePipeline.RLock() + calls = mock.calls.PreparePipeline + lockBoltConnMockPreparePipeline.RUnlock() + return calls +} + +// QueryNeo calls QueryNeoFunc. +func (mock *BoltConnMock) QueryNeo(query string, params map[string]interface{}) (golangNeo4jBoltDriver.Rows, error) { + if mock.QueryNeoFunc == nil { + panic("moq: BoltConnMock.QueryNeoFunc is nil but BoltConn.QueryNeo was just called") + } + callInfo := struct { + Query string + Params map[string]interface{} + }{ + Query: query, + Params: params, + } + lockBoltConnMockQueryNeo.Lock() + mock.calls.QueryNeo = append(mock.calls.QueryNeo, callInfo) + lockBoltConnMockQueryNeo.Unlock() + return mock.QueryNeoFunc(query, params) +} + +// QueryNeoCalls gets all the calls that were made to QueryNeo. +// Check the length with: +// len(mockedBoltConn.QueryNeoCalls()) +func (mock *BoltConnMock) QueryNeoCalls() []struct { + Query string + Params map[string]interface{} +} { + var calls []struct { + Query string + Params map[string]interface{} + } + lockBoltConnMockQueryNeo.RLock() + calls = mock.calls.QueryNeo + lockBoltConnMockQueryNeo.RUnlock() + return calls +} + +// QueryNeoAll calls QueryNeoAllFunc. +func (mock *BoltConnMock) QueryNeoAll(query string, params map[string]interface{}) ([][]interface{}, map[string]interface{}, map[string]interface{}, error) { + if mock.QueryNeoAllFunc == nil { + panic("moq: BoltConnMock.QueryNeoAllFunc is nil but BoltConn.QueryNeoAll was just called") + } + callInfo := struct { + Query string + Params map[string]interface{} + }{ + Query: query, + Params: params, + } + lockBoltConnMockQueryNeoAll.Lock() + mock.calls.QueryNeoAll = append(mock.calls.QueryNeoAll, callInfo) + lockBoltConnMockQueryNeoAll.Unlock() + return mock.QueryNeoAllFunc(query, params) +} + +// QueryNeoAllCalls gets all the calls that were made to QueryNeoAll. +// Check the length with: +// len(mockedBoltConn.QueryNeoAllCalls()) +func (mock *BoltConnMock) QueryNeoAllCalls() []struct { + Query string + Params map[string]interface{} +} { + var calls []struct { + Query string + Params map[string]interface{} + } + lockBoltConnMockQueryNeoAll.RLock() + calls = mock.calls.QueryNeoAll + lockBoltConnMockQueryNeoAll.RUnlock() + return calls +} + +// QueryPipeline calls QueryPipelineFunc. +func (mock *BoltConnMock) QueryPipeline(query []string, params ...map[string]interface{}) (golangNeo4jBoltDriver.PipelineRows, error) { + if mock.QueryPipelineFunc == nil { + panic("moq: BoltConnMock.QueryPipelineFunc is nil but BoltConn.QueryPipeline was just called") + } + callInfo := struct { + Query []string + Params []map[string]interface{} + }{ + Query: query, + Params: params, + } + lockBoltConnMockQueryPipeline.Lock() + mock.calls.QueryPipeline = append(mock.calls.QueryPipeline, callInfo) + lockBoltConnMockQueryPipeline.Unlock() + return mock.QueryPipelineFunc(query, params...) +} + +// QueryPipelineCalls gets all the calls that were made to QueryPipeline. +// Check the length with: +// len(mockedBoltConn.QueryPipelineCalls()) +func (mock *BoltConnMock) QueryPipelineCalls() []struct { + Query []string + Params []map[string]interface{} +} { + var calls []struct { + Query []string + Params []map[string]interface{} + } + lockBoltConnMockQueryPipeline.RLock() + calls = mock.calls.QueryPipeline + lockBoltConnMockQueryPipeline.RUnlock() + return calls +} + +// SetChunkSize calls SetChunkSizeFunc. +func (mock *BoltConnMock) SetChunkSize(in1 uint16) { + if mock.SetChunkSizeFunc == nil { + panic("moq: BoltConnMock.SetChunkSizeFunc is nil but BoltConn.SetChunkSize was just called") + } + callInfo := struct { + In1 uint16 + }{ + In1: in1, + } + lockBoltConnMockSetChunkSize.Lock() + mock.calls.SetChunkSize = append(mock.calls.SetChunkSize, callInfo) + lockBoltConnMockSetChunkSize.Unlock() + mock.SetChunkSizeFunc(in1) +} + +// SetChunkSizeCalls gets all the calls that were made to SetChunkSize. +// Check the length with: +// len(mockedBoltConn.SetChunkSizeCalls()) +func (mock *BoltConnMock) SetChunkSizeCalls() []struct { + In1 uint16 +} { + var calls []struct { + In1 uint16 + } + lockBoltConnMockSetChunkSize.RLock() + calls = mock.calls.SetChunkSize + lockBoltConnMockSetChunkSize.RUnlock() + return calls +} + +// SetTimeout calls SetTimeoutFunc. +func (mock *BoltConnMock) SetTimeout(in1 time.Duration) { + if mock.SetTimeoutFunc == nil { + panic("moq: BoltConnMock.SetTimeoutFunc is nil but BoltConn.SetTimeout was just called") + } + callInfo := struct { + In1 time.Duration + }{ + In1: in1, + } + lockBoltConnMockSetTimeout.Lock() + mock.calls.SetTimeout = append(mock.calls.SetTimeout, callInfo) + lockBoltConnMockSetTimeout.Unlock() + mock.SetTimeoutFunc(in1) +} + +// SetTimeoutCalls gets all the calls that were made to SetTimeout. +// Check the length with: +// len(mockedBoltConn.SetTimeoutCalls()) +func (mock *BoltConnMock) SetTimeoutCalls() []struct { + In1 time.Duration +} { + var calls []struct { + In1 time.Duration + } + lockBoltConnMockSetTimeout.RLock() + calls = mock.calls.SetTimeout + lockBoltConnMockSetTimeout.RUnlock() + return calls +} + +var ( + lockBoltStmtMockClose sync.RWMutex + lockBoltStmtMockExecNeo sync.RWMutex + lockBoltStmtMockQueryNeo sync.RWMutex +) + +// BoltStmtMock is a mock implementation of BoltStmt. +// +// func TestSomethingThatUsesBoltStmt(t *testing.T) { +// +// // make and configure a mocked BoltStmt +// mockedBoltStmt := &BoltStmtMock{ +// CloseFunc: func() error { +// panic("TODO: mock out the Close method") +// }, +// ExecNeoFunc: func(params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) { +// panic("TODO: mock out the ExecNeo method") +// }, +// QueryNeoFunc: func(params map[string]interface{}) (golangNeo4jBoltDriver.Rows, error) { +// panic("TODO: mock out the QueryNeo method") +// }, +// } +// +// // TODO: use mockedBoltStmt in code that requires BoltStmt +// // and then make assertions. +// +// } +type BoltStmtMock struct { + // CloseFunc mocks the Close method. + CloseFunc func() error + + // ExecNeoFunc mocks the ExecNeo method. + ExecNeoFunc func(params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) + + // QueryNeoFunc mocks the QueryNeo method. + QueryNeoFunc func(params map[string]interface{}) (golangNeo4jBoltDriver.Rows, error) + + // calls tracks calls to the methods. + calls struct { + // Close holds details about calls to the Close method. + Close []struct { + } + // ExecNeo holds details about calls to the ExecNeo method. + ExecNeo []struct { + // Params is the params argument value. + Params map[string]interface{} + } + // QueryNeo holds details about calls to the QueryNeo method. + QueryNeo []struct { + // Params is the params argument value. + Params map[string]interface{} + } + } +} + +// Close calls CloseFunc. +func (mock *BoltStmtMock) Close() error { + if mock.CloseFunc == nil { + panic("moq: BoltStmtMock.CloseFunc is nil but BoltStmt.Close was just called") + } + callInfo := struct { + }{} + lockBoltStmtMockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + lockBoltStmtMockClose.Unlock() + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// len(mockedBoltStmt.CloseCalls()) +func (mock *BoltStmtMock) CloseCalls() []struct { +} { + var calls []struct { + } + lockBoltStmtMockClose.RLock() + calls = mock.calls.Close + lockBoltStmtMockClose.RUnlock() + return calls +} + +// ExecNeo calls ExecNeoFunc. +func (mock *BoltStmtMock) ExecNeo(params map[string]interface{}) (golangNeo4jBoltDriver.Result, error) { + if mock.ExecNeoFunc == nil { + panic("moq: BoltStmtMock.ExecNeoFunc is nil but BoltStmt.ExecNeo was just called") + } + callInfo := struct { + Params map[string]interface{} + }{ + Params: params, + } + lockBoltStmtMockExecNeo.Lock() + mock.calls.ExecNeo = append(mock.calls.ExecNeo, callInfo) + lockBoltStmtMockExecNeo.Unlock() + return mock.ExecNeoFunc(params) +} + +// ExecNeoCalls gets all the calls that were made to ExecNeo. +// Check the length with: +// len(mockedBoltStmt.ExecNeoCalls()) +func (mock *BoltStmtMock) ExecNeoCalls() []struct { + Params map[string]interface{} +} { + var calls []struct { + Params map[string]interface{} + } + lockBoltStmtMockExecNeo.RLock() + calls = mock.calls.ExecNeo + lockBoltStmtMockExecNeo.RUnlock() + return calls +} + +// QueryNeo calls QueryNeoFunc. +func (mock *BoltStmtMock) QueryNeo(params map[string]interface{}) (golangNeo4jBoltDriver.Rows, error) { + if mock.QueryNeoFunc == nil { + panic("moq: BoltStmtMock.QueryNeoFunc is nil but BoltStmt.QueryNeo was just called") + } + callInfo := struct { + Params map[string]interface{} + }{ + Params: params, + } + lockBoltStmtMockQueryNeo.Lock() + mock.calls.QueryNeo = append(mock.calls.QueryNeo, callInfo) + lockBoltStmtMockQueryNeo.Unlock() + return mock.QueryNeoFunc(params) +} + +// QueryNeoCalls gets all the calls that were made to QueryNeo. +// Check the length with: +// len(mockedBoltStmt.QueryNeoCalls()) +func (mock *BoltStmtMock) QueryNeoCalls() []struct { + Params map[string]interface{} +} { + var calls []struct { + Params map[string]interface{} + } + lockBoltStmtMockQueryNeo.RLock() + calls = mock.calls.QueryNeo + lockBoltStmtMockQueryNeo.RUnlock() + return calls +} + +var ( + lockBoltResultMockLastInsertId sync.RWMutex + lockBoltResultMockMetadata sync.RWMutex + lockBoltResultMockRowsAffected sync.RWMutex +) + +// BoltResultMock is a mock implementation of BoltResult. +// +// func TestSomethingThatUsesBoltResult(t *testing.T) { +// +// // make and configure a mocked BoltResult +// mockedBoltResult := &BoltResultMock{ +// LastInsertIdFunc: func() (int64, error) { +// panic("TODO: mock out the LastInsertId method") +// }, +// MetadataFunc: func() map[string]interface{} { +// panic("TODO: mock out the Metadata method") +// }, +// RowsAffectedFunc: func() (int64, error) { +// panic("TODO: mock out the RowsAffected method") +// }, +// } +// +// // TODO: use mockedBoltResult in code that requires BoltResult +// // and then make assertions. +// +// } +type BoltResultMock struct { + // LastInsertIdFunc mocks the LastInsertId method. + LastInsertIdFunc func() (int64, error) + + // MetadataFunc mocks the Metadata method. + MetadataFunc func() map[string]interface{} + + // RowsAffectedFunc mocks the RowsAffected method. + RowsAffectedFunc func() (int64, error) + + // calls tracks calls to the methods. + calls struct { + // LastInsertId holds details about calls to the LastInsertId method. + LastInsertId []struct { + } + // Metadata holds details about calls to the Metadata method. + Metadata []struct { + } + // RowsAffected holds details about calls to the RowsAffected method. + RowsAffected []struct { + } + } +} + +// LastInsertId calls LastInsertIdFunc. +func (mock *BoltResultMock) LastInsertId() (int64, error) { + if mock.LastInsertIdFunc == nil { + panic("moq: BoltResultMock.LastInsertIdFunc is nil but BoltResult.LastInsertId was just called") + } + callInfo := struct { + }{} + lockBoltResultMockLastInsertId.Lock() + mock.calls.LastInsertId = append(mock.calls.LastInsertId, callInfo) + lockBoltResultMockLastInsertId.Unlock() + return mock.LastInsertIdFunc() +} + +// LastInsertIdCalls gets all the calls that were made to LastInsertId. +// Check the length with: +// len(mockedBoltResult.LastInsertIdCalls()) +func (mock *BoltResultMock) LastInsertIdCalls() []struct { +} { + var calls []struct { + } + lockBoltResultMockLastInsertId.RLock() + calls = mock.calls.LastInsertId + lockBoltResultMockLastInsertId.RUnlock() + return calls +} + +// Metadata calls MetadataFunc. +func (mock *BoltResultMock) Metadata() map[string]interface{} { + if mock.MetadataFunc == nil { + panic("moq: BoltResultMock.MetadataFunc is nil but BoltResult.Metadata was just called") + } + callInfo := struct { + }{} + lockBoltResultMockMetadata.Lock() + mock.calls.Metadata = append(mock.calls.Metadata, callInfo) + lockBoltResultMockMetadata.Unlock() + return mock.MetadataFunc() +} + +// MetadataCalls gets all the calls that were made to Metadata. +// Check the length with: +// len(mockedBoltResult.MetadataCalls()) +func (mock *BoltResultMock) MetadataCalls() []struct { +} { + var calls []struct { + } + lockBoltResultMockMetadata.RLock() + calls = mock.calls.Metadata + lockBoltResultMockMetadata.RUnlock() + return calls +} + +// RowsAffected calls RowsAffectedFunc. +func (mock *BoltResultMock) RowsAffected() (int64, error) { + if mock.RowsAffectedFunc == nil { + panic("moq: BoltResultMock.RowsAffectedFunc is nil but BoltResult.RowsAffected was just called") + } + callInfo := struct { + }{} + lockBoltResultMockRowsAffected.Lock() + mock.calls.RowsAffected = append(mock.calls.RowsAffected, callInfo) + lockBoltResultMockRowsAffected.Unlock() + return mock.RowsAffectedFunc() +} + +// RowsAffectedCalls gets all the calls that were made to RowsAffected. +// Check the length with: +// len(mockedBoltResult.RowsAffectedCalls()) +func (mock *BoltResultMock) RowsAffectedCalls() []struct { +} { + var calls []struct { + } + lockBoltResultMockRowsAffected.RLock() + calls = mock.calls.RowsAffected + lockBoltResultMockRowsAffected.RUnlock() + return calls +} diff --git a/store/datastore.go b/store/datastore.go index d4cd3fd0..1c07ff8a 100644 --- a/store/datastore.go +++ b/store/datastore.go @@ -2,10 +2,9 @@ package store import ( "context" - "time" "github.com/ONSdigital/dp-dataset-api/models" - "gopkg.in/mgo.v2/bson" + "github.com/gedge/mgo/bson" ) // DataStore provides a datastore.Storer interface used to store, retrieve, remove or update datasets @@ -24,7 +23,7 @@ type Storer interface { CheckEditionExists(ID, editionID, state string) error GetDataset(ID string) (*models.DatasetUpdate, error) GetDatasets() ([]models.DatasetUpdate, error) - GetDimensionNodesFromInstance(ID string) (*models.DimensionNodeResults, error) + GetDimensionsFromInstance(ID string) (*models.DimensionNodeResults, error) GetDimensions(datasetID, versionID string) ([]bson.M, error) GetDimensionOptions(version *models.Version, dimension string) (*models.DimensionOptionResults, error) GetEdition(ID, editionID, state string) (*models.EditionUpdate, error) @@ -32,7 +31,7 @@ type Storer interface { GetInstances(filters []string) (*models.InstanceResults, error) GetInstance(ID string) (*models.Instance, error) GetNextVersion(datasetID, editionID string) (int, error) - GetUniqueDimensionValues(ID, dimension string) (*models.DimensionValues, error) + GetUniqueDimensionAndOptions(ID, dimension string) (*models.DimensionValues, error) GetVersion(datasetID, editionID, version, state string) (*models.Version, error) GetVersions(datasetID, editionID, state string) (*models.VersionResults, error) UpdateDataset(ID string, dataset *models.Dataset, currentState string) error @@ -49,6 +48,6 @@ type Storer interface { UpsertDataset(ID string, datasetDoc *models.DatasetUpdate) error UpsertEdition(datasetID, edition string, editionDoc *models.EditionUpdate) error UpsertVersion(ID string, versionDoc *models.Version) error - Ping(ctx context.Context) (time.Time, error) DeleteDataset(ID string) error + AddVersionDetailsToInstance(ctx context.Context, instanceID string, datasetID string, edition string, version int) error } diff --git a/store/datastoretest/datastore.go b/store/datastoretest/datastore.go index b1cb1616..81f15f67 100755 --- a/store/datastoretest/datastore.go +++ b/store/datastoretest/datastore.go @@ -5,34 +5,34 @@ package storetest import ( "context" + "sync" - "time" "github.com/ONSdigital/dp-dataset-api/models" - "gopkg.in/mgo.v2/bson" + "github.com/gedge/mgo/bson" ) var ( lockStorerMockAddDimensionToInstance sync.RWMutex lockStorerMockAddEventToInstance sync.RWMutex lockStorerMockAddInstance sync.RWMutex + lockStorerMockAddVersionDetailsToInstance sync.RWMutex lockStorerMockCheckDatasetExists sync.RWMutex lockStorerMockCheckEditionExists sync.RWMutex lockStorerMockDeleteDataset sync.RWMutex lockStorerMockGetDataset sync.RWMutex lockStorerMockGetDatasets sync.RWMutex - lockStorerMockGetDimensionNodesFromInstance sync.RWMutex lockStorerMockGetDimensionOptions sync.RWMutex lockStorerMockGetDimensions sync.RWMutex + lockStorerMockGetDimensionsFromInstance sync.RWMutex lockStorerMockGetEdition sync.RWMutex lockStorerMockGetEditions sync.RWMutex lockStorerMockGetInstance sync.RWMutex lockStorerMockGetInstances sync.RWMutex lockStorerMockGetNextVersion sync.RWMutex - lockStorerMockGetUniqueDimensionValues sync.RWMutex + lockStorerMockGetUniqueDimensionAndOptions sync.RWMutex lockStorerMockGetVersion sync.RWMutex lockStorerMockGetVersions sync.RWMutex - lockStorerMockPing sync.RWMutex lockStorerMockUpdateBuildHierarchyTaskState sync.RWMutex lockStorerMockUpdateBuildSearchTaskState sync.RWMutex lockStorerMockUpdateDataset sync.RWMutex @@ -64,6 +64,9 @@ var ( // AddInstanceFunc: func(instance *models.Instance) (*models.Instance, error) { // panic("TODO: mock out the AddInstance method") // }, +// AddVersionDetailsToInstanceFunc: func(ctx context.Context, instanceID string, datasetID string, edition string, version int) error { +// panic("TODO: mock out the AddVersionDetailsToInstance method") +// }, // CheckDatasetExistsFunc: func(ID string, state string) error { // panic("TODO: mock out the CheckDatasetExists method") // }, @@ -79,19 +82,19 @@ var ( // GetDatasetsFunc: func() ([]models.DatasetUpdate, error) { // panic("TODO: mock out the GetDatasets method") // }, -// GetDimensionNodesFromInstanceFunc: func(ID string) (*models.DimensionNodeResults, error) { -// panic("TODO: mock out the GetDimensionNodesFromInstance method") -// }, // GetDimensionOptionsFunc: func(version *models.Version, dimension string) (*models.DimensionOptionResults, error) { // panic("TODO: mock out the GetDimensionOptions method") // }, // GetDimensionsFunc: func(datasetID string, versionID string) ([]bson.M, error) { // panic("TODO: mock out the GetDimensions method") // }, -// GetEditionFunc: func(ID string, editionID string, state string) (*models.Edition, error) { +// GetDimensionsFromInstanceFunc: func(ID string) (*models.DimensionNodeResults, error) { +// panic("TODO: mock out the GetDimensionsFromInstance method") +// }, +// GetEditionFunc: func(ID string, editionID string, state string) (*models.EditionUpdate, error) { // panic("TODO: mock out the GetEdition method") // }, -// GetEditionsFunc: func(ID string, state string) (*models.EditionResults, error) { +// GetEditionsFunc: func(ID string, state string) (*models.EditionUpdateResults, error) { // panic("TODO: mock out the GetEditions method") // }, // GetInstanceFunc: func(ID string) (*models.Instance, error) { @@ -103,8 +106,8 @@ var ( // GetNextVersionFunc: func(datasetID string, editionID string) (int, error) { // panic("TODO: mock out the GetNextVersion method") // }, -// GetUniqueDimensionValuesFunc: func(ID string, dimension string) (*models.DimensionValues, error) { -// panic("TODO: mock out the GetUniqueDimensionValues method") +// GetUniqueDimensionAndOptionsFunc: func(ID string, dimension string) (*models.DimensionValues, error) { +// panic("TODO: mock out the GetUniqueDimensionAndOptions method") // }, // GetVersionFunc: func(datasetID string, editionID string, version string, state string) (*models.Version, error) { // panic("TODO: mock out the GetVersion method") @@ -112,9 +115,6 @@ var ( // GetVersionsFunc: func(datasetID string, editionID string, state string) (*models.VersionResults, error) { // panic("TODO: mock out the GetVersions method") // }, -// PingFunc: func(ctx context.Context) (time.Time, error) { -// panic("TODO: mock out the Ping method") -// }, // UpdateBuildHierarchyTaskStateFunc: func(id string, dimension string, state string) error { // panic("TODO: mock out the UpdateBuildHierarchyTaskState method") // }, @@ -151,7 +151,7 @@ var ( // UpsertDatasetFunc: func(ID string, datasetDoc *models.DatasetUpdate) error { // panic("TODO: mock out the UpsertDataset method") // }, -// UpsertEditionFunc: func(datasetID string, edition string, editionDoc *models.Edition) error { +// UpsertEditionFunc: func(datasetID string, edition string, editionDoc *models.EditionUpdate) error { // panic("TODO: mock out the UpsertEdition method") // }, // UpsertVersionFunc: func(ID string, versionDoc *models.Version) error { @@ -173,6 +173,9 @@ type StorerMock struct { // AddInstanceFunc mocks the AddInstance method. AddInstanceFunc func(instance *models.Instance) (*models.Instance, error) + // AddVersionDetailsToInstanceFunc mocks the AddVersionDetailsToInstance method. + AddVersionDetailsToInstanceFunc func(ctx context.Context, instanceID string, datasetID string, edition string, version int) error + // CheckDatasetExistsFunc mocks the CheckDatasetExists method. CheckDatasetExistsFunc func(ID string, state string) error @@ -188,20 +191,20 @@ type StorerMock struct { // GetDatasetsFunc mocks the GetDatasets method. GetDatasetsFunc func() ([]models.DatasetUpdate, error) - // GetDimensionNodesFromInstanceFunc mocks the GetDimensionNodesFromInstance method. - GetDimensionNodesFromInstanceFunc func(ID string) (*models.DimensionNodeResults, error) - // GetDimensionOptionsFunc mocks the GetDimensionOptions method. GetDimensionOptionsFunc func(version *models.Version, dimension string) (*models.DimensionOptionResults, error) // GetDimensionsFunc mocks the GetDimensions method. GetDimensionsFunc func(datasetID string, versionID string) ([]bson.M, error) + // GetDimensionsFromInstanceFunc mocks the GetDimensionsFromInstance method. + GetDimensionsFromInstanceFunc func(ID string) (*models.DimensionNodeResults, error) + // GetEditionFunc mocks the GetEdition method. - GetEditionFunc func(ID, editionID, state string) (*models.EditionUpdate, error) + GetEditionFunc func(ID string, editionID string, state string) (*models.EditionUpdate, error) // GetEditionsFunc mocks the GetEditions method. - GetEditionsFunc func(id, state string) (*models.EditionUpdateResults, error) + GetEditionsFunc func(ID string, state string) (*models.EditionUpdateResults, error) // GetInstanceFunc mocks the GetInstance method. GetInstanceFunc func(ID string) (*models.Instance, error) @@ -212,8 +215,8 @@ type StorerMock struct { // GetNextVersionFunc mocks the GetNextVersion method. GetNextVersionFunc func(datasetID string, editionID string) (int, error) - // GetUniqueDimensionValuesFunc mocks the GetUniqueDimensionValues method. - GetUniqueDimensionValuesFunc func(ID string, dimension string) (*models.DimensionValues, error) + // GetUniqueDimensionAndOptionsFunc mocks the GetUniqueDimensionAndOptions method. + GetUniqueDimensionAndOptionsFunc func(ID string, dimension string) (*models.DimensionValues, error) // GetVersionFunc mocks the GetVersion method. GetVersionFunc func(datasetID string, editionID string, version string, state string) (*models.Version, error) @@ -221,9 +224,6 @@ type StorerMock struct { // GetVersionsFunc mocks the GetVersions method. GetVersionsFunc func(datasetID string, editionID string, state string) (*models.VersionResults, error) - // PingFunc mocks the Ping method. - PingFunc func(ctx context.Context) (time.Time, error) - // UpdateBuildHierarchyTaskStateFunc mocks the UpdateBuildHierarchyTaskState method. UpdateBuildHierarchyTaskStateFunc func(id string, dimension string, state string) error @@ -285,6 +285,19 @@ type StorerMock struct { // Instance is the instance argument value. Instance *models.Instance } + // AddVersionDetailsToInstance holds details about calls to the AddVersionDetailsToInstance method. + AddVersionDetailsToInstance []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // InstanceID is the instanceID argument value. + InstanceID string + // DatasetID is the datasetID argument value. + DatasetID string + // Edition is the edition argument value. + Edition string + // Version is the version argument value. + Version int + } // CheckDatasetExists holds details about calls to the CheckDatasetExists method. CheckDatasetExists []struct { // ID is the ID argument value. @@ -314,11 +327,6 @@ type StorerMock struct { // GetDatasets holds details about calls to the GetDatasets method. GetDatasets []struct { } - // GetDimensionNodesFromInstance holds details about calls to the GetDimensionNodesFromInstance method. - GetDimensionNodesFromInstance []struct { - // ID is the ID argument value. - ID string - } // GetDimensionOptions holds details about calls to the GetDimensionOptions method. GetDimensionOptions []struct { // Version is the version argument value. @@ -333,20 +341,25 @@ type StorerMock struct { // VersionID is the versionID argument value. VersionID string } + // GetDimensionsFromInstance holds details about calls to the GetDimensionsFromInstance method. + GetDimensionsFromInstance []struct { + // ID is the ID argument value. + ID string + } // GetEdition holds details about calls to the GetEdition method. GetEdition []struct { // ID is the ID argument value. ID string // EditionID is the editionID argument value. EditionID string - // Auth reflects if the requester is authorised. + // State is the state argument value. State string } // GetEditions holds details about calls to the GetEditions method. GetEditions []struct { // ID is the ID argument value. ID string - // State reflects if the requester is authorised. + // State is the state argument value. State string } // GetInstance holds details about calls to the GetInstance method. @@ -366,8 +379,8 @@ type StorerMock struct { // EditionID is the editionID argument value. EditionID string } - // GetUniqueDimensionValues holds details about calls to the GetUniqueDimensionValues method. - GetUniqueDimensionValues []struct { + // GetUniqueDimensionAndOptions holds details about calls to the GetUniqueDimensionAndOptions method. + GetUniqueDimensionAndOptions []struct { // ID is the ID argument value. ID string // Dimension is the dimension argument value. @@ -393,15 +406,10 @@ type StorerMock struct { // State is the state argument value. State string } - // Ping holds details about calls to the Ping method. - Ping []struct { - // Ctx is the ctx argument value. - Ctx context.Context - } // UpdateBuildHierarchyTaskState holds details about calls to the UpdateBuildHierarchyTaskState method. UpdateBuildHierarchyTaskState []struct { - // Id is the id argument value. - Id string + // ID is the id argument value. + ID string // Dimension is the dimension argument value. Dimension string // State is the state argument value. @@ -409,8 +417,8 @@ type StorerMock struct { } // UpdateBuildSearchTaskState holds details about calls to the UpdateBuildSearchTaskState method. UpdateBuildSearchTaskState []struct { - // Id is the id argument value. - Id string + // ID is the id argument value. + ID string // Dimension is the dimension argument value. Dimension string // State is the state argument value. @@ -450,8 +458,8 @@ type StorerMock struct { } // UpdateImportObservationsTaskState holds details about calls to the UpdateImportObservationsTaskState method. UpdateImportObservationsTaskState []struct { - // Id is the id argument value. - Id string + // ID is the id argument value. + ID string // State is the state argument value. State string } @@ -606,6 +614,53 @@ func (mock *StorerMock) AddInstanceCalls() []struct { return calls } +// AddVersionDetailsToInstance calls AddVersionDetailsToInstanceFunc. +func (mock *StorerMock) AddVersionDetailsToInstance(ctx context.Context, instanceID string, datasetID string, edition string, version int) error { + if mock.AddVersionDetailsToInstanceFunc == nil { + panic("moq: StorerMock.AddVersionDetailsToInstanceFunc is nil but Storer.AddVersionDetailsToInstance was just called") + } + callInfo := struct { + Ctx context.Context + InstanceID string + DatasetID string + Edition string + Version int + }{ + Ctx: ctx, + InstanceID: instanceID, + DatasetID: datasetID, + Edition: edition, + Version: version, + } + lockStorerMockAddVersionDetailsToInstance.Lock() + mock.calls.AddVersionDetailsToInstance = append(mock.calls.AddVersionDetailsToInstance, callInfo) + lockStorerMockAddVersionDetailsToInstance.Unlock() + return mock.AddVersionDetailsToInstanceFunc(ctx, instanceID, datasetID, edition, version) +} + +// AddVersionDetailsToInstanceCalls gets all the calls that were made to AddVersionDetailsToInstance. +// Check the length with: +// len(mockedStorer.AddVersionDetailsToInstanceCalls()) +func (mock *StorerMock) AddVersionDetailsToInstanceCalls() []struct { + Ctx context.Context + InstanceID string + DatasetID string + Edition string + Version int +} { + var calls []struct { + Ctx context.Context + InstanceID string + DatasetID string + Edition string + Version int + } + lockStorerMockAddVersionDetailsToInstance.RLock() + calls = mock.calls.AddVersionDetailsToInstance + lockStorerMockAddVersionDetailsToInstance.RUnlock() + return calls +} + // CheckDatasetExists calls CheckDatasetExistsFunc. func (mock *StorerMock) CheckDatasetExists(ID string, state string) error { if mock.CheckDatasetExistsFunc == nil { @@ -768,37 +823,6 @@ func (mock *StorerMock) GetDatasetsCalls() []struct { return calls } -// GetDimensionNodesFromInstance calls GetDimensionNodesFromInstanceFunc. -func (mock *StorerMock) GetDimensionNodesFromInstance(ID string) (*models.DimensionNodeResults, error) { - if mock.GetDimensionNodesFromInstanceFunc == nil { - panic("moq: StorerMock.GetDimensionNodesFromInstanceFunc is nil but Storer.GetDimensionNodesFromInstance was just called") - } - callInfo := struct { - ID string - }{ - ID: ID, - } - lockStorerMockGetDimensionNodesFromInstance.Lock() - mock.calls.GetDimensionNodesFromInstance = append(mock.calls.GetDimensionNodesFromInstance, callInfo) - lockStorerMockGetDimensionNodesFromInstance.Unlock() - return mock.GetDimensionNodesFromInstanceFunc(ID) -} - -// GetDimensionNodesFromInstanceCalls gets all the calls that were made to GetDimensionNodesFromInstance. -// Check the length with: -// len(mockedStorer.GetDimensionNodesFromInstanceCalls()) -func (mock *StorerMock) GetDimensionNodesFromInstanceCalls() []struct { - ID string -} { - var calls []struct { - ID string - } - lockStorerMockGetDimensionNodesFromInstance.RLock() - calls = mock.calls.GetDimensionNodesFromInstance - lockStorerMockGetDimensionNodesFromInstance.RUnlock() - return calls -} - // GetDimensionOptions calls GetDimensionOptionsFunc. func (mock *StorerMock) GetDimensionOptions(version *models.Version, dimension string) (*models.DimensionOptionResults, error) { if mock.GetDimensionOptionsFunc == nil { @@ -869,6 +893,37 @@ func (mock *StorerMock) GetDimensionsCalls() []struct { return calls } +// GetDimensionsFromInstance calls GetDimensionsFromInstanceFunc. +func (mock *StorerMock) GetDimensionsFromInstance(ID string) (*models.DimensionNodeResults, error) { + if mock.GetDimensionsFromInstanceFunc == nil { + panic("moq: StorerMock.GetDimensionsFromInstanceFunc is nil but Storer.GetDimensionsFromInstance was just called") + } + callInfo := struct { + ID string + }{ + ID: ID, + } + lockStorerMockGetDimensionsFromInstance.Lock() + mock.calls.GetDimensionsFromInstance = append(mock.calls.GetDimensionsFromInstance, callInfo) + lockStorerMockGetDimensionsFromInstance.Unlock() + return mock.GetDimensionsFromInstanceFunc(ID) +} + +// GetDimensionsFromInstanceCalls gets all the calls that were made to GetDimensionsFromInstance. +// Check the length with: +// len(mockedStorer.GetDimensionsFromInstanceCalls()) +func (mock *StorerMock) GetDimensionsFromInstanceCalls() []struct { + ID string +} { + var calls []struct { + ID string + } + lockStorerMockGetDimensionsFromInstance.RLock() + calls = mock.calls.GetDimensionsFromInstance + lockStorerMockGetDimensionsFromInstance.RUnlock() + return calls +} + // GetEdition calls GetEditionFunc. func (mock *StorerMock) GetEdition(ID string, editionID string, state string) (*models.EditionUpdate, error) { if mock.GetEditionFunc == nil { @@ -1040,10 +1095,10 @@ func (mock *StorerMock) GetNextVersionCalls() []struct { return calls } -// GetUniqueDimensionValues calls GetUniqueDimensionValuesFunc. -func (mock *StorerMock) GetUniqueDimensionValues(ID string, dimension string) (*models.DimensionValues, error) { - if mock.GetUniqueDimensionValuesFunc == nil { - panic("moq: StorerMock.GetUniqueDimensionValuesFunc is nil but Storer.GetUniqueDimensionValues was just called") +// GetUniqueDimensionAndOptions calls GetUniqueDimensionAndOptionsFunc. +func (mock *StorerMock) GetUniqueDimensionAndOptions(ID string, dimension string) (*models.DimensionValues, error) { + if mock.GetUniqueDimensionAndOptionsFunc == nil { + panic("moq: StorerMock.GetUniqueDimensionAndOptionsFunc is nil but Storer.GetUniqueDimensionAndOptions was just called") } callInfo := struct { ID string @@ -1052,16 +1107,16 @@ func (mock *StorerMock) GetUniqueDimensionValues(ID string, dimension string) (* ID: ID, Dimension: dimension, } - lockStorerMockGetUniqueDimensionValues.Lock() - mock.calls.GetUniqueDimensionValues = append(mock.calls.GetUniqueDimensionValues, callInfo) - lockStorerMockGetUniqueDimensionValues.Unlock() - return mock.GetUniqueDimensionValuesFunc(ID, dimension) + lockStorerMockGetUniqueDimensionAndOptions.Lock() + mock.calls.GetUniqueDimensionAndOptions = append(mock.calls.GetUniqueDimensionAndOptions, callInfo) + lockStorerMockGetUniqueDimensionAndOptions.Unlock() + return mock.GetUniqueDimensionAndOptionsFunc(ID, dimension) } -// GetUniqueDimensionValuesCalls gets all the calls that were made to GetUniqueDimensionValues. +// GetUniqueDimensionAndOptionsCalls gets all the calls that were made to GetUniqueDimensionAndOptions. // Check the length with: -// len(mockedStorer.GetUniqueDimensionValuesCalls()) -func (mock *StorerMock) GetUniqueDimensionValuesCalls() []struct { +// len(mockedStorer.GetUniqueDimensionAndOptionsCalls()) +func (mock *StorerMock) GetUniqueDimensionAndOptionsCalls() []struct { ID string Dimension string } { @@ -1069,9 +1124,9 @@ func (mock *StorerMock) GetUniqueDimensionValuesCalls() []struct { ID string Dimension string } - lockStorerMockGetUniqueDimensionValues.RLock() - calls = mock.calls.GetUniqueDimensionValues - lockStorerMockGetUniqueDimensionValues.RUnlock() + lockStorerMockGetUniqueDimensionAndOptions.RLock() + calls = mock.calls.GetUniqueDimensionAndOptions + lockStorerMockGetUniqueDimensionAndOptions.RUnlock() return calls } @@ -1157,48 +1212,17 @@ func (mock *StorerMock) GetVersionsCalls() []struct { return calls } -// Ping calls PingFunc. -func (mock *StorerMock) Ping(ctx context.Context) (time.Time, error) { - if mock.PingFunc == nil { - panic("moq: StorerMock.PingFunc is nil but Storer.Ping was just called") - } - callInfo := struct { - Ctx context.Context - }{ - Ctx: ctx, - } - lockStorerMockPing.Lock() - mock.calls.Ping = append(mock.calls.Ping, callInfo) - lockStorerMockPing.Unlock() - return mock.PingFunc(ctx) -} - -// PingCalls gets all the calls that were made to Ping. -// Check the length with: -// len(mockedStorer.PingCalls()) -func (mock *StorerMock) PingCalls() []struct { - Ctx context.Context -} { - var calls []struct { - Ctx context.Context - } - lockStorerMockPing.RLock() - calls = mock.calls.Ping - lockStorerMockPing.RUnlock() - return calls -} - // UpdateBuildHierarchyTaskState calls UpdateBuildHierarchyTaskStateFunc. func (mock *StorerMock) UpdateBuildHierarchyTaskState(id string, dimension string, state string) error { if mock.UpdateBuildHierarchyTaskStateFunc == nil { panic("moq: StorerMock.UpdateBuildHierarchyTaskStateFunc is nil but Storer.UpdateBuildHierarchyTaskState was just called") } callInfo := struct { - Id string + ID string Dimension string State string }{ - Id: id, + ID: id, Dimension: dimension, State: state, } @@ -1212,12 +1236,12 @@ func (mock *StorerMock) UpdateBuildHierarchyTaskState(id string, dimension strin // Check the length with: // len(mockedStorer.UpdateBuildHierarchyTaskStateCalls()) func (mock *StorerMock) UpdateBuildHierarchyTaskStateCalls() []struct { - Id string + ID string Dimension string State string } { var calls []struct { - Id string + ID string Dimension string State string } @@ -1233,11 +1257,11 @@ func (mock *StorerMock) UpdateBuildSearchTaskState(id string, dimension string, panic("moq: StorerMock.UpdateBuildSearchTaskStateFunc is nil but Storer.UpdateBuildSearchTaskState was just called") } callInfo := struct { - Id string + ID string Dimension string State string }{ - Id: id, + ID: id, Dimension: dimension, State: state, } @@ -1251,12 +1275,12 @@ func (mock *StorerMock) UpdateBuildSearchTaskState(id string, dimension string, // Check the length with: // len(mockedStorer.UpdateBuildSearchTaskStateCalls()) func (mock *StorerMock) UpdateBuildSearchTaskStateCalls() []struct { - Id string + ID string Dimension string State string } { var calls []struct { - Id string + ID string Dimension string State string } @@ -1420,10 +1444,10 @@ func (mock *StorerMock) UpdateImportObservationsTaskState(id string, state strin panic("moq: StorerMock.UpdateImportObservationsTaskStateFunc is nil but Storer.UpdateImportObservationsTaskState was just called") } callInfo := struct { - Id string + ID string State string }{ - Id: id, + ID: id, State: state, } lockStorerMockUpdateImportObservationsTaskState.Lock() @@ -1436,11 +1460,11 @@ func (mock *StorerMock) UpdateImportObservationsTaskState(id string, state strin // Check the length with: // len(mockedStorer.UpdateImportObservationsTaskStateCalls()) func (mock *StorerMock) UpdateImportObservationsTaskStateCalls() []struct { - Id string + ID string State string } { var calls []struct { - Id string + ID string State string } lockStorerMockUpdateImportObservationsTaskState.RLock() diff --git a/vendor/github.com/ONSdigital/dp-filter/observation/row_reader.go b/vendor/github.com/ONSdigital/dp-filter/observation/row_reader.go index 29825dc2..e2085cbc 100644 --- a/vendor/github.com/ONSdigital/dp-filter/observation/row_reader.go +++ b/vendor/github.com/ONSdigital/dp-filter/observation/row_reader.go @@ -3,8 +3,8 @@ package observation import ( "io" - bolt "github.com/ONSdigital/golang-neo4j-bolt-driver" - "github.com/ONSdigital/golang-neo4j-bolt-driver/errors" + bolt "github.com/johnnadratowski/golang-neo4j-bolt-driver" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors" ) //go:generate moq -out observationtest/bolt_rows.go -pkg observationtest . BoltRows diff --git a/vendor/github.com/ONSdigital/dp-filter/observation/store.go b/vendor/github.com/ONSdigital/dp-filter/observation/store.go index 83884666..870b42ff 100644 --- a/vendor/github.com/ONSdigital/dp-filter/observation/store.go +++ b/vendor/github.com/ONSdigital/dp-filter/observation/store.go @@ -6,7 +6,7 @@ import ( "strconv" "github.com/ONSdigital/go-ns/log" - bolt "github.com/ONSdigital/golang-neo4j-bolt-driver" + bolt "github.com/johnnadratowski/golang-neo4j-bolt-driver" ) //go:generate moq -out observationtest/db_pool.go -pkg observationtest . DBPool diff --git a/vendor/github.com/ONSdigital/go-ns/audit/audit.go b/vendor/github.com/ONSdigital/go-ns/audit/audit.go index d77ef003..0e6cfa88 100644 --- a/vendor/github.com/ONSdigital/go-ns/audit/audit.go +++ b/vendor/github.com/ONSdigital/go-ns/audit/audit.go @@ -3,15 +3,25 @@ package audit import ( "context" "fmt" - "github.com/ONSdigital/go-ns/common" - "github.com/ONSdigital/go-ns/handlers/requestID" - "github.com/ONSdigital/go-ns/log" "sort" "time" + + "github.com/ONSdigital/go-ns/common" + "github.com/ONSdigital/go-ns/log" ) //go:generate moq -out generated_mocks.go -pkg audit . AuditorService OutboundProducer +// List of audit messages +const ( + Attempted = "attempted" + Successful = "successful" + Unsuccessful = "unsuccessful" + + AuditError = "error while attempting to record audit event, failing request" + AuditActionErr = "failed to audit action" +) + // Error represents containing details of an attempt to audit and action that failed. type Error struct { Cause string @@ -74,28 +84,40 @@ func New(producer OutboundProducer, namespace string) *Auditor { // decide what do with the error in these cases. // NOTE: Record relies on the identity middleware having run first. If no user / service identity is available in the // provided context an error will be returned. -func (a *Auditor) Record(ctx context.Context, attemptedAction string, actionResult string, params common.Params) error { +func (a *Auditor) Record(ctx context.Context, attemptedAction string, actionResult string, params common.Params) (err error) { + var e Event + defer func() { + if err != nil { + LogActionFailure(ctx, attemptedAction, actionResult, err, ToLogData(params)) + } else { + LogInfo(ctx, "captured audit event", log.Data{"auditEvent": e}) + } + }() + //NOTE: for now we are only auditing user actions - this may be subject to change user := common.User(ctx) service := common.Caller(ctx) if user == "" && service == "" { - return NewAuditError("expected user or caller identity but none found", attemptedAction, actionResult, params) + err = NewAuditError("expected user or caller identity but none found", attemptedAction, actionResult, params) + return } if user == "" { - log.Debug("not user attempted action: skipping audit event", nil) - return nil + log.DebugCtx(ctx, "not user attempted action: skipping audit event", nil) + return } if attemptedAction == "" { - return NewAuditError("attemptedAction required but was empty", "", actionResult, params) + err = NewAuditError("attemptedAction required but was empty", "", actionResult, params) + return } if actionResult == "" { - return NewAuditError("actionResult required but was empty", attemptedAction, "", params) + err = NewAuditError("actionResult required but was empty", attemptedAction, "", params) + return } - e := Event{ + e = Event{ Service: a.service, User: user, AttemptedAction: attemptedAction, @@ -104,15 +126,16 @@ func (a *Auditor) Record(ctx context.Context, attemptedAction string, actionResu Params: params, } - e.RequestID = requestID.Get(ctx) + e.RequestID = common.GetRequestId(ctx) avroBytes, err := a.marshalToAvro(e) if err != nil { - return NewAuditError("error marshalling event to arvo", attemptedAction, actionResult, params) + err = NewAuditError("error marshalling event to avro", attemptedAction, actionResult, params) + return } a.producer.Output() <- avroBytes - return nil + return } //NewAuditError creates new audit.Error with default field values where necessary and orders the params alphabetically. @@ -166,3 +189,16 @@ func (e Error) formatParams() string { result += "]" return result } + +//ToLogData convert common.Params to log.Data +func ToLogData(p common.Params) log.Data { + if len(p) == 0 { + return nil + } + + data := log.Data{} + for k, v := range p { + data[k] = v + } + return data +} diff --git a/vendor/github.com/ONSdigital/go-ns/audit/audit_mock/README.md b/vendor/github.com/ONSdigital/go-ns/audit/audit_mock/README.md new file mode 100644 index 00000000..adf064b7 --- /dev/null +++ b/vendor/github.com/ONSdigital/go-ns/audit/audit_mock/README.md @@ -0,0 +1,22 @@ +### Testing auditing + +The `audit_mocks` package provides useful methods for verifying calls to `audit.Record()` reducing the amount of +code duplication to setup a mock auditor and verify its invocations during a test case. + +### Getting started +Create an auditor mock that returns no error. +```go +auditor := audit_mock.New(t) +``` +Create an auditor mock that returns an error when `Record()` is called with particular action and result values +```go +auditor := audit_mock.NewErroring(t, "some task", "the outcome") +``` +Assert `auditor.Record()` is called the expected number of times and the `action`, `result` and `auditParam` values in + each call are as expected. +```go +auditor.AssertRecordCalls( + audit_mock.Expected{"my_action", audit.Attempted, common.Params{"key":"value"}, + audit_mock.Expected{instance.GetInstancesAction, audit.Successful, nil}, +) +``` diff --git a/vendor/github.com/ONSdigital/go-ns/audit/audit_mock/helper.go b/vendor/github.com/ONSdigital/go-ns/audit/audit_mock/helper.go new file mode 100644 index 00000000..edb4dfc3 --- /dev/null +++ b/vendor/github.com/ONSdigital/go-ns/audit/audit_mock/helper.go @@ -0,0 +1,137 @@ +package audit_mock + +import ( + "context" + "errors" + "fmt" + "github.com/ONSdigital/go-ns/audit" + "github.com/ONSdigital/go-ns/common" + . "github.com/smartystreets/goconvey/convey" + "reflect" +) + +//ErrAudit is the test error returned from a MockAuditor if the audit action & result match error trigger criteria +var ErrAudit = errors.New("auditing error") + +//Expected is a struct encapsulating the method parameters to audit.Record +type Expected struct { + Action string + Result string + Params common.Params +} + +// actual the actual calls to auditor.Record +type actual Expected + +//MockAuditor is wrapper around the generated mock implementation of audit.AuditorService which can be configured +// to return an error when specified audit action / result values are passed into the Record method, also provides +//convenience test methods for asserting calls & params made to the mock. +type MockAuditor struct { + *audit.AuditorServiceMock +} + +//NewExpectation is a constructor for creating a new Expected struct +func NewExpectation(action string, result string, params common.Params) Expected { + return Expected{ + Action: action, + Result: result, + Params: params, + } +} + +//actualCalls convenience method for converting the call values to the right format. +func (m *MockAuditor) actualCalls() []actual { + if len(m.RecordCalls()) == 0 { + return []actual{} + } + + actuals := make([]actual, 0) + for _, a := range m.RecordCalls() { + actuals = append(actuals, actual{Action: a.Action, Result: a.Result, Params: a.Params}) + } + return actuals +} + +//New creates new instance of MockAuditor that does not return any errors +func New() *MockAuditor { + return &MockAuditor{ + &audit.AuditorServiceMock{ + RecordFunc: func(ctx context.Context, action string, result string, params common.Params) error { + return nil + }, + }, + } +} + +//NewErroring creates new instance of MockAuditor that will return ErrAudit if the supplied audit action and result +// match the specified error trigger values. +func NewErroring(a string, r string) *MockAuditor { + return &MockAuditor{ + &audit.AuditorServiceMock{ + RecordFunc: func(ctx context.Context, action string, result string, params common.Params) error { + if action == a && r == result { + audit.LogActionFailure(ctx, a, r, ErrAudit, audit.ToLogData(params)) + return ErrAudit + } + return nil + }, + }, + } +} + +//AssertRecordCalls is a convenience method which asserts the expected number of Record calls are made and +// the parameters of each match the expected values. +func (m *MockAuditor) AssertRecordCalls(expected ...Expected) { + Convey("auditor.Record is called the expected number of times with the expected parameters", func() { + + // shouldAuditAsExpected is a custom implementation of a Goconvey assertion which adds additional context to + // test failures reports to aid understanding/debugging test failures. + shouldAuditAsExpected := func(a interface{}, e ...interface{}) string { + if a == nil { + return "auditor.Record could not assert audit.Record calls: actual parameter required but was empty" + } + + if e == nil || len(e) == 0 || e[0] == nil { + return "auditor.Record could not assert audit.Record calls: expected parameter required but was empty" + } + + actualCalls, ok := a.([]actual) + if !ok { + return fmt.Sprintf("auditor.Record could not assert audit.Record calls: incorrect type for actual parameter expected: %s, actual: %s", reflect.TypeOf(actual{}), reflect.TypeOf(a)) + } + + expectedCalls, ok := e[0].([]Expected) + if !ok { + return fmt.Sprintf("auditor.Record could not assert audit.Record calls: incorrect type for expected parameter: expected: %s, actual: %s", reflect.TypeOf(Expected{}), reflect.TypeOf(e[0])) + } + + if len(actualCalls) != len(expectedCalls) { + return fmt.Sprintf("auditor.Record incorrect number of invocations, expected: %d, actual: %d", len(expectedCalls), len(actualCalls)) + } + + total := len(actualCalls) + var invocation int + for i, call := range actualCalls { + invocation = i + 1 + + action := expectedCalls[i].Action + if equalErr := ShouldEqual(call.Action, action); equalErr != "" { + return fmt.Sprintf("auditor.Record invocation %d/%d incorrect audit action - expected: %q, actual: %q", invocation, total, action, call.Action) + } + + result := expectedCalls[i].Result + if equalErr := ShouldEqual(call.Result, result); equalErr != "" { + return fmt.Sprintf("auditor.Record invocation %d/%d incorrect audit result - expected: %q, actual: %q", invocation, total, result, call.Result) + } + + params := expectedCalls[i].Params + if equalErr := ShouldResemble(call.Params, params); equalErr != "" { + return fmt.Sprintf("auditor.Record invocation %d/%d incorrect auditParams - expected: %+v, actual: %+v", invocation, total, params, call.Params) + } + } + return "" + } + + So(m.actualCalls(), shouldAuditAsExpected, expected) + }) +} diff --git a/vendor/github.com/ONSdigital/go-ns/audit/log.go b/vendor/github.com/ONSdigital/go-ns/audit/log.go new file mode 100644 index 00000000..1907e636 --- /dev/null +++ b/vendor/github.com/ONSdigital/go-ns/audit/log.go @@ -0,0 +1,56 @@ +package audit + +import ( + "context" + + "github.com/ONSdigital/go-ns/common" + "github.com/ONSdigital/go-ns/log" + "github.com/pkg/errors" +) + +const ( + reqUser = "req_user" + reqCaller = "req_caller" +) + +// LogActionFailure adds auditting data to log.Data before calling LogError +func LogActionFailure(ctx context.Context, auditedAction string, auditedResult string, err error, logData log.Data) { + if logData == nil { + logData = log.Data{} + } + + logData["auditAction"] = auditedAction + logData["auditResult"] = auditedResult + + LogError(ctx, errors.WithMessage(err, AuditActionErr), logData) +} + +// LogError creates a structured error message when auditing fails +func LogError(ctx context.Context, err error, data log.Data) { + data = addLogData(ctx, data) + + log.ErrorCtx(ctx, err, data) +} + +// LogInfo creates a structured info message when auditing succeeds +func LogInfo(ctx context.Context, message string, data log.Data) { + data = addLogData(ctx, data) + + log.InfoCtx(ctx, message, data) +} + +func addLogData(ctx context.Context, data log.Data) log.Data { + if data == nil { + data = log.Data{} + } + + if user := common.User(ctx); user != "" { + data[reqUser] = user + } + + if caller := common.Caller(ctx); caller != "" { + data[reqCaller] = caller + } + + return data +} diff --git a/vendor/github.com/ONSdigital/go-ns/clients/identity/authentication.go b/vendor/github.com/ONSdigital/go-ns/clients/identity/authentication.go index 8c69dd6a..505e4cb3 100644 --- a/vendor/github.com/ONSdigital/go-ns/clients/identity/authentication.go +++ b/vendor/github.com/ONSdigital/go-ns/clients/identity/authentication.go @@ -5,12 +5,19 @@ import ( "encoding/json" "io/ioutil" "net/http" + "strings" "github.com/ONSdigital/go-ns/common" "github.com/ONSdigital/go-ns/log" "github.com/ONSdigital/go-ns/rchttp" ) +type tokenObject struct { + numberOfParts int + hasPrefix bool + tokenPart string +} + type IdentityClient common.APIClient type IdentityClienter interface { @@ -27,12 +34,20 @@ func NewAPIClient(cli common.RCHTTPClienter, url string) (api *IdentityClient) { // CheckRequest calls the AuthAPI to check florenceToken or authToken func (api IdentityClient) CheckRequest(req *http.Request) (context.Context, int, error) { - log.DebugR(req, "CheckRequest called", nil) ctx := req.Context() florenceToken := req.Header.Get(common.FlorenceHeaderKey) + if len(florenceToken) < 1 { + c, err := req.Cookie(common.FlorenceCookieKey) + if err != nil { + log.DebugR(req, err.Error(), nil) + } else { + florenceToken = c.Value + } + } + authToken := req.Header.Get(common.AuthHeaderKey) isUserReq := len(florenceToken) > 0 @@ -44,7 +59,14 @@ func (api IdentityClient) CheckRequest(req *http.Request) (context.Context, int, } url := api.BaseURL + "/identity" - logData := log.Data{"is_user_request": isUserReq, "is_service_request": isServiceReq, "url": url} + + logData := log.Data{ + "is_user_request": isUserReq, + "is_service_request": isServiceReq, + "url": url, + } + splitTokens(florenceToken, authToken, logData) + log.DebugR(req, "calling AuthAPI to authenticate", logData) outboundAuthReq, err := http.NewRequest("GET", url, nil) @@ -101,6 +123,31 @@ func (api IdentityClient) CheckRequest(req *http.Request) (context.Context, int, return ctx, http.StatusOK, nil } +func splitTokens(florenceToken, authToken string, logData log.Data) { + if len(florenceToken) > 0 { + logData["florence_token"] = splitToken(florenceToken) + } + if len(authToken) > 0 { + logData["auth_token"] = splitToken(authToken) + } +} + +func splitToken(token string) (tokenObj tokenObject) { + splitToken := strings.Split(token, " ") + tokenObj.numberOfParts = len(splitToken) + tokenObj.hasPrefix = strings.HasPrefix(token, common.BearerPrefix) + + // sample last 6 chars (or half, if smaller) of last token part + lastTokenPart := len(splitToken) - 1 + tokenSampleStart := len(splitToken[lastTokenPart]) - 6 + if tokenSampleStart < 1 { + tokenSampleStart = len(splitToken[lastTokenPart]) / 2 + } + tokenObj.tokenPart = splitToken[lastTokenPart][tokenSampleStart:] + + return tokenObj +} + // unmarshalIdentityResponse converts a resp.Body (JSON) into an IdentityResponse func unmarshalIdentityResponse(resp *http.Response) (identityResp *common.IdentityResponse, err error) { b, err := ioutil.ReadAll(resp.Body) @@ -109,8 +156,8 @@ func unmarshalIdentityResponse(resp *http.Response) (identityResp *common.Identi } defer func() { - if err := resp.Body.Close(); err != nil { - log.ErrorC("error closing response body", err, nil) + if errClose := resp.Body.Close(); errClose != nil { + log.ErrorC("error closing response body", errClose, nil) } }() diff --git a/vendor/github.com/ONSdigital/go-ns/common/client.go b/vendor/github.com/ONSdigital/go-ns/common/client.go index 66276b69..e05eea91 100644 --- a/vendor/github.com/ONSdigital/go-ns/common/client.go +++ b/vendor/github.com/ONSdigital/go-ns/common/client.go @@ -2,6 +2,7 @@ package common import "sync" +// APIClient represents a common structure for any api client type APIClient struct { BaseURL string AuthToken string diff --git a/vendor/github.com/ONSdigital/go-ns/common/errors.go b/vendor/github.com/ONSdigital/go-ns/common/errors.go index 19f696d4..d834a746 100644 --- a/vendor/github.com/ONSdigital/go-ns/common/errors.go +++ b/vendor/github.com/ONSdigital/go-ns/common/errors.go @@ -1,20 +1,24 @@ package common +// ONSError encapsulates an error with additional parameters type ONSError struct { Parameters map[string]interface{} RootError error } +// NewONSError creates a new ONS error func NewONSError(e error, description string) *ONSError { err := &ONSError{RootError: e} err.AddParameter("ErrorDescription", description) return err } +// Error returns the ONSError RootError message func (e ONSError) Error() string { return e.RootError.Error() } +// AddParameter method creates or overwrites parameters attatched to an ONSError func (e *ONSError) AddParameter(name string, value interface{}) *ONSError { if e.Parameters == nil { e.Parameters = make(map[string]interface{}, 0) diff --git a/vendor/github.com/ONSdigital/go-ns/common/identity.go b/vendor/github.com/ONSdigital/go-ns/common/identity.go index 5312aea2..e2931c49 100644 --- a/vendor/github.com/ONSdigital/go-ns/common/identity.go +++ b/vendor/github.com/ONSdigital/go-ns/common/identity.go @@ -2,17 +2,24 @@ package common import ( "context" + "math/rand" "net/http" + "time" ) +// ContextKey is an alias of type string type ContextKey string +// A list of common constants used across go-ns packages const ( FlorenceHeaderKey = "X-Florence-Token" DownloadServiceHeaderKey = "X-Download-Service-Token" - AuthHeaderKey = "Authorization" - UserHeaderKey = "User-Identity" + FlorenceCookieKey = "access_token" + + AuthHeaderKey = "Authorization" + UserHeaderKey = "User-Identity" + RequestHeaderKey = "X-Request-Id" DeprecatedAuthHeader = "Internal-Token" LegacyUser = "legacyUser" @@ -20,13 +27,15 @@ const ( UserIdentityKey = ContextKey("User-Identity") CallerIdentityKey = ContextKey("Caller-Identity") + RequestIdKey = ContextKey("request-id") ) -// interface to allow mocking of auth.CheckRequest +// CheckRequester is an interface to allow mocking of auth.CheckRequest type CheckRequester interface { CheckRequest(*http.Request) (context.Context, int, error) } +// IdentityResponse represents the response from the identity service type IdentityResponse struct { Identifier string `json:"identifier"` } @@ -68,6 +77,7 @@ func SetUser(ctx context.Context, user string) context.Context { return context.WithValue(ctx, UserIdentityKey, user) } +// AddAuthHeaders sets authentication headers for request func AddAuthHeaders(ctx context.Context, r *http.Request, serviceToken string) { if IsUserPresent(ctx) { AddUserHeader(r, User(ctx)) @@ -103,3 +113,33 @@ func SetCaller(ctx context.Context, caller string) context.Context { return context.WithValue(ctx, CallerIdentityKey, caller) } + +// GetRequestId gets the correlation id on the context +func GetRequestId(ctx context.Context) string { + correlationId, _ := ctx.Value(RequestIdKey).(string) + return correlationId +} + +// WithRequestId sets the correlation id on the context +func WithRequestId(ctx context.Context, correlationId string) context.Context { + return context.WithValue(ctx, RequestIdKey, correlationId) +} + +// AddRequestIdHeader add header for given correlation ID +func AddRequestIdHeader(r *http.Request, token string) { + if len(token) > 0 { + r.Header.Add(RequestHeaderKey, token) + } +} + +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") +var requestIDRandom = rand.New(rand.NewSource(time.Now().UnixNano())) + +// NewRequestID generates a random string of requested length +func NewRequestID(size int) string { + b := make([]rune, size) + for i := range b { + b[i] = letters[requestIDRandom.Intn(len(letters))] + } + return string(b) +} diff --git a/vendor/github.com/ONSdigital/go-ns/handlers/requestID/handler.go b/vendor/github.com/ONSdigital/go-ns/handlers/requestID/handler.go index 30d788ae..1ef461c0 100644 --- a/vendor/github.com/ONSdigital/go-ns/handlers/requestID/handler.go +++ b/vendor/github.com/ONSdigital/go-ns/handlers/requestID/handler.go @@ -1,42 +1,24 @@ package requestID import ( - "context" - "math/rand" "net/http" - "time" -) - -var requestIDRandom = rand.New(rand.NewSource(time.Now().UnixNano())) - -type contextKey string -const ContextKey = contextKey("request-id") + "github.com/ONSdigital/go-ns/common" +) // Handler is a wrapper which adds an X-Request-Id header if one does not yet exist func Handler(size int) func(http.Handler) http.Handler { return func(h http.Handler) http.Handler { - var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - requestID := req.Header.Get("X-Request-Id") + requestID := req.Header.Get(common.RequestHeaderKey) if len(requestID) == 0 { - b := make([]rune, size) - for i := range b { - b[i] = letters[requestIDRandom.Intn(len(letters))] - } - requestID = string(b) - req.Header.Set("X-Request-Id", requestID) + requestID = common.NewRequestID(size) + common.AddRequestIdHeader(req, requestID) } - ctx := context.WithValue(req.Context(), ContextKey, requestID) - h.ServeHTTP(w, req.WithContext(ctx)) + h.ServeHTTP(w, req.WithContext(common.WithRequestId(req.Context(), requestID))) }) } } - -func Get(ctx context.Context) string { - id, _ := ctx.Value(ContextKey).(string) - return id -} diff --git a/vendor/github.com/ONSdigital/go-ns/kafka/consumer-group.go b/vendor/github.com/ONSdigital/go-ns/kafka/consumer-group.go index 12582139..48b81b8d 100644 --- a/vendor/github.com/ONSdigital/go-ns/kafka/consumer-group.go +++ b/vendor/github.com/ONSdigital/go-ns/kafka/consumer-group.go @@ -117,8 +117,9 @@ func newConsumer(brokers []string, topic string, group string, offset int64, syn config.Consumer.Return.Errors = true config.Consumer.MaxWaitTime = 50 * time.Millisecond config.Consumer.Offsets.Initial = offset + config.Consumer.Offsets.Retention = 0 // indefinite retention - logData := log.Data{"topic": topic, "group": group} + logData := log.Data{"topic": topic, "group": group, "config": config} consumer, err := cluster.NewConsumer(brokers, group, []string{topic}, config) if err != nil { @@ -149,7 +150,7 @@ func newConsumer(brokers []string, topic string, group string, offset int64, syn // listener goroutine - listen to consumer.Messages() and upstream them // if this blocks while upstreaming a message, we can shutdown consumer via the following goroutine go func() { - logData := log.Data{"topic": topic, "group": group} + logData := log.Data{"topic": topic, "group": group, "config": config} log.Info("Started kafka consumer listener", logData) defer close(cg.closed) diff --git a/vendor/github.com/ONSdigital/go-ns/log/log.go b/vendor/github.com/ONSdigital/go-ns/log/log.go index adc19bcb..e83ffc42 100644 --- a/vendor/github.com/ONSdigital/go-ns/log/log.go +++ b/vendor/github.com/ONSdigital/go-ns/log/log.go @@ -2,6 +2,7 @@ package log import ( "bufio" + "context" "encoding/json" "errors" "fmt" @@ -12,13 +13,15 @@ import ( "sync" "time" + "github.com/ONSdigital/go-ns/common" "github.com/mgutz/ansi" ) // Namespace is the service namespace used for logging var Namespace = "service-namespace" -// HumanReadable, if true, outputs log events in a human readable format +// HumanReadable represents a flag to determine if log events +// will be in a human readable format var HumanReadable bool var hrMutex sync.Mutex @@ -33,9 +36,9 @@ func configureHumanReadable() { // Data contains structured log data type Data map[string]interface{} -// Context returns a context ID from a request (using X-Request-Id) -func Context(req *http.Request) string { - return req.Header.Get("X-Request-Id") +// GetRequestID returns the request ID from a request (using X-Request-Id) +func GetRequestID(req *http.Request) string { + return req.Header.Get(common.RequestHeaderKey) } // Handler wraps a http.Handler and logs the status code and total response time @@ -48,14 +51,18 @@ func Handler(h http.Handler) http.Handler { e := time.Now() d := e.Sub(s) - Event("request", Context(req), Data{ + data := Data{ "start": s, "end": e, "duration": d, "status": rc.statusCode, "method": req.Method, "path": req.URL.Path, - }) + } + if len(req.URL.RawQuery) > 0 { + data["query"] = req.URL.Query() + } + Event("request", GetRequestID(req), data) }) } @@ -85,15 +92,15 @@ func (r *responseCapture) Hijack() (net.Conn, *bufio.ReadWriter, error) { // Event records an event var Event = event -func event(name string, context string, data Data) { +func event(name string, correlationKey string, data Data) { m := map[string]interface{}{ "created": time.Now(), "event": name, "namespace": Namespace, } - if len(context) > 0 { - m["context"] = context + if len(correlationKey) > 0 { + m["correlation_key"] = correlationKey } if data != nil { @@ -101,7 +108,7 @@ func event(name string, context string, data Data) { } if HumanReadable { - printHumanReadable(name, context, data, m) + printHumanReadable(name, correlationKey, data, m) return } @@ -111,24 +118,24 @@ func event(name string, context string, data Data) { // We'll log the error (which for our purposes, can't fail), which // gives us an indication we have something to investigate b, _ = json.Marshal(map[string]interface{}{ - "created": time.Now(), - "event": "log_error", - "namespace": Namespace, - "context": context, - "data": map[string]interface{}{"error": err.Error()}, + "created": time.Now(), + "event": "log_error", + "namespace": Namespace, + "correlation_key": correlationKey, + "data": map[string]interface{}{"error": err.Error()}, }) } fmt.Fprintf(os.Stdout, "%s\n", b) } -func printHumanReadable(name, context string, data Data, m map[string]interface{}) { +func printHumanReadable(name, correlationKey string, data Data, m map[string]interface{}) { hrMutex.Lock() defer hrMutex.Unlock() ctx := "" - if len(context) > 0 { - ctx = "[" + context + "] " + if len(correlationKey) > 0 { + ctx = "[" + correlationKey + "] " } msg := "" if message, ok := data["message"]; ok { @@ -163,21 +170,27 @@ func printHumanReadable(name, context string, data Data, m map[string]interface{ } } -// ErrorC is a structured error message with context -func ErrorC(context string, err error, data Data) { +// ErrorC is a structured error message with correlationKey +func ErrorC(correlationKey string, err error, data Data) { if data == nil { data = Data{} } - if _, ok := data["error"]; !ok { + if err != nil { data["message"] = err.Error() data["error"] = err } - Event("error", context, data) + Event("error", correlationKey, data) +} + +// ErrorCtx is a structured error message and retrieves the correlationKey from go context +func ErrorCtx(ctx context.Context, err error, data Data) { + correlationKey := common.GetRequestId(ctx) + ErrorC(correlationKey, err, data) } // ErrorR is a structured error message for a request func ErrorR(req *http.Request, err error, data Data) { - ErrorC(Context(req), err, data) + ErrorC(GetRequestID(req), err, data) } // Error is a structured error message @@ -185,20 +198,26 @@ func Error(err error, data Data) { ErrorC("", err, data) } -// DebugC is a structured debug message with context -func DebugC(context string, message string, data Data) { +// DebugC is a structured debug message with correlationKey +func DebugC(correlationKey string, message string, data Data) { if data == nil { data = Data{} } - if _, ok := data["message"]; !ok { + if len(message) > 0 { data["message"] = message } - Event("debug", context, data) + Event("debug", correlationKey, data) +} + +// DebugCtx is a structured debug message and retrieves the correlationKey from go context +func DebugCtx(ctx context.Context, message string, data Data) { + correlationKey := common.GetRequestId(ctx) + DebugC(correlationKey, message, data) } // DebugR is a structured debug message for a request func DebugR(req *http.Request, message string, data Data) { - DebugC(Context(req), message, data) + DebugC(GetRequestID(req), message, data) } // Debug is a structured trace message @@ -206,20 +225,26 @@ func Debug(message string, data Data) { DebugC("", message, data) } -// TraceC is a structured trace message with context -func TraceC(context string, message string, data Data) { +// TraceC is a structured trace message with correlationKey +func TraceC(correlationKey string, message string, data Data) { if data == nil { data = Data{} } - if _, ok := data["message"]; !ok { + if len(message) > 0 { data["message"] = message } - Event("trace", context, data) + Event("trace", correlationKey, data) +} + +// TraceCtx is a structured trace message and retrieves the correlationKey from go context +func TraceCtx(ctx context.Context, message string, data Data) { + correlationKey := common.GetRequestId(ctx) + TraceC(correlationKey, message, data) } // TraceR is a structured trace message for a request func TraceR(req *http.Request, message string, data Data) { - TraceC(Context(req), message, data) + TraceC(GetRequestID(req), message, data) } // Trace is a structured trace message @@ -227,20 +252,26 @@ func Trace(message string, data Data) { TraceC("", message, data) } -// InfoC is a structured info message with context -func InfoC(context string, message string, data Data) { +// InfoC is a structured info message with correlationKey +func InfoC(correlationKey string, message string, data Data) { if data == nil { data = Data{} } - if _, ok := data["message"]; !ok { + if len(message) > 0 { data["message"] = message } - Event("info", context, data) + Event("info", correlationKey, data) +} + +// InfoCtx is a structured info message and retrieves the correlationKey from go context +func InfoCtx(ctx context.Context, message string, data Data) { + correlationKey := common.GetRequestId(ctx) + InfoC(correlationKey, message, data) } // InfoR is a structured info message for a request func InfoR(req *http.Request, message string, data Data) { - InfoC(Context(req), message, data) + InfoC(GetRequestID(req), message, data) } // Info is a structured info message diff --git a/vendor/github.com/ONSdigital/go-ns/mongo/healthcheck.go b/vendor/github.com/ONSdigital/go-ns/mongo/healthcheck.go index d0b67e72..214c91f2 100644 --- a/vendor/github.com/ONSdigital/go-ns/mongo/healthcheck.go +++ b/vendor/github.com/ONSdigital/go-ns/mongo/healthcheck.go @@ -2,7 +2,7 @@ package mongo import ( "github.com/ONSdigital/go-ns/log" - mgo "gopkg.in/mgo.v2" + mgo "github.com/gedge/mgo" ) // HealthCheckClient provides a healthcheck.Client implementation for health checking the service diff --git a/vendor/github.com/ONSdigital/go-ns/mongo/mongo.go b/vendor/github.com/ONSdigital/go-ns/mongo/mongo.go index 5ea92c98..7db1ab80 100644 --- a/vendor/github.com/ONSdigital/go-ns/mongo/mongo.go +++ b/vendor/github.com/ONSdigital/go-ns/mongo/mongo.go @@ -5,9 +5,22 @@ import ( "errors" "time" - mgo "gopkg.in/mgo.v2" + mgo "github.com/gedge/mgo" + "github.com/gedge/mgo/bson" ) +// keep these in sync with Timestamps tags below +const ( + lastUpdatedKey = "last_updated" + uniqueTimestampKey = "unique_timestamp" +) + +// keep tags in sync with above const +type Timestamps struct { + LastUpdated time.Time `bson:"last_updated,omitempty" json:"last_updated,omitempty"` + UniqueTimestamp *bson.MongoTimestamp `bson:"unique_timestamp,omitempty" json:"-"` +} + // Shutdown represents an interface to the shutdown method type Shutdown interface { shutdown(ctx context.Context, session *mgo.Session, closedChannel chan bool) @@ -53,3 +66,85 @@ func Close(ctx context.Context, session *mgo.Session) error { return ctx.Err() } } + +// withCurrentDate creates or adds $currentDate to updateDoc - populates that with key:val +func withCurrentDate(updateDoc bson.M, key string, val interface{}) (bson.M, error) { + var currentDate bson.M + var ok bool + if currentDate, ok = updateDoc["$currentDate"].(bson.M); !ok { + currentDate = bson.M{} + } + switch v := val.(type) { + case bool, bson.M: + currentDate[key] = v + default: + return nil, errors.New("withCurrentDate: Cannot handle that type") + } + updateDoc["$currentDate"] = currentDate + return updateDoc, nil +} + +// WithUpdates adds all timestamps to updateDoc +func WithUpdates(updateDoc bson.M) (bson.M, error) { + newUpdateDoc, err := WithLastUpdatedUpdate(updateDoc) + if err != nil { + return nil, err + } + return WithUniqueTimestampUpdate(newUpdateDoc) +} + +// WithNamespacedUpdates adds all timestamps to updateDoc +func WithNamespacedUpdates(updateDoc bson.M, prefixes []string) (bson.M, error) { + newUpdateDoc, err := WithNamespacedLastUpdatedUpdate(updateDoc, prefixes) + if err != nil { + return nil, err + } + return WithNamespacedUniqueTimestampUpdate(newUpdateDoc, prefixes) +} + +// WithLastUpdatedUpdate adds last_updated to updateDoc +func WithLastUpdatedUpdate(updateDoc bson.M) (bson.M, error) { + return withCurrentDate(updateDoc, lastUpdatedKey, true) +} + +// WithNamespacedLastUpdatedUpdate adds unique timestamp to updateDoc +func WithNamespacedLastUpdatedUpdate(updateDoc bson.M, prefixes []string) (newUpdateDoc bson.M, err error) { + newUpdateDoc = updateDoc + for _, prefix := range prefixes { + if newUpdateDoc, err = withCurrentDate(newUpdateDoc, prefix+lastUpdatedKey, true); err != nil { + return nil, err + } + } + return newUpdateDoc, nil +} + +// WithUniqueTimestampUpdate adds unique timestamp to updateDoc +func WithUniqueTimestampUpdate(updateDoc bson.M) (bson.M, error) { + return withCurrentDate(updateDoc, uniqueTimestampKey, bson.M{"$type": "timestamp"}) +} + +// WithNamespacedUniqueTimestampUpdate adds unique timestamp to updateDoc +func WithNamespacedUniqueTimestampUpdate(updateDoc bson.M, prefixes []string) (newUpdateDoc bson.M, err error) { + newUpdateDoc = updateDoc + for _, prefix := range prefixes { + if newUpdateDoc, err = withCurrentDate(newUpdateDoc, prefix+uniqueTimestampKey, bson.M{"$type": "timestamp"}); err != nil { + return nil, err + } + } + return newUpdateDoc, nil +} + +// WithUniqueTimestampQuery adds unique timestamp to queryDoc +func WithUniqueTimestampQuery(queryDoc bson.M, timestamp bson.MongoTimestamp) bson.M { + queryDoc[uniqueTimestampKey] = timestamp + return queryDoc +} + +// WithNamespacedUniqueTimestampQuery adds unique timestamps to queryDoc sub-docs +func WithNamespacedUniqueTimestampQuery(queryDoc bson.M, timestamps []bson.MongoTimestamp, prefixes []string) bson.M { + newQueryDoc := queryDoc + for idx, prefix := range prefixes { + newQueryDoc[prefix+uniqueTimestampKey] = timestamps[idx] + } + return newQueryDoc +} diff --git a/vendor/github.com/ONSdigital/go-ns/neo4j/healthcheck.go b/vendor/github.com/ONSdigital/go-ns/neo4j/healthcheck.go index 1b33d768..631c5930 100644 --- a/vendor/github.com/ONSdigital/go-ns/neo4j/healthcheck.go +++ b/vendor/github.com/ONSdigital/go-ns/neo4j/healthcheck.go @@ -3,7 +3,7 @@ package neo4j import ( "github.com/ONSdigital/go-ns/healthcheck" "github.com/ONSdigital/go-ns/log" - bolt "github.com/ONSdigital/golang-neo4j-bolt-driver" + bolt "github.com/johnnadratowski/golang-neo4j-bolt-driver" ) // ensure the Neo4jClient satisfies the Client interface. diff --git a/vendor/github.com/ONSdigital/go-ns/rchttp/client.go b/vendor/github.com/ONSdigital/go-ns/rchttp/client.go index e526d2fa..c677c341 100644 --- a/vendor/github.com/ONSdigital/go-ns/rchttp/client.go +++ b/vendor/github.com/ONSdigital/go-ns/rchttp/client.go @@ -119,6 +119,19 @@ func (c *Client) Do(ctx context.Context, req *http.Request) (*http.Response, err } } + // get any existing correlation-id (might be "id1,id2"), append a new one, add to headers + upstreamCorrelationIds := common.GetRequestId(ctx) + addedIdLen := 20 + if upstreamCorrelationIds != "" { + // get length of (first of) IDs (e.g. "id1" is 3), new ID will be half that size + addedIdLen = len(upstreamCorrelationIds) / 2 + if commaPosition := strings.Index(upstreamCorrelationIds, ","); commaPosition > 1 { + addedIdLen = commaPosition / 2 + } + upstreamCorrelationIds += "," + } + common.AddRequestIdHeader(req, upstreamCorrelationIds+common.NewRequestID(addedIdLen)) + doer := func(args ...interface{}) (*http.Response, error) { req := args[2].(*http.Request) if req.ContentLength > 0 { diff --git a/vendor/github.com/gedge/mgo/CONTRIBUTING.md b/vendor/github.com/gedge/mgo/CONTRIBUTING.md new file mode 100644 index 00000000..79539955 --- /dev/null +++ b/vendor/github.com/gedge/mgo/CONTRIBUTING.md @@ -0,0 +1,14 @@ +Contributing +------------------------- + +We really appreciate contributions, but they must meet the following requirements: + +* A PR should have a brief description of the problem/feature being proposed +* Pull requests should target the `development` branch +* Existing tests should pass and any new code should be covered with it's own test(s) (use [travis-ci](https://travis-ci.org)) +* New functions should be [documented](https://blog.golang.org/godoc-documenting-go-code) clearly +* Code should pass `golint`, `go vet` and `go fmt` + +We merge PRs into `development`, which is then tested in a sharded, replicated environment in our datacenter for regressions. Once everyone is happy, we merge to master - this is to maintain a bit of quality control past the usual PR process. + +**Thanks** for helping! diff --git a/vendor/gopkg.in/mgo.v2/LICENSE b/vendor/github.com/gedge/mgo/LICENSE similarity index 100% rename from vendor/gopkg.in/mgo.v2/LICENSE rename to vendor/github.com/gedge/mgo/LICENSE diff --git a/vendor/gopkg.in/mgo.v2/Makefile b/vendor/github.com/gedge/mgo/Makefile similarity index 100% rename from vendor/gopkg.in/mgo.v2/Makefile rename to vendor/github.com/gedge/mgo/Makefile diff --git a/vendor/github.com/gedge/mgo/README.md b/vendor/github.com/gedge/mgo/README.md new file mode 100644 index 00000000..6c87fa90 --- /dev/null +++ b/vendor/github.com/gedge/mgo/README.md @@ -0,0 +1,81 @@ +[![Build Status](https://travis-ci.org/globalsign/mgo.svg?branch=master)](https://travis-ci.org/globalsign/mgo) [![GoDoc](https://godoc.org/github.com/globalsign/mgo?status.svg)](https://godoc.org/github.com/globalsign/mgo) + +The MongoDB driver for Go +------------------------- + +This fork has had a few improvements by ourselves as well as several PR's merged from the original mgo repo that are currently awaiting review. +Changes are mostly geared towards performance improvements and bug fixes, though a few new features have been added. + +Further PR's (with tests) are welcome, but please maintain backwards compatibility. + +Detailed documentation of the API is available at +[GoDoc](https://godoc.org/github.com/globalsign/mgo). + +A [sub-package](https://godoc.org/github.com/globalsign/mgo/bson) that implements the [BSON](http://bsonspec.org) specification is also included, and may be used independently of the driver. + +## Changes +* Fixes attempting to authenticate before every query ([details](https://github.com/go-mgo/mgo/issues/254)) +* Removes bulk update / delete batch size limitations ([details](https://github.com/go-mgo/mgo/issues/288)) +* Adds native support for `time.Duration` marshalling ([details](https://github.com/go-mgo/mgo/pull/373)) +* Reduce memory footprint / garbage collection pressure by reusing buffers ([details](https://github.com/go-mgo/mgo/pull/229), [more](https://github.com/globalsign/mgo/pull/56)) +* Support majority read concerns ([details](https://github.com/globalsign/mgo/pull/2)) +* Improved connection handling ([details](https://github.com/globalsign/mgo/pull/5)) +* Hides SASL warnings ([details](https://github.com/globalsign/mgo/pull/7)) +* Support for partial indexes ([details](https://github.com/domodwyer/mgo/commit/5efe8eccb028238d93c222828cae4806aeae9f51)) +* Fixes timezone handling ([details](https://github.com/go-mgo/mgo/pull/464)) +* Integration tests run against MongoDB 3.2 & 3.4 releases ([details](https://github.com/globalsign/mgo/pull/4), [more](https://github.com/globalsign/mgo/pull/24), [more](https://github.com/globalsign/mgo/pull/35)) +* Improved multi-document transaction performance ([details](https://github.com/globalsign/mgo/pull/10), [more](https://github.com/globalsign/mgo/pull/11), [more](https://github.com/globalsign/mgo/pull/16)) +* Fixes cursor timeouts ([details](https://jira.mongodb.org/browse/SERVER-24899)) +* Support index hints and timeouts for count queries ([details](https://github.com/globalsign/mgo/pull/17)) +* Don't panic when handling indexed `int64` fields ([details](https://github.com/go-mgo/mgo/issues/475)) +* Supports dropping all indexes on a collection ([details](https://github.com/globalsign/mgo/pull/25)) +* Annotates log entries/profiler output with optional appName on 3.4+ ([details](https://github.com/globalsign/mgo/pull/28)) +* Support for read-only [views](https://docs.mongodb.com/manual/core/views/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/33)) +* Support for [collations](https://docs.mongodb.com/manual/reference/collation/) in 3.4+ ([details](https://github.com/globalsign/mgo/pull/37)) +* Provide BSON constants for convenience/sanity ([details](https://github.com/globalsign/mgo/pull/41)) +* Consistently unmarshal time.Time values as UTC ([details](https://github.com/globalsign/mgo/pull/42)) +* Enforces best practise coding guidelines ([details](https://github.com/globalsign/mgo/pull/44)) +* GetBSON correctly handles structs with both fields and pointers ([details](https://github.com/globalsign/mgo/pull/40)) +* Improved bson.Raw unmarshalling performance ([details](https://github.com/globalsign/mgo/pull/49)) +* Minimise socket connection timeouts due to excessive locking ([details](https://github.com/globalsign/mgo/pull/52)) +* Natively support X509 client authentication ([details](https://github.com/globalsign/mgo/pull/55)) +* Gracefully recover from a temporarily unreachable server ([details](https://github.com/globalsign/mgo/pull/69)) +* Use JSON tags when no explicit BSON are tags set ([details](https://github.com/globalsign/mgo/pull/91)) +* Support [$changeStream](https://docs.mongodb.com/manual/changeStreams/) tailing on 3.6+ ([details](https://github.com/globalsign/mgo/pull/97)) +* Fix deadlock in cluster synchronisation ([details](https://github.com/globalsign/mgo/issues/120)) +* Implement `maxIdleTimeout` for pooled connections ([details](https://github.com/globalsign/mgo/pull/116)) +* Connection pool waiting improvements ([details](https://github.com/globalsign/mgo/pull/115)) +* Fixes BSON encoding for `$in` and friends ([details](https://github.com/globalsign/mgo/pull/128)) +* Add BSON stream encoders ([details](https://github.com/globalsign/mgo/pull/127)) +* Add integer map key support in the BSON encoder ([details](https://github.com/globalsign/mgo/pull/140)) +* Support aggregation [collations](https://docs.mongodb.com/manual/reference/collation/) ([details](https://github.com/globalsign/mgo/pull/144)) + +--- + +### Thanks to +* @aksentyev +* @bachue +* @bozaro +* @BenLubar +* @carldunham +* @carter2000 +* @cezarsa +* @drichelson +* @dvic +* @eaglerayp +* @feliixx +* @fmpwizard +* @gazoon +* @gnawux +* @idy +* @jameinel +* @johnlawsharrison +* @KJTsanaktsidis +* @mapete94 +* @maxnoel +* @mcspring +* @peterdeka +* @Reenjii +* @smoya +* @steve-gray +* @wgallagher diff --git a/vendor/gopkg.in/mgo.v2/auth.go b/vendor/github.com/gedge/mgo/auth.go similarity index 99% rename from vendor/gopkg.in/mgo.v2/auth.go rename to vendor/github.com/gedge/mgo/auth.go index dc26e52f..7e961467 100644 --- a/vendor/gopkg.in/mgo.v2/auth.go +++ b/vendor/github.com/gedge/mgo/auth.go @@ -34,8 +34,8 @@ import ( "fmt" "sync" - "gopkg.in/mgo.v2/bson" - "gopkg.in/mgo.v2/internal/scram" + "github.com/gedge/mgo/bson" + "github.com/gedge/mgo/internal/scram" ) type authCmd struct { @@ -61,7 +61,7 @@ type getNonceCmd struct { type getNonceResult struct { Nonce string - Err string "$err" + Err string `bson:"$err"` Code int } diff --git a/vendor/gopkg.in/mgo.v2/bson/LICENSE b/vendor/github.com/gedge/mgo/bson/LICENSE similarity index 100% rename from vendor/gopkg.in/mgo.v2/bson/LICENSE rename to vendor/github.com/gedge/mgo/bson/LICENSE diff --git a/vendor/github.com/gedge/mgo/bson/README.md b/vendor/github.com/gedge/mgo/bson/README.md new file mode 100644 index 00000000..5c5819e6 --- /dev/null +++ b/vendor/github.com/gedge/mgo/bson/README.md @@ -0,0 +1,12 @@ +[![GoDoc](https://godoc.org/github.com/globalsign/mgo/bson?status.svg)](https://godoc.org/github.com/globalsign/mgo/bson) + +An Implementation of BSON for Go +-------------------------------- + +Package bson is an implementation of the [BSON specification](http://bsonspec.org) for Go. + +While the BSON package implements the BSON spec as faithfully as possible, there +is some MongoDB specific behaviour (such as map keys `$in`, `$all`, etc) in the +`bson` package. The priority is for backwards compatibility for the `mgo` +driver, though fixes for obviously buggy behaviour is welcome (and features, etc +behind feature flags). diff --git a/vendor/gopkg.in/mgo.v2/bson/bson.go b/vendor/github.com/gedge/mgo/bson/bson.go similarity index 84% rename from vendor/gopkg.in/mgo.v2/bson/bson.go rename to vendor/github.com/gedge/mgo/bson/bson.go index 7fb7f8ca..2577dbd1 100644 --- a/vendor/gopkg.in/mgo.v2/bson/bson.go +++ b/vendor/github.com/gedge/mgo/bson/bson.go @@ -42,6 +42,7 @@ import ( "errors" "fmt" "io" + "math" "os" "reflect" "runtime" @@ -51,10 +52,45 @@ import ( "time" ) +//go:generate go run bson_corpus_spec_test_generator.go + // -------------------------------------------------------------------------- // The public API. -// A value implementing the bson.Getter interface will have its GetBSON +// Element types constants from BSON specification. +const ( + ElementFloat64 byte = 0x01 + ElementString byte = 0x02 + ElementDocument byte = 0x03 + ElementArray byte = 0x04 + ElementBinary byte = 0x05 + Element06 byte = 0x06 + ElementObjectId byte = 0x07 + ElementBool byte = 0x08 + ElementDatetime byte = 0x09 + ElementNil byte = 0x0A + ElementRegEx byte = 0x0B + ElementDBPointer byte = 0x0C + ElementJavaScriptWithoutScope byte = 0x0D + ElementSymbol byte = 0x0E + ElementJavaScriptWithScope byte = 0x0F + ElementInt32 byte = 0x10 + ElementTimestamp byte = 0x11 + ElementInt64 byte = 0x12 + ElementDecimal128 byte = 0x13 + ElementMinKey byte = 0xFF + ElementMaxKey byte = 0x7F + + BinaryGeneric byte = 0x00 + BinaryFunction byte = 0x01 + BinaryBinaryOld byte = 0x02 + BinaryUUIDOld byte = 0x03 + BinaryUUID byte = 0x04 + BinaryMD5 byte = 0x05 + BinaryUserDefined byte = 0x80 +) + +// Getter interface: a value implementing the bson.Getter interface will have its GetBSON // method called when the given value has to be marshalled, and the result // of this method will be marshaled in place of the actual object. // @@ -64,12 +100,12 @@ type Getter interface { GetBSON() (interface{}, error) } -// A value implementing the bson.Setter interface will receive the BSON +// Setter interface: a value implementing the bson.Setter interface will receive the BSON // value via the SetBSON method during unmarshaling, and the object // itself will not be changed as usual. // // If setting the value works, the method should return nil or alternatively -// bson.SetZero to set the respective field to its zero value (nil for +// bson.ErrSetZero to set the respective field to its zero value (nil for // pointer types). If SetBSON returns a value of type bson.TypeError, the // BSON value will be omitted from a map or slice being decoded and the // unmarshalling will continue. If it returns any other non-nil error, the @@ -95,10 +131,10 @@ type Setter interface { SetBSON(raw Raw) error } -// SetZero may be returned from a SetBSON method to have the value set to +// ErrSetZero may be returned from a SetBSON method to have the value set to // its respective zero value. When used in pointer values, this will set the // field to nil rather than to the pre-allocated value. -var SetZero = errors.New("set to zero") +var ErrSetZero = errors.New("set to zero") // M is a convenient alias for a map[string]interface{} map, useful for // dealing with BSON in a native way. For instance: @@ -154,7 +190,7 @@ type Raw struct { // documents in general. type RawD []RawDocElem -// See the RawD type. +// RawDocElem elements of RawD type. type RawDocElem struct { Name string Value Raw @@ -164,7 +200,7 @@ type RawDocElem struct { // long. MongoDB objects by default have such a property set in their "_id" // property. // -// http://www.mongodb.org/display/DOCS/Object+IDs +// http://www.mongodb.org/display/DOCS/Object+Ids type ObjectId string // ObjectIdHex returns an ObjectId from the provided hex representation. @@ -190,7 +226,7 @@ func IsObjectIdHex(s string) bool { // objectIdCounter is atomically incremented when generating a new ObjectId // using NewObjectId() function. It's used as a counter part of an id. -var objectIdCounter uint32 = readRandomUint32() +var objectIdCounter = readRandomUint32() // readRandomUint32 returns a random objectIdCounter. func readRandomUint32() uint32 { @@ -279,7 +315,7 @@ var nullBytes = []byte("null") func (id *ObjectId) UnmarshalJSON(data []byte) error { if len(data) > 0 && (data[0] == '{' || data[0] == 'O') { var v struct { - Id json.RawMessage `json:"$oid"` + Id json.RawMessage `json:"$oid"` Func struct { Id json.RawMessage } `json:"$oidFunc"` @@ -298,12 +334,12 @@ func (id *ObjectId) UnmarshalJSON(data []byte) error { return nil } if len(data) != 26 || data[0] != '"' || data[25] != '"' { - return errors.New(fmt.Sprintf("invalid ObjectId in JSON: %s", string(data))) + return fmt.Errorf("invalid ObjectId in JSON: %s", string(data)) } var buf [12]byte _, err := hex.Decode(buf[:], data[1:25]) if err != nil { - return errors.New(fmt.Sprintf("invalid ObjectId in JSON: %s (%s)", string(data), err)) + return fmt.Errorf("invalid ObjectId in JSON: %s (%s)", string(data), err) } *id = ObjectId(string(buf[:])) return nil @@ -391,6 +427,36 @@ func Now() time.Time { // strange reason has its own datatype defined in BSON. type MongoTimestamp int64 +// Time returns the time part of ts which is stored with second precision. +func (ts MongoTimestamp) Time() time.Time { + return time.Unix(int64(uint64(ts)>>32), 0) +} + +// Counter returns the counter part of ts. +func (ts MongoTimestamp) Counter() uint32 { + return uint32(ts) +} + +// NewMongoTimestamp creates a timestamp using the given +// date `t` (with second precision) and counter `c` (unique for `t`). +// +// Returns an error if time `t` is not between 1970-01-01T00:00:00Z +// and 2106-02-07T06:28:15Z (inclusive). +// +// Note that two MongoTimestamps should never have the same (time, counter) combination: +// the caller must ensure the counter `c` is increased if creating multiple MongoTimestamp +// values for the same time `t` (ignoring fractions of seconds). +func NewMongoTimestamp(t time.Time, c uint32) (MongoTimestamp, error) { + u := t.Unix() + if u < 0 || u > math.MaxUint32 { + return -1, errors.New("invalid value for time") + } + + i := int64(u<<32 | int64(c)) + + return MongoTimestamp(i), nil +} + type orderKey int64 // MaxKey is a special value that compares higher than all other possible BSON @@ -506,8 +572,15 @@ func handleErr(err *error) { // } // func Marshal(in interface{}) (out []byte, err error) { + return MarshalBuffer(in, make([]byte, 0, initialBufferSize)) +} + +// MarshalBuffer behaves the same way as Marshal, except that instead of +// allocating a new byte slice it tries to use the received byte slice and +// only allocates more memory if necessary to fit the marshaled value. +func MarshalBuffer(in interface{}, buf []byte) (out []byte, err error) { defer handleErr(&err) - e := &encoder{make([]byte, 0, initialBufferSize)} + e := &encoder{buf} e.addDoc(reflect.ValueOf(in)) return e.out, nil } @@ -561,10 +634,13 @@ func Unmarshal(in []byte, out interface{}) (err error) { case reflect.Map: d := newDecoder(in) d.readDocTo(v) + if d.i < len(d.in) { + return errors.New("document is corrupted") + } case reflect.Struct: - return errors.New("Unmarshal can't deal with struct values. Use a pointer.") + return errors.New("unmarshal can't deal with struct values. Use a pointer") default: - return errors.New("Unmarshal needs a map or a pointer to a struct.") + return errors.New("unmarshal needs a map or a pointer to a struct") } return nil } @@ -588,13 +664,15 @@ func (raw Raw) Unmarshal(out interface{}) (err error) { return &TypeError{v.Type(), raw.Kind} } case reflect.Struct: - return errors.New("Raw Unmarshal can't deal with struct values. Use a pointer.") + return errors.New("raw Unmarshal can't deal with struct values. Use a pointer") default: - return errors.New("Raw Unmarshal needs a map or a valid pointer.") + return errors.New("raw Unmarshal needs a map or a valid pointer") } return nil } +// TypeError store details for type error occuring +// during unmarshaling type TypeError struct { Type reflect.Type Kind byte @@ -651,9 +729,21 @@ func getStructInfo(st reflect.Type) (*structInfo, error) { info := fieldInfo{Num: i} tag := field.Tag.Get("bson") - if tag == "" && strings.Index(string(field.Tag), ":") < 0 { - tag = string(field.Tag) + + // Fall-back to JSON struct tag, if feature flag is set. + if tag == "" && useJSONTagFallback { + tag = field.Tag.Get("json") } + + // If there's no bson/json tag available. + if tag == "" { + // If there's no tag, and also no tag: value splits (i.e. no colon) + // then assume the entire tag is the value + if strings.Index(string(field.Tag), ":") < 0 { + tag = string(field.Tag) + } + } + if tag == "-" { continue } diff --git a/vendor/github.com/gedge/mgo/bson/bson_corpus_spec_test_generator.go b/vendor/github.com/gedge/mgo/bson/bson_corpus_spec_test_generator.go new file mode 100644 index 00000000..b4eabe83 --- /dev/null +++ b/vendor/github.com/gedge/mgo/bson/bson_corpus_spec_test_generator.go @@ -0,0 +1,294 @@ +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "go/format" + "html/template" + "io/ioutil" + "log" + "path/filepath" + "strings" + + "github.com/gedge/mgo/internal/json" +) + +func main() { + log.SetFlags(0) + log.SetPrefix(name + ": ") + + var g Generator + + fmt.Fprintf(&g, "// Code generated by \"%s.go\"; DO NOT EDIT\n\n", name) + + src := g.generate() + + err := ioutil.WriteFile(fmt.Sprintf("%s.go", strings.TrimSuffix(name, "_generator")), src, 0644) + if err != nil { + log.Fatalf("writing output: %s", err) + } +} + +// Generator holds the state of the analysis. Primarily used to buffer +// the output for format.Source. +type Generator struct { + bytes.Buffer // Accumulated output. +} + +// format returns the gofmt-ed contents of the Generator's buffer. +func (g *Generator) format() []byte { + src, err := format.Source(g.Bytes()) + if err != nil { + // Should never happen, but can arise when developing this code. + // The user can compile the output to see the error. + log.Printf("warning: internal error: invalid Go generated: %s", err) + log.Printf("warning: compile the package to analyze the error") + return g.Bytes() + } + return src +} + +// EVERYTHING ABOVE IS CONSTANT BETWEEN THE GENERATORS + +const name = "bson_corpus_spec_test_generator" + +func (g *Generator) generate() []byte { + + testFiles, err := filepath.Glob("./specdata/specifications/source/bson-corpus/tests/*.json") + if err != nil { + log.Fatalf("error reading bson-corpus files: %s", err) + } + + tests, err := g.loadTests(testFiles) + if err != nil { + log.Fatalf("error loading tests: %s", err) + } + + tmpl, err := g.getTemplate() + if err != nil { + log.Fatalf("error loading template: %s", err) + } + + tmpl.Execute(&g.Buffer, tests) + + return g.format() +} + +func (g *Generator) loadTests(filenames []string) ([]*testDef, error) { + var tests []*testDef + for _, filename := range filenames { + test, err := g.loadTest(filename) + if err != nil { + return nil, err + } + + tests = append(tests, test) + } + + return tests, nil +} + +func (g *Generator) loadTest(filename string) (*testDef, error) { + content, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + var testDef testDef + err = json.Unmarshal(content, &testDef) + if err != nil { + return nil, err + } + + names := make(map[string]struct{}) + + for i := len(testDef.Valid) - 1; i >= 0; i-- { + if testDef.BsonType == "0x05" && testDef.Valid[i].Description == "subtype 0x02" { + testDef.Valid = append(testDef.Valid[:i], testDef.Valid[i+1:]...) + continue + } + + name := cleanupFuncName(testDef.Description + "_" + testDef.Valid[i].Description) + nameIdx := name + j := 1 + for { + if _, ok := names[nameIdx]; !ok { + break + } + + nameIdx = fmt.Sprintf("%s_%d", name, j) + } + + names[nameIdx] = struct{}{} + + testDef.Valid[i].TestDef = &testDef + testDef.Valid[i].Name = nameIdx + testDef.Valid[i].StructTest = testDef.TestKey != "" && + (testDef.BsonType != "0x05" || strings.Contains(testDef.Valid[i].Description, "0x00")) && + !testDef.Deprecated + } + + for i := len(testDef.DecodeErrors) - 1; i >= 0; i-- { + if strings.Contains(testDef.DecodeErrors[i].Description, "UTF-8") { + testDef.DecodeErrors = append(testDef.DecodeErrors[:i], testDef.DecodeErrors[i+1:]...) + continue + } + + name := cleanupFuncName(testDef.Description + "_" + testDef.DecodeErrors[i].Description) + nameIdx := name + j := 1 + for { + if _, ok := names[nameIdx]; !ok { + break + } + + nameIdx = fmt.Sprintf("%s_%d", name, j) + } + names[nameIdx] = struct{}{} + + testDef.DecodeErrors[i].Name = nameIdx + } + + return &testDef, nil +} + +func (g *Generator) getTemplate() (*template.Template, error) { + content := `package bson_test + +import ( + "encoding/hex" + "time" + + . "gopkg.in/check.v1" + "github.com/gedge/mgo/bson" +) + +func testValid(c *C, in []byte, expected []byte, result interface{}) { + err := bson.Unmarshal(in, result) + c.Assert(err, IsNil) + + out, err := bson.Marshal(result) + c.Assert(err, IsNil) + + c.Assert(string(expected), Equals, string(out), Commentf("roundtrip failed for %T, expected '%x' but got '%x'", result, expected, out)) +} + +func testDecodeSkip(c *C, in []byte) { + err := bson.Unmarshal(in, &struct{}{}) + c.Assert(err, IsNil) +} + +func testDecodeError(c *C, in []byte, result interface{}) { + err := bson.Unmarshal(in, result) + c.Assert(err, Not(IsNil)) +} + +{{range .}} +{{range .Valid}} +func (s *S) Test{{.Name}}(c *C) { + b, err := hex.DecodeString("{{.Bson}}") + c.Assert(err, IsNil) + + {{if .CanonicalBson}} + cb, err := hex.DecodeString("{{.CanonicalBson}}") + c.Assert(err, IsNil) + {{else}} + cb := b + {{end}} + + var resultD bson.D + testValid(c, b, cb, &resultD) + {{if .StructTest}}var resultS struct { + Element {{.TestDef.GoType}} ` + "`bson:\"{{.TestDef.TestKey}}\"`" + ` + } + testValid(c, b, cb, &resultS){{end}} + + testDecodeSkip(c, b) +} +{{end}} + +{{range .DecodeErrors}} +func (s *S) Test{{.Name}}(c *C) { + b, err := hex.DecodeString("{{.Bson}}") + c.Assert(err, IsNil) + + var resultD bson.D + testDecodeError(c, b, &resultD) +} +{{end}} +{{end}} +` + tmpl, err := template.New("").Parse(content) + if err != nil { + return nil, err + } + return tmpl, nil +} + +func cleanupFuncName(name string) string { + return strings.Map(func(r rune) rune { + if (r >= 48 && r <= 57) || (r >= 65 && r <= 90) || (r >= 97 && r <= 122) { + return r + } + return '_' + }, name) +} + +type testDef struct { + Description string `json:"description"` + BsonType string `json:"bson_type"` + TestKey string `json:"test_key"` + Valid []*valid `json:"valid"` + DecodeErrors []*decodeError `json:"decodeErrors"` + Deprecated bool `json:"deprecated"` +} + +func (t *testDef) GoType() string { + switch t.BsonType { + case "0x01": + return "float64" + case "0x02": + return "string" + case "0x03": + return "bson.D" + case "0x04": + return "[]interface{}" + case "0x05": + return "[]byte" + case "0x07": + return "bson.ObjectId" + case "0x08": + return "bool" + case "0x09": + return "time.Time" + case "0x0E": + return "string" + case "0x10": + return "int32" + case "0x12": + return "int64" + case "0x13": + return "bson.Decimal" + default: + return "interface{}" + } +} + +type valid struct { + Description string `json:"description"` + Bson string `json:"bson"` + CanonicalBson string `json:"canonical_bson"` + + Name string + StructTest bool + TestDef *testDef +} + +type decodeError struct { + Description string `json:"description"` + Bson string `json:"bson"` + + Name string +} diff --git a/vendor/github.com/gedge/mgo/bson/compatibility.go b/vendor/github.com/gedge/mgo/bson/compatibility.go new file mode 100644 index 00000000..6afecf53 --- /dev/null +++ b/vendor/github.com/gedge/mgo/bson/compatibility.go @@ -0,0 +1,16 @@ +package bson + +// Current state of the JSON tag fallback option. +var useJSONTagFallback = false + +// SetJSONTagFallback enables or disables the JSON-tag fallback for structure tagging. When this is enabled, structures +// without BSON tags on a field will fall-back to using the JSON tag (if present). +func SetJSONTagFallback(state bool) { + useJSONTagFallback = state +} + +// JSONTagFallbackState returns the current status of the JSON tag fallback compatability option. See SetJSONTagFallback +// for more information. +func JSONTagFallbackState() bool { + return useJSONTagFallback +} diff --git a/vendor/gopkg.in/mgo.v2/bson/decimal.go b/vendor/github.com/gedge/mgo/bson/decimal.go similarity index 98% rename from vendor/gopkg.in/mgo.v2/bson/decimal.go rename to vendor/github.com/gedge/mgo/bson/decimal.go index 3d2f7002..672ba182 100644 --- a/vendor/gopkg.in/mgo.v2/bson/decimal.go +++ b/vendor/github.com/gedge/mgo/bson/decimal.go @@ -144,6 +144,8 @@ func dErr(s string) (Decimal128, error) { return dNaN, fmt.Errorf("cannot parse %q as a decimal128", s) } +// ParseDecimal128 parse a string and return the corresponding value as +// a decimal128 func ParseDecimal128(s string) (Decimal128, error) { orig := s if s == "" { diff --git a/vendor/gopkg.in/mgo.v2/bson/decode.go b/vendor/github.com/gedge/mgo/bson/decode.go similarity index 68% rename from vendor/gopkg.in/mgo.v2/bson/decode.go rename to vendor/github.com/gedge/mgo/bson/decode.go index 7c2d8416..658856ad 100644 --- a/vendor/gopkg.in/mgo.v2/bson/decode.go +++ b/vendor/github.com/gedge/mgo/bson/decode.go @@ -28,7 +28,9 @@ package bson import ( + "errors" "fmt" + "io" "math" "net/url" "reflect" @@ -56,13 +58,6 @@ func corrupted() { panic("Document is corrupted") } -func settableValueOf(i interface{}) reflect.Value { - v := reflect.ValueOf(i) - sv := reflect.New(v.Type()).Elem() - sv.Set(v) - return sv -} - // -------------------------------------------------------------------------- // Unmarshaling of documents. @@ -87,18 +82,20 @@ func setterStyle(outt reflect.Type) int { setterMutex.RLock() style := setterStyles[outt] setterMutex.RUnlock() - if style == setterUnknown { - setterMutex.Lock() - defer setterMutex.Unlock() - if outt.Implements(setterIface) { - setterStyles[outt] = setterType - } else if reflect.PtrTo(outt).Implements(setterIface) { - setterStyles[outt] = setterAddr - } else { - setterStyles[outt] = setterNone - } - style = setterStyles[outt] + if style != setterUnknown { + return style + } + + setterMutex.Lock() + defer setterMutex.Unlock() + if outt.Implements(setterIface) { + style = setterType + } else if reflect.PtrTo(outt).Implements(setterIface) { + style = setterAddr + } else { + style = setterNone } + setterStyles[outt] = style return style } @@ -135,8 +132,7 @@ func (d *decoder) readDocTo(out reflect.Value) { out.Set(reflect.New(outt.Elem())) } if setter := getSetter(outt, out); setter != nil { - var raw Raw - d.readDocTo(reflect.ValueOf(&raw)) + raw := d.readRaw(ElementDocument) err := setter.SetBSON(raw) if _, ok := err.(*TypeError); err != nil && !ok { panic(err) @@ -154,7 +150,10 @@ func (d *decoder) readDocTo(out reflect.Value) { var fieldsMap map[string]fieldInfo var inlineMap reflect.Value - start := d.i + if outt == typeRaw { + out.Set(reflect.ValueOf(d.readRaw(ElementDocument))) + return + } origout := out if outk == reflect.Interface { @@ -177,9 +176,6 @@ func (d *decoder) readDocTo(out reflect.Value) { switch outk { case reflect.Map: keyType = outt.Key() - if keyType.Kind() != reflect.String { - panic("BSON map must have string keys. Got: " + outt.String()) - } if keyType != typeString { convertKey = true } @@ -193,22 +189,20 @@ func (d *decoder) readDocTo(out reflect.Value) { clearMap(out) } case reflect.Struct: - if outt != typeRaw { - sinfo, err := getStructInfo(out.Type()) - if err != nil { - panic(err) + sinfo, err := getStructInfo(out.Type()) + if err != nil { + panic(err) + } + fieldsMap = sinfo.FieldsMap + out.Set(sinfo.Zero) + if sinfo.InlineMap != -1 { + inlineMap = out.Field(sinfo.InlineMap) + if !inlineMap.IsNil() && inlineMap.Len() > 0 { + clearMap(inlineMap) } - fieldsMap = sinfo.FieldsMap - out.Set(sinfo.Zero) - if sinfo.InlineMap != -1 { - inlineMap = out.Field(sinfo.InlineMap) - if !inlineMap.IsNil() && inlineMap.Len() > 0 { - clearMap(inlineMap) - } - elemType = inlineMap.Type().Elem() - if elemType == typeIface { - d.docType = inlineMap.Type() - } + elemType = inlineMap.Type().Elem() + if elemType == typeIface { + d.docType = inlineMap.Type() } } case reflect.Slice: @@ -243,31 +237,62 @@ func (d *decoder) readDocTo(out reflect.Value) { if d.readElemTo(e, kind) { k := reflect.ValueOf(name) if convertKey { - k = k.Convert(keyType) + mapKeyType := out.Type().Key() + mapKeyKind := mapKeyType.Kind() + + switch mapKeyKind { + case reflect.Int: + fallthrough + case reflect.Int8: + fallthrough + case reflect.Int16: + fallthrough + case reflect.Int32: + fallthrough + case reflect.Int64: + fallthrough + case reflect.Uint: + fallthrough + case reflect.Uint8: + fallthrough + case reflect.Uint16: + fallthrough + case reflect.Uint32: + fallthrough + case reflect.Uint64: + fallthrough + case reflect.Float32: + fallthrough + case reflect.Float64: + parsed := d.parseMapKeyAsFloat(k, mapKeyKind) + k = reflect.ValueOf(parsed) + case reflect.String: + mapKeyType = keyType + default: + panic("BSON map must have string or decimal keys. Got: " + outt.String()) + } + + k = k.Convert(mapKeyType) } out.SetMapIndex(k, e) } case reflect.Struct: - if outt == typeRaw { - d.dropElem(kind) - } else { - if info, ok := fieldsMap[name]; ok { - if info.Inline == nil { - d.readElemTo(out.Field(info.Num), kind) - } else { - d.readElemTo(out.FieldByIndex(info.Inline), kind) - } - } else if inlineMap.IsValid() { - if inlineMap.IsNil() { - inlineMap.Set(reflect.MakeMap(inlineMap.Type())) - } - e := reflect.New(elemType).Elem() - if d.readElemTo(e, kind) { - inlineMap.SetMapIndex(reflect.ValueOf(name), e) - } + if info, ok := fieldsMap[name]; ok { + if info.Inline == nil { + d.readElemTo(out.Field(info.Num), kind) } else { - d.dropElem(kind) + d.readElemTo(out.FieldByIndex(info.Inline), kind) } + } else if inlineMap.IsValid() { + if inlineMap.IsNil() { + inlineMap.Set(reflect.MakeMap(inlineMap.Type())) + } + e := reflect.New(elemType).Elem() + if d.readElemTo(e, kind) { + inlineMap.SetMapIndex(reflect.ValueOf(name), e) + } + } else { + d.dropElem(kind) } case reflect.Slice: } @@ -281,10 +306,16 @@ func (d *decoder) readDocTo(out reflect.Value) { corrupted() } d.docType = docType +} - if outt == typeRaw { - out.Set(reflect.ValueOf(Raw{0x03, d.in[start:d.i]})) +func (decoder) parseMapKeyAsFloat(k reflect.Value, mapKeyKind reflect.Kind) float64 { + parsed, err := strconv.ParseFloat(k.String(), 64) + if err != nil { + panic("Map key is defined to be a decimal type (" + mapKeyKind.String() + ") but got error " + + err.Error()) } + + return parsed } func (d *decoder) readArrayDocTo(out reflect.Value) { @@ -326,9 +357,12 @@ func (d *decoder) readSliceDoc(t reflect.Type) interface{} { tmp := make([]reflect.Value, 0, 8) elemType := t.Elem() if elemType == typeRawDocElem { - d.dropElem(0x04) + d.dropElem(ElementArray) return reflect.Zero(t).Interface() } + if elemType == typeRaw { + return d.readSliceOfRaw() + } end := int(d.readInt32()) end += d.i - 4 @@ -365,6 +399,151 @@ func (d *decoder) readSliceDoc(t reflect.Type) interface{} { return slice.Interface() } +func BSONElementSize(kind byte, offset int, buffer []byte) (int, error) { + switch kind { + case ElementFloat64: // Float64 + return 8, nil + case ElementJavaScriptWithoutScope: // JavaScript without scope + fallthrough + case ElementSymbol: // Symbol + fallthrough + case ElementString: // UTF-8 string + size, err := getSize(offset, buffer) + if err != nil { + return 0, err + } + if size < 1 { + return 0, errors.New("String size can't be less then one byte") + } + size += 4 + if offset+size > len(buffer) { + return 0, io.ErrUnexpectedEOF + } + if buffer[offset+size-1] != 0 { + return 0, errors.New("Invalid string: non zero-terminated") + } + return size, nil + case ElementArray: // Array + fallthrough + case ElementDocument: // Document + size, err := getSize(offset, buffer) + if err != nil { + return 0, err + } + if size < 5 { + return 0, errors.New("Declared document size is too small") + } + return size, nil + case ElementBinary: // Binary + size, err := getSize(offset, buffer) + if err != nil { + return 0, err + } + if size < 0 { + return 0, errors.New("Binary data size can't be negative") + } + return size + 5, nil + case Element06: // Undefined (obsolete, but still seen in the wild) + return 0, nil + case ElementObjectId: // ObjectId + return 12, nil + case ElementBool: // Bool + return 1, nil + case ElementDatetime: // Timestamp + return 8, nil + case ElementNil: // Nil + return 0, nil + case ElementRegEx: // RegEx + end := offset + for i := 0; i < 2; i++ { + for end < len(buffer) && buffer[end] != '\x00' { + end++ + } + end++ + } + if end > len(buffer) { + return 0, io.ErrUnexpectedEOF + } + return end - offset, nil + case ElementDBPointer: // DBPointer + size, err := getSize(offset, buffer) + if err != nil { + return 0, err + } + if size < 1 { + return 0, errors.New("String size can't be less then one byte") + } + return size + 12 + 4, nil + case ElementJavaScriptWithScope: // JavaScript with scope + size, err := getSize(offset, buffer) + if err != nil { + return 0, err + } + if size < 4+5+5 { + return 0, errors.New("Declared document element is too small") + } + return size, nil + case ElementInt32: // Int32 + return 4, nil + case ElementTimestamp: // Mongo-specific timestamp + return 8, nil + case ElementInt64: // Int64 + return 8, nil + case ElementDecimal128: // Decimal128 + return 16, nil + case ElementMaxKey: // Max key + return 0, nil + case ElementMinKey: // Min key + return 0, nil + default: + return 0, errors.New(fmt.Sprintf("Unknown element kind (0x%02X)", kind)) + } +} + +func (d *decoder) readRaw(kind byte) Raw { + size, err := BSONElementSize(kind, d.i, d.in) + if err != nil { + corrupted() + } + if d.i+size > len(d.in) { + corrupted() + } + d.i += size + return Raw{ + Kind: kind, + Data: d.in[d.i-size : d.i], + } +} + +func (d *decoder) readSliceOfRaw() interface{} { + tmp := make([]Raw, 0, 8) + end := int(d.readInt32()) + end += d.i - 4 + if end <= d.i || end > len(d.in) || d.in[end-1] != '\x00' { + corrupted() + } + for d.in[d.i] != '\x00' { + kind := d.readByte() + for d.i < end && d.in[d.i] != '\x00' { + d.i++ + } + if d.i >= end { + corrupted() + } + d.i++ + e := d.readRaw(kind) + tmp = append(tmp, e) + if d.i >= end { + corrupted() + } + } + d.i++ // '\x00' + if d.i != end { + corrupted() + } + return tmp +} + var typeSlice = reflect.TypeOf([]interface{}{}) var typeIface = typeSlice.Elem() @@ -390,11 +569,8 @@ func (d *decoder) readRawDocElems(typ reflect.Type) reflect.Value { d.docType = typ slice := make([]RawDocElem, 0, 8) d.readDocWith(func(kind byte, name string) { - e := RawDocElem{Name: name} - v := reflect.ValueOf(&e.Value) - if d.readElemTo(v.Elem(), kind) { - slice = append(slice, e) - } + e := RawDocElem{Name: name, Value: d.readRaw(kind)} + slice = append(slice, e) }) slicev := reflect.New(typ).Elem() slicev.Set(reflect.ValueOf(slice)) @@ -427,21 +603,35 @@ func (d *decoder) readDocWith(f func(kind byte, name string)) { // -------------------------------------------------------------------------- // Unmarshaling of individual elements within a document. - -var blackHole = settableValueOf(struct{}{}) - func (d *decoder) dropElem(kind byte) { - d.readElemTo(blackHole, kind) + size, err := BSONElementSize(kind, d.i, d.in) + if err != nil { + corrupted() + } + if d.i+size > len(d.in) { + corrupted() + } + d.i += size } // Attempt to decode an element from the document and put it into out. // If the types are not compatible, the returned ok value will be // false and out will be unchanged. func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { + outt := out.Type() - start := d.i + if outt == typeRaw { + out.Set(reflect.ValueOf(d.readRaw(kind))) + return true + } + + if outt == typeRawPtr { + raw := d.readRaw(kind) + out.Set(reflect.ValueOf(&raw)) + return true + } - if kind == 0x03 { + if kind == ElementDocument { // Delegate unmarshaling of documents. outt := out.Type() outk := out.Kind() @@ -461,24 +651,39 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { case typeRawDocElem: out.Set(d.readRawDocElems(outt)) default: - d.readDocTo(blackHole) + d.dropElem(kind) } return true } - d.readDocTo(blackHole) + d.dropElem(kind) return true } + if setter := getSetter(outt, out); setter != nil { + err := setter.SetBSON(d.readRaw(kind)) + if err == ErrSetZero { + out.Set(reflect.Zero(outt)) + return true + } + if err == nil { + return true + } + if _, ok := err.(*TypeError); !ok { + panic(err) + } + return false + } + var in interface{} switch kind { - case 0x01: // Float64 + case ElementFloat64: in = d.readFloat64() - case 0x02: // UTF-8 string + case ElementString: in = d.readStr() - case 0x03: // Document + case ElementDocument: panic("Can't happen. Handled above.") - case 0x04: // Array + case ElementArray: outt := out.Type() if setterStyle(outt) != setterNone { // Skip the value so its data is handed to the setter below. @@ -497,83 +702,70 @@ func (d *decoder) readElemTo(out reflect.Value, kind byte) (good bool) { default: in = d.readSliceDoc(typeSlice) } - case 0x05: // Binary + case ElementBinary: b := d.readBinary() - if b.Kind == 0x00 || b.Kind == 0x02 { + if b.Kind == BinaryGeneric || b.Kind == BinaryBinaryOld { in = b.Data } else { in = b } - case 0x06: // Undefined (obsolete, but still seen in the wild) + case Element06: // Undefined (obsolete, but still seen in the wild) in = Undefined - case 0x07: // ObjectId + case ElementObjectId: in = ObjectId(d.readBytes(12)) - case 0x08: // Bool + case ElementBool: in = d.readBool() - case 0x09: // Timestamp + case ElementDatetime: // Timestamp // MongoDB handles timestamps as milliseconds. i := d.readInt64() if i == -62135596800000 { in = time.Time{} // In UTC for convenience. } else { - in = time.Unix(i/1e3, i%1e3*1e6) + in = time.Unix(i/1e3, i%1e3*1e6).UTC() } - case 0x0A: // Nil + case ElementNil: in = nil - case 0x0B: // RegEx + case ElementRegEx: in = d.readRegEx() - case 0x0C: + case ElementDBPointer: in = DBPointer{Namespace: d.readStr(), Id: ObjectId(d.readBytes(12))} - case 0x0D: // JavaScript without scope + case ElementJavaScriptWithoutScope: in = JavaScript{Code: d.readStr()} - case 0x0E: // Symbol + case ElementSymbol: in = Symbol(d.readStr()) - case 0x0F: // JavaScript with scope - d.i += 4 // Skip length + case ElementJavaScriptWithScope: + start := d.i + l := int(d.readInt32()) js := JavaScript{d.readStr(), make(M)} d.readDocTo(reflect.ValueOf(js.Scope)) + if d.i != start+l { + corrupted() + } in = js - case 0x10: // Int32 + case ElementInt32: in = int(d.readInt32()) - case 0x11: // Mongo-specific timestamp + case ElementTimestamp: // Mongo-specific timestamp in = MongoTimestamp(d.readInt64()) - case 0x12: // Int64 - in = d.readInt64() - case 0x13: // Decimal128 + case ElementInt64: + switch out.Type() { + case typeTimeDuration: + in = time.Duration(time.Duration(d.readInt64()) * time.Millisecond) + default: + in = d.readInt64() + } + case ElementDecimal128: in = Decimal128{ l: uint64(d.readInt64()), h: uint64(d.readInt64()), } - case 0x7F: // Max key + case ElementMaxKey: in = MaxKey - case 0xFF: // Min key + case ElementMinKey: in = MinKey default: panic(fmt.Sprintf("Unknown element kind (0x%02X)", kind)) } - outt := out.Type() - - if outt == typeRaw { - out.Set(reflect.ValueOf(Raw{kind, d.in[start:d.i]})) - return true - } - - if setter := getSetter(outt, out); setter != nil { - err := setter.SetBSON(Raw{kind, d.in[start:d.i]}) - if err == SetZero { - out.Set(reflect.Zero(outt)) - return true - } - if err == nil { - return true - } - if _, ok := err.(*TypeError); !ok { - panic(err) - } - return false - } - if in == nil { out.Set(reflect.Zero(outt)) return true @@ -759,11 +951,15 @@ func (d *decoder) readBinary() Binary { l := d.readInt32() b := Binary{} b.Kind = d.readByte() - b.Data = d.readBytes(l) - if b.Kind == 0x02 && len(b.Data) >= 4 { + if b.Kind == BinaryBinaryOld && l > 4 { // Weird obsolete format with redundant length. - b.Data = b.Data[4:] + rl := d.readInt32() + if rl != l-4 { + corrupted() + } + l = rl } + b.Data = d.readBytes(l) return b } @@ -815,6 +1011,16 @@ func (d *decoder) readInt32() int32 { (uint32(b[3]) << 24)) } +func getSize(offset int, b []byte) (int, error) { + if offset+4 > len(b) { + return 0, io.ErrUnexpectedEOF + } + return int((uint32(b[offset]) << 0) | + (uint32(b[offset+1]) << 8) | + (uint32(b[offset+2]) << 16) | + (uint32(b[offset+3]) << 24)), nil +} + func (d *decoder) readInt64() int64 { b := d.readBytes(8) return int64((uint64(b[0]) << 0) | diff --git a/vendor/gopkg.in/mgo.v2/bson/encode.go b/vendor/github.com/gedge/mgo/bson/encode.go similarity index 82% rename from vendor/gopkg.in/mgo.v2/bson/encode.go rename to vendor/github.com/gedge/mgo/bson/encode.go index add39e86..7e0b84d7 100644 --- a/vendor/gopkg.in/mgo.v2/bson/encode.go +++ b/vendor/github.com/gedge/mgo/bson/encode.go @@ -33,7 +33,9 @@ import ( "math" "net/url" "reflect" + "sort" "strconv" + "sync" "time" ) @@ -50,21 +52,47 @@ var ( typeDocElem = reflect.TypeOf(DocElem{}) typeRawDocElem = reflect.TypeOf(RawDocElem{}) typeRaw = reflect.TypeOf(Raw{}) + typeRawPtr = reflect.PtrTo(reflect.TypeOf(Raw{})) typeURL = reflect.TypeOf(url.URL{}) typeTime = reflect.TypeOf(time.Time{}) typeString = reflect.TypeOf("") typeJSONNumber = reflect.TypeOf(json.Number("")) + typeTimeDuration = reflect.TypeOf(time.Duration(0)) +) + +var ( + // spec for []uint8 or []byte encoding + arrayOps = map[string]bool{ + "$in": true, + "$nin": true, + "$all": true, + } ) const itoaCacheSize = 32 +const ( + getterUnknown = iota + getterNone + getterTypeVal + getterTypePtr + getterAddr +) + var itoaCache []string +var getterStyles map[reflect.Type]int +var getterIface reflect.Type +var getterMutex sync.RWMutex + func init() { itoaCache = make([]string, itoaCacheSize) for i := 0; i != itoaCacheSize; i++ { itoaCache[i] = strconv.Itoa(i) } + var iface Getter + getterIface = reflect.TypeOf(&iface).Elem() + getterStyles = make(map[reflect.Type]int) } func itoa(i int) string { @@ -74,6 +102,52 @@ func itoa(i int) string { return strconv.Itoa(i) } +func getterStyle(outt reflect.Type) int { + getterMutex.RLock() + style := getterStyles[outt] + getterMutex.RUnlock() + if style != getterUnknown { + return style + } + + getterMutex.Lock() + defer getterMutex.Unlock() + if outt.Implements(getterIface) { + vt := outt + for vt.Kind() == reflect.Ptr { + vt = vt.Elem() + } + if vt.Implements(getterIface) { + style = getterTypeVal + } else { + style = getterTypePtr + } + } else if reflect.PtrTo(outt).Implements(getterIface) { + style = getterAddr + } else { + style = getterNone + } + getterStyles[outt] = style + return style +} + +func getGetter(outt reflect.Type, out reflect.Value) Getter { + style := getterStyle(outt) + if style == getterNone { + return nil + } + if style == getterAddr { + if !out.CanAddr() { + return nil + } + return out.Addr().Interface().(Getter) + } + if style == getterTypeVal && out.Kind() == reflect.Ptr && out.IsNil() { + return nil + } + return out.Interface().(Getter) +} + // -------------------------------------------------------------------------- // Marshaling of the document value itself. @@ -129,7 +203,7 @@ func (e *encoder) addDoc(v reflect.Value) { func (e *encoder) addMap(v reflect.Value) { for _, k := range v.MapKeys() { - e.addElem(k.String(), v.MapIndex(k), false) + e.addElem(fmt.Sprint(k), v.MapIndex(k), false) } } @@ -251,7 +325,7 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { return } - if getter, ok := v.Interface().(Getter); ok { + if getter := getGetter(v.Type(), v); getter != nil { getv, err := getter.GetBSON() if err != nil { panic(err) @@ -325,7 +399,11 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { } else { e.addElemName(0xFF, name) } + case typeTimeDuration: + // Stored as int64 + e.addElemName(0x12, name) + e.addInt64(int64(v.Int() / 1e6)) default: i := v.Int() if (minSize || v.Type().Kind() != reflect.Int64) && i >= math.MinInt32 && i <= math.MaxInt32 { @@ -354,8 +432,13 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { vt := v.Type() et := vt.Elem() if et.Kind() == reflect.Uint8 { - e.addElemName(0x05, name) - e.addBinary(0x00, v.Bytes()) + if arrayOps[name] { + e.addElemName(0x04, name) + e.addDoc(v) + } else { + e.addElemName(0x05, name) + e.addBinary(0x00, v.Bytes()) + } } else if et == typeDocElem || et == typeRawDocElem { e.addElemName(0x03, name) e.addDoc(v) @@ -367,16 +450,21 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { case reflect.Array: et := v.Type().Elem() if et.Kind() == reflect.Uint8 { - e.addElemName(0x05, name) - if v.CanAddr() { - e.addBinary(0x00, v.Slice(0, v.Len()).Interface().([]byte)) + if arrayOps[name] { + e.addElemName(0x04, name) + e.addDoc(v) } else { - n := v.Len() - e.addInt32(int32(n)) - e.addBytes(0x00) - for i := 0; i < n; i++ { - el := v.Index(i) - e.addBytes(byte(el.Uint())) + e.addElemName(0x05, name) + if v.CanAddr() { + e.addBinary(0x00, v.Slice(0, v.Len()).Interface().([]byte)) + } else { + n := v.Len() + e.addInt32(int32(n)) + e.addBytes(0x00) + for i := 0; i < n; i++ { + el := v.Index(i) + e.addBytes(byte(el.Uint())) + } } } } else { @@ -419,7 +507,9 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { case RegEx: e.addElemName(0x0B, name) e.addCStr(s.Pattern) - e.addCStr(s.Options) + options := runes(s.Options) + sort.Sort(options) + e.addCStr(string(options)) case JavaScript: if s.Scope == nil { @@ -455,6 +545,14 @@ func (e *encoder) addElem(name string, v reflect.Value, minSize bool) { } } +// ------------- +// Helper method for sorting regex options +type runes []rune + +func (a runes) Len() int { return len(a) } +func (a runes) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a runes) Less(i, j int) bool { return a[i] < a[j] } + // -------------------------------------------------------------------------- // Marshaling of base types. diff --git a/vendor/gopkg.in/mgo.v2/bson/json.go b/vendor/github.com/gedge/mgo/bson/json.go similarity index 97% rename from vendor/gopkg.in/mgo.v2/bson/json.go rename to vendor/github.com/gedge/mgo/bson/json.go index 09df8260..0c310786 100644 --- a/vendor/gopkg.in/mgo.v2/bson/json.go +++ b/vendor/github.com/gedge/mgo/bson/json.go @@ -4,9 +4,11 @@ import ( "bytes" "encoding/base64" "fmt" - "gopkg.in/mgo.v2/internal/json" "strconv" + "strings" "time" + + "github.com/gedge/mgo/internal/json" ) // UnmarshalJSON unmarshals a JSON value that may hold non-standard @@ -155,7 +157,7 @@ func jencBinaryType(v interface{}) ([]byte, error) { return fbytes(`{"$binary":"%s","$type":"0x%x"}`, out, in.Kind), nil } -const jdateFormat = "2006-01-02T15:04:05.999Z" +const jdateFormat = "2006-01-02T15:04:05.999Z07:00" func jdecDate(data []byte) (interface{}, error) { var v struct { @@ -169,13 +171,15 @@ func jdecDate(data []byte) (interface{}, error) { v.S = v.Func.S } if v.S != "" { + var errs []string for _, format := range []string{jdateFormat, "2006-01-02"} { t, err := time.Parse(format, v.S) if err == nil { return t, nil } + errs = append(errs, err.Error()) } - return nil, fmt.Errorf("cannot parse date: %q", v.S) + return nil, fmt.Errorf("cannot parse date: %q [%s]", v.S, strings.Join(errs, ", ")) } var vn struct { diff --git a/vendor/github.com/gedge/mgo/bson/stream.go b/vendor/github.com/gedge/mgo/bson/stream.go new file mode 100644 index 00000000..46652845 --- /dev/null +++ b/vendor/github.com/gedge/mgo/bson/stream.go @@ -0,0 +1,90 @@ +package bson + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" +) + +const ( + // MinDocumentSize is the size of the smallest possible valid BSON document: + // an int32 size header + 0x00 (end of document). + MinDocumentSize = 5 + + // MaxDocumentSize is the largest possible size for a BSON document allowed by MongoDB, + // that is, 16 MiB (see https://docs.mongodb.com/manual/reference/limits/). + MaxDocumentSize = 16777216 +) + +// ErrInvalidDocumentSize is an error returned when a BSON document's header +// contains a size smaller than MinDocumentSize or greater than MaxDocumentSize. +type ErrInvalidDocumentSize struct { + DocumentSize int32 +} + +func (e ErrInvalidDocumentSize) Error() string { + return fmt.Sprintf("invalid document size %d", e.DocumentSize) +} + +// A Decoder reads and decodes BSON values from an input stream. +type Decoder struct { + source io.Reader +} + +// NewDecoder returns a new Decoder that reads from source. +// It does not add any extra buffering, and may not read data from source beyond the BSON values requested. +func NewDecoder(source io.Reader) *Decoder { + return &Decoder{source: source} +} + +// Decode reads the next BSON-encoded value from its input and stores it in the value pointed to by v. +// See the documentation for Unmarshal for details about the conversion of BSON into a Go value. +func (dec *Decoder) Decode(v interface{}) (err error) { + // BSON documents start with their size as a *signed* int32. + var docSize int32 + if err = binary.Read(dec.source, binary.LittleEndian, &docSize); err != nil { + return + } + + if docSize < MinDocumentSize || docSize > MaxDocumentSize { + return ErrInvalidDocumentSize{DocumentSize: docSize} + } + + docBuffer := bytes.NewBuffer(make([]byte, 0, docSize)) + if err = binary.Write(docBuffer, binary.LittleEndian, docSize); err != nil { + return + } + + // docSize is the *full* document's size (including the 4-byte size header, + // which has already been read). + if _, err = io.CopyN(docBuffer, dec.source, int64(docSize-4)); err != nil { + return + } + + // Let Unmarshal handle the rest. + defer handleErr(&err) + return Unmarshal(docBuffer.Bytes(), v) +} + +// An Encoder encodes and writes BSON values to an output stream. +type Encoder struct { + target io.Writer +} + +// NewEncoder returns a new Encoder that writes to target. +func NewEncoder(target io.Writer) *Encoder { + return &Encoder{target: target} +} + +// Encode encodes v to BSON, and if successful writes it to the Encoder's output stream. +// See the documentation for Marshal for details about the conversion of Go values to BSON. +func (enc *Encoder) Encode(v interface{}) error { + data, err := Marshal(v) + if err != nil { + return err + } + + _, err = enc.target.Write(data) + return err +} diff --git a/vendor/gopkg.in/mgo.v2/bulk.go b/vendor/github.com/gedge/mgo/bulk.go similarity index 96% rename from vendor/gopkg.in/mgo.v2/bulk.go rename to vendor/github.com/gedge/mgo/bulk.go index 072a5206..5c00bdbb 100644 --- a/vendor/gopkg.in/mgo.v2/bulk.go +++ b/vendor/github.com/gedge/mgo/bulk.go @@ -3,8 +3,9 @@ package mgo import ( "bytes" "sort" + "sync" - "gopkg.in/mgo.v2/bson" + "github.com/gedge/mgo/bson" ) // Bulk represents an operation that can be prepared with several @@ -118,6 +119,15 @@ func (e *BulkError) Cases() []BulkErrorCase { return e.ecases } +var actionPool = sync.Pool{ + New: func() interface{} { + return &bulkAction{ + docs: make([]interface{}, 0), + idxs: make([]int, 0), + } + }, +} + // Bulk returns a value to prepare the execution of a bulk operation. func (c *Collection) Bulk() *Bulk { return &Bulk{c: c, ordered: true} @@ -145,7 +155,9 @@ func (b *Bulk) action(op bulkOp, opcount int) *bulkAction { } } if action == nil { - b.actions = append(b.actions, bulkAction{op: op}) + a := actionPool.Get().(*bulkAction) + a.op = op + b.actions = append(b.actions, *a) action = &b.actions[len(b.actions)-1] } for i := 0; i < opcount; i++ { @@ -288,6 +300,9 @@ func (b *Bulk) Run() (*BulkResult, error) { default: panic("unknown bulk operation") } + action.idxs = action.idxs[0:0] + action.docs = action.docs[0:0] + actionPool.Put(action) if !ok { failed = true if b.ordered { diff --git a/vendor/github.com/gedge/mgo/changestreams.go b/vendor/github.com/gedge/mgo/changestreams.go new file mode 100644 index 00000000..17fb7b49 --- /dev/null +++ b/vendor/github.com/gedge/mgo/changestreams.go @@ -0,0 +1,357 @@ +package mgo + +import ( + "errors" + "fmt" + "reflect" + "sync" + "time" + + "github.com/gedge/mgo/bson" +) + +type FullDocument string + +const ( + Default = "default" + UpdateLookup = "updateLookup" +) + +type ChangeStream struct { + iter *Iter + isClosed bool + options ChangeStreamOptions + pipeline interface{} + resumeToken *bson.Raw + collection *Collection + readPreference *ReadPreference + err error + m sync.Mutex + sessionCopied bool +} + +type ChangeStreamOptions struct { + + // FullDocument controls the amount of data that the server will return when + // returning a changes document. + FullDocument FullDocument + + // ResumeAfter specifies the logical starting point for the new change stream. + ResumeAfter *bson.Raw + + // MaxAwaitTimeMS specifies the maximum amount of time for the server to wait + // on new documents to satisfy a change stream query. + MaxAwaitTimeMS time.Duration + + // BatchSize specifies the number of documents to return per batch. + BatchSize int + + // Collation specifies the way the server should collate returned data. + //TODO Collation *Collation +} + +var errMissingResumeToken = errors.New("resume token missing from result") + +// Watch constructs a new ChangeStream capable of receiving continuing data +// from the database. +func (coll *Collection) Watch(pipeline interface{}, + options ChangeStreamOptions) (*ChangeStream, error) { + + if pipeline == nil { + pipeline = []bson.M{} + } + + csPipe := constructChangeStreamPipeline(pipeline, options) + pipe := coll.Pipe(&csPipe) + if options.MaxAwaitTimeMS > 0 { + pipe.SetMaxTime(options.MaxAwaitTimeMS) + } + if options.BatchSize > 0 { + pipe.Batch(options.BatchSize) + } + pIter := pipe.Iter() + + // check that there was no issue creating the iterator. + // this will fail immediately with an error from the server if running against + // a standalone. + if err := pIter.Err(); err != nil { + return nil, err + } + + pIter.isChangeStream = true + return &ChangeStream{ + iter: pIter, + collection: coll, + resumeToken: nil, + options: options, + pipeline: pipeline, + }, nil +} + +// Next retrieves the next document from the change stream, blocking if necessary. +// Next returns true if a document was successfully unmarshalled into result, +// and false if an error occured. When Next returns false, the Err method should +// be called to check what error occurred during iteration. If there were no events +// available (ErrNotFound), the Err method returns nil so the user can retry the invocaton. +// +// For example: +// +// pipeline := []bson.M{} +// +// changeStream := collection.Watch(pipeline, ChangeStreamOptions{}) +// for changeStream.Next(&changeDoc) { +// fmt.Printf("Change: %v\n", changeDoc) +// } +// +// if err := changeStream.Close(); err != nil { +// return err +// } +// +// If the pipeline used removes the _id field from the result, Next will error +// because the _id field is needed to resume iteration when an error occurs. +// +func (changeStream *ChangeStream) Next(result interface{}) bool { + // the err field is being constantly overwritten and we don't want the user to + // attempt to read it at this point so we lock. + changeStream.m.Lock() + + defer changeStream.m.Unlock() + + // if we are in a state of error, then don't continue. + if changeStream.err != nil { + return false + } + + if changeStream.isClosed { + changeStream.err = fmt.Errorf("illegal use of a closed ChangeStream") + return false + } + + var err error + + // attempt to fetch the change stream result. + err = changeStream.fetchResultSet(result) + if err == nil { + return true + } + + // if we get no results we return false with no errors so the user can call Next + // again, resuming is not needed as the iterator is simply timed out as no events happened. + // The user will call Timeout in order to understand if this was the case. + if err == ErrNotFound { + return false + } + + // check if the error is resumable + if !isResumableError(err) { + // error is not resumable, give up and return it to the user. + changeStream.err = err + return false + } + + // try to resume. + err = changeStream.resume() + if err != nil { + // we've not been able to successfully resume and should only try once, + // so we give up. + changeStream.err = err + return false + } + + // we've successfully resumed the changestream. + // try to fetch the next result. + err = changeStream.fetchResultSet(result) + if err != nil { + changeStream.err = err + return false + } + + return true +} + +// Err returns nil if no errors happened during iteration, or the actual +// error otherwise. +func (changeStream *ChangeStream) Err() error { + changeStream.m.Lock() + defer changeStream.m.Unlock() + return changeStream.err +} + +// Close kills the server cursor used by the iterator, if any, and returns +// nil if no errors happened during iteration, or the actual error otherwise. +func (changeStream *ChangeStream) Close() error { + changeStream.m.Lock() + defer changeStream.m.Unlock() + changeStream.isClosed = true + err := changeStream.iter.Close() + if err != nil { + changeStream.err = err + } + if changeStream.sessionCopied { + changeStream.iter.session.Close() + changeStream.sessionCopied = false + } + return err +} + +// ResumeToken returns a copy of the current resume token held by the change stream. +// This token should be treated as an opaque token that can be provided to instantiate +// a new change stream. +func (changeStream *ChangeStream) ResumeToken() *bson.Raw { + changeStream.m.Lock() + defer changeStream.m.Unlock() + if changeStream.resumeToken == nil { + return nil + } + var tokenCopy = *changeStream.resumeToken + return &tokenCopy +} + +// Timeout returns true if the last call of Next returned false because of an iterator timeout. +func (changeStream *ChangeStream) Timeout() bool { + return changeStream.iter.Timeout() +} + +func constructChangeStreamPipeline(pipeline interface{}, + options ChangeStreamOptions) interface{} { + pipelinev := reflect.ValueOf(pipeline) + + // ensure that the pipeline passed in is a slice. + if pipelinev.Kind() != reflect.Slice { + panic("pipeline argument must be a slice") + } + + // construct the options to be used by the change notification + // pipeline stage. + changeStreamStageOptions := bson.M{} + + if options.FullDocument != "" { + changeStreamStageOptions["fullDocument"] = options.FullDocument + } + if options.ResumeAfter != nil { + changeStreamStageOptions["resumeAfter"] = options.ResumeAfter + } + + changeStreamStage := bson.M{"$changeStream": changeStreamStageOptions} + + pipeOfInterfaces := make([]interface{}, pipelinev.Len()+1) + + // insert the change notification pipeline stage at the beginning of the + // aggregation. + pipeOfInterfaces[0] = changeStreamStage + + // convert the passed in slice to a slice of interfaces. + for i := 0; i < pipelinev.Len(); i++ { + pipeOfInterfaces[1+i] = pipelinev.Index(i).Addr().Interface() + } + var pipelineAsInterface interface{} = pipeOfInterfaces + return pipelineAsInterface +} + +func (changeStream *ChangeStream) resume() error { + // copy the information for the new socket. + + // Thanks to Copy() future uses will acquire a new socket against the newly selected DB. + newSession := changeStream.iter.session.Copy() + + // fetch the cursor from the iterator and use it to run a killCursors + // on the connection. + cursorId := changeStream.iter.op.cursorId + err := runKillCursorsOnSession(newSession, cursorId) + if err != nil { + return err + } + + // change out the old connection to the database with the new connection. + if changeStream.sessionCopied { + changeStream.collection.Database.Session.Close() + } + changeStream.collection.Database.Session = newSession + changeStream.sessionCopied = true + + opts := changeStream.options + if changeStream.resumeToken != nil { + opts.ResumeAfter = changeStream.resumeToken + } + // make a new pipeline containing the resume token. + changeStreamPipeline := constructChangeStreamPipeline(changeStream.pipeline, opts) + + // generate the new iterator with the new connection. + newPipe := changeStream.collection.Pipe(changeStreamPipeline) + changeStream.iter = newPipe.Iter() + if err := changeStream.iter.Err(); err != nil { + return err + } + changeStream.iter.isChangeStream = true + return nil +} + +// fetchResumeToken unmarshals the _id field from the document, setting an error +// on the changeStream if it is unable to. +func (changeStream *ChangeStream) fetchResumeToken(rawResult *bson.Raw) error { + changeStreamResult := struct { + ResumeToken *bson.Raw `bson:"_id,omitempty"` + }{} + + err := rawResult.Unmarshal(&changeStreamResult) + if err != nil { + return err + } + + if changeStreamResult.ResumeToken == nil { + return errMissingResumeToken + } + + changeStream.resumeToken = changeStreamResult.ResumeToken + return nil +} + +func (changeStream *ChangeStream) fetchResultSet(result interface{}) error { + rawResult := bson.Raw{} + + // fetch the next set of documents from the cursor. + gotNext := changeStream.iter.Next(&rawResult) + err := changeStream.iter.Err() + if err != nil { + return err + } + + if !gotNext && err == nil { + // If the iter.Err() method returns nil despite us not getting a next batch, + // it is becuase iter.Err() silences this case. + return ErrNotFound + } + + // grab the resumeToken from the results + if err := changeStream.fetchResumeToken(&rawResult); err != nil { + return err + } + + // put the raw results into the data structure the user provided. + if err := rawResult.Unmarshal(result); err != nil { + return err + } + return nil +} + +func isResumableError(err error) bool { + _, isQueryError := err.(*QueryError) + // if it is not a database error OR it is a database error, + // but the error is a notMaster error + //and is not a missingResumeToken error (caused by the user provided pipeline) + return (!isQueryError || isNotMasterError(err)) && (err != errMissingResumeToken) +} + +func runKillCursorsOnSession(session *Session, cursorId int64) error { + socket, err := session.acquireSocket(true) + if err != nil { + return err + } + err = socket.Query(&killCursorsOp{[]int64{cursorId}}) + if err != nil { + return err + } + socket.Release() + + return nil +} diff --git a/vendor/gopkg.in/mgo.v2/cluster.go b/vendor/github.com/gedge/mgo/cluster.go similarity index 87% rename from vendor/gopkg.in/mgo.v2/cluster.go rename to vendor/github.com/gedge/mgo/cluster.go index c3bf8b01..dabe7965 100644 --- a/vendor/gopkg.in/mgo.v2/cluster.go +++ b/vendor/github.com/gedge/mgo/cluster.go @@ -30,12 +30,13 @@ import ( "errors" "fmt" "net" + "runtime" "strconv" "strings" "sync" "time" - "gopkg.in/mgo.v2/bson" + "github.com/gedge/mgo/bson" ) // --------------------------------------------------------------------------- @@ -47,23 +48,26 @@ import ( type mongoCluster struct { sync.RWMutex - serverSynced sync.Cond - userSeeds []string - dynaSeeds []string - servers mongoServers - masters mongoServers - references int - syncing bool - direct bool - failFast bool - syncCount uint - setName string - cachedIndex map[string]bool - sync chan bool - dial dialer + serverSynced sync.Cond + userSeeds []string + dynaSeeds []string + servers mongoServers + masters mongoServers + references int + syncing bool + direct bool + failFast bool + syncCount uint + setName string + cachedIndex map[string]bool + sync chan bool + dial dialer + appName string + minPoolSize int + maxIdleTimeMS int } -func newCluster(userSeeds []string, direct, failFast bool, dial dialer, setName string) *mongoCluster { +func newCluster(userSeeds []string, direct, failFast bool, dial dialer, setName string, appName string) *mongoCluster { cluster := &mongoCluster{ userSeeds: userSeeds, references: 1, @@ -71,6 +75,7 @@ func newCluster(userSeeds []string, direct, failFast bool, dial dialer, setName failFast: failFast, dial: dial, setName: setName, + appName: appName, } cluster.serverSynced.L = cluster.RWMutex.RLocker() cluster.sync = make(chan bool, 1) @@ -122,10 +127,10 @@ func (cluster *mongoCluster) removeServer(server *mongoServer) { other := cluster.servers.Remove(server) cluster.Unlock() if other != nil { - other.Close() + other.CloseIdle() log("Removed server ", server.Addr, " from cluster.") } - server.Close() + server.CloseIdle() } type isMasterResult struct { @@ -144,7 +149,39 @@ func (cluster *mongoCluster) isMaster(socket *mongoSocket, result *isMasterResul // Monotonic let's it talk to a slave and still hold the socket. session := newSession(Monotonic, cluster, 10*time.Second) session.setSocket(socket) - err := session.Run("ismaster", result) + + var cmd = bson.D{{Name: "isMaster", Value: 1}} + + // Send client metadata to the server to identify this socket if this is + // the first isMaster call only. + // + // isMaster commands issued after the initial connection handshake MUST NOT contain handshake arguments + // https://github.com/mongodb/specifications/blob/master/source/mongodb-handshake/handshake.rst#connection-handshake + // + socket.sendMeta.Do(func() { + var meta = bson.M{ + "driver": bson.M{ + "name": "mgo", + "version": "globalsign", + }, + "os": bson.M{ + "type": runtime.GOOS, + "architecture": runtime.GOARCH, + }, + } + + // Include the application name if set + if cluster.appName != "" { + meta["application"] = bson.M{"name": cluster.appName} + } + + cmd = append(cmd, bson.DocElem{ + Name: "client", + Value: meta, + }) + }) + + err := session.runOnSocket(socket, cmd, result) session.Close() return err } @@ -402,11 +439,13 @@ func (cluster *mongoCluster) syncServersLoop() { func (cluster *mongoCluster) server(addr string, tcpaddr *net.TCPAddr) *mongoServer { cluster.RLock() server := cluster.servers.Search(tcpaddr.String()) + minPoolSize := cluster.minPoolSize + maxIdleTimeMS := cluster.maxIdleTimeMS cluster.RUnlock() if server != nil { return server } - return newServer(addr, tcpaddr, cluster.sync, cluster.dial) + return newServer(addr, tcpaddr, cluster.sync, cluster.dial, minPoolSize, maxIdleTimeMS) } func resolveAddr(addr string) (*net.TCPAddr, error) { @@ -579,9 +618,17 @@ func (cluster *mongoCluster) syncServersIteration(direct bool) { // true, it will attempt to return a socket to a slave server. If it is // false, the socket will necessarily be to a master server. func (cluster *mongoCluster) AcquireSocket(mode Mode, slaveOk bool, syncTimeout time.Duration, socketTimeout time.Duration, serverTags []bson.D, poolLimit int) (s *mongoSocket, err error) { + return cluster.AcquireSocketWithPoolTimeout(mode, slaveOk, syncTimeout, socketTimeout, serverTags, poolLimit, 0) +} + +// AcquireSocketWithPoolTimeout returns a socket to a server in the cluster. If slaveOk is +// true, it will attempt to return a socket to a slave server. If it is +// false, the socket will necessarily be to a master server. +func (cluster *mongoCluster) AcquireSocketWithPoolTimeout( + mode Mode, slaveOk bool, syncTimeout time.Duration, socketTimeout time.Duration, serverTags []bson.D, poolLimit int, poolTimeout time.Duration, +) (s *mongoSocket, err error) { var started time.Time var syncCount uint - warnedLimit := false for { cluster.RLock() for { @@ -623,14 +670,10 @@ func (cluster *mongoCluster) AcquireSocket(mode Mode, slaveOk bool, syncTimeout continue } - s, abended, err := server.AcquireSocket(poolLimit, socketTimeout) - if err == errPoolLimit { - if !warnedLimit { - warnedLimit = true - log("WARNING: Per-server connection limit reached.") - } - time.Sleep(100 * time.Millisecond) - continue + s, abended, err := server.AcquireSocketWithBlocking(poolLimit, socketTimeout, poolTimeout) + if err == errPoolTimeout { + // No need to remove servers from the topology if acquiring a socket fails for this reason. + return nil, err } if err != nil { cluster.removeServer(server) @@ -646,11 +689,15 @@ func (cluster *mongoCluster) AcquireSocket(mode Mode, slaveOk bool, syncTimeout cluster.syncServers() time.Sleep(100 * time.Millisecond) continue + } else { + // We've managed to successfully reconnect to the master, we are no longer abnormaly ended + server.Lock() + server.abended = false + server.Unlock() } } return s, nil } - panic("unreached") } func (cluster *mongoCluster) CacheIndex(cacheKey string, exists bool) { diff --git a/vendor/github.com/gedge/mgo/coarse_time.go b/vendor/github.com/gedge/mgo/coarse_time.go new file mode 100644 index 00000000..e54dd17c --- /dev/null +++ b/vendor/github.com/gedge/mgo/coarse_time.go @@ -0,0 +1,62 @@ +package mgo + +import ( + "sync" + "sync/atomic" + "time" +) + +// coarseTimeProvider provides a periodically updated (approximate) time value to +// amortise the cost of frequent calls to time.Now. +// +// A read throughput increase of ~6% was measured when using coarseTimeProvider with the +// high-precision event timer (HPET) on FreeBSD 11.1 and Go 1.10.1 after merging +// #116. +// +// Calling Now returns a time.Time that is updated at the configured interval, +// however due to scheduling the value may be marginally older than expected. +// +// coarseTimeProvider is safe for concurrent use. +type coarseTimeProvider struct { + once sync.Once + stop chan struct{} + last atomic.Value +} + +// Now returns the most recently acquired time.Time value. +func (t *coarseTimeProvider) Now() time.Time { + return t.last.Load().(time.Time) +} + +// Close stops the periodic update of t. +// +// Any subsequent calls to Now will return the same value forever. +func (t *coarseTimeProvider) Close() { + t.once.Do(func() { + close(t.stop) + }) +} + +// newcoarseTimeProvider returns a coarseTimeProvider configured to update at granularity. +func newcoarseTimeProvider(granularity time.Duration) *coarseTimeProvider { + t := &coarseTimeProvider{ + stop: make(chan struct{}), + } + + t.last.Store(time.Now()) + + go func() { + ticker := time.NewTicker(granularity) + for { + select { + case <-t.stop: + ticker.Stop() + return + case <-ticker.C: + t.last.Store(time.Now()) + } + } + }() + + return t +} diff --git a/vendor/gopkg.in/mgo.v2/doc.go b/vendor/github.com/gedge/mgo/doc.go similarity index 75% rename from vendor/gopkg.in/mgo.v2/doc.go rename to vendor/github.com/gedge/mgo/doc.go index 859fd9b8..f3f373bf 100644 --- a/vendor/gopkg.in/mgo.v2/doc.go +++ b/vendor/github.com/gedge/mgo/doc.go @@ -1,9 +1,8 @@ -// Package mgo offers a rich MongoDB driver for Go. +// Package mgo (pronounced as "mango") offers a rich MongoDB driver for Go. // -// Details about the mgo project (pronounced as "mango") are found -// in its web page: +// Detailed documentation of the API is available at GoDoc: // -// http://labix.org/mgo +// https://godoc.org/github.com/globalsign/mgo // // Usage of the driver revolves around the concept of sessions. To // get started, obtain a session using the Dial function: @@ -26,6 +25,11 @@ // of its life time, so its resources may be put back in the pool or // collected, depending on the case. // +// There is a sub-package that provides support for BSON, which can be +// used by itself as well: +// +// https://godoc.org/github.com/globalsign/mgo/bson +// // For more details, see the documentation for the types and methods. // package mgo diff --git a/vendor/gopkg.in/mgo.v2/gridfs.go b/vendor/github.com/gedge/mgo/gridfs.go similarity index 92% rename from vendor/gopkg.in/mgo.v2/gridfs.go rename to vendor/github.com/gedge/mgo/gridfs.go index 42147209..6e63d1d3 100644 --- a/vendor/gopkg.in/mgo.v2/gridfs.go +++ b/vendor/github.com/gedge/mgo/gridfs.go @@ -36,9 +36,29 @@ import ( "sync" "time" - "gopkg.in/mgo.v2/bson" + "github.com/gedge/mgo/bson" ) +// GridFS stores files in two collections: +// +// - chunks stores the binary chunks. For details, see the chunks Collection. +// - files stores the file’s metadata. For details, see the files Collection. +// +// GridFS places the collections in a common bucket by prefixing each with the bucket name. +// By default, GridFS uses two collections with a bucket named fs: +// +// - fs.files +// - fs.chunks +// +// You can choose a different bucket name, as well as create multiple buckets in a single database. +// The full collection name, which includes the bucket name, is subject to the namespace length limit. +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/core/gridfs/ +// https://docs.mongodb.com/manual/core/gridfs/#gridfs-chunks-collection +// https://docs.mongodb.com/manual/core/gridfs/#gridfs-files-collection +// type GridFS struct { Files *Collection Chunks *Collection @@ -52,6 +72,7 @@ const ( gfsWriting gfsFileMode = 2 ) +// GridFile document in files collection type GridFile struct { m sync.Mutex c sync.Cond @@ -73,19 +94,19 @@ type GridFile struct { } type gfsFile struct { - Id interface{} "_id" - ChunkSize int "chunkSize" - UploadDate time.Time "uploadDate" - Length int64 ",minsize" + Id interface{} `bson:"_id"` + ChunkSize int `bson:"chunkSize"` + UploadDate time.Time `bson:"uploadDate"` + Length int64 `bson:",minsize"` MD5 string - Filename string ",omitempty" - ContentType string "contentType,omitempty" - Metadata *bson.Raw ",omitempty" + Filename string `bson:",omitempty"` + ContentType string `bson:"contentType,omitempty"` + Metadata *bson.Raw `bson:",omitempty"` } type gfsChunk struct { - Id interface{} "_id" - FilesId interface{} "files_id" + Id interface{} `bson:"_id"` + FilesId interface{} `bson:"files_id"` N int Data []byte } @@ -319,12 +340,12 @@ func (gfs *GridFS) RemoveId(id interface{}) error { if err != nil { return err } - _, err = gfs.Chunks.RemoveAll(bson.D{{"files_id", id}}) + _, err = gfs.Chunks.RemoveAll(bson.D{{Name: "files_id", Value: id}}) return err } type gfsDocId struct { - Id interface{} "_id" + Id interface{} `bson:"_id"` } // Remove deletes all files with the provided name from the GridFS. @@ -411,7 +432,7 @@ func (file *GridFile) ContentType() string { return file.doc.ContentType } -// ContentType changes the optional file content type. An empty string may be +// SetContentType changes the optional file content type. An empty string may be // used to unset it. // // It is a runtime error to call this function when the file is not open @@ -530,7 +551,7 @@ func (file *GridFile) completeWrite() { file.err = file.gfs.Files.Insert(file.doc) } if file.err != nil { - file.gfs.Chunks.RemoveAll(bson.D{{"files_id", file.doc.Id}}) + file.gfs.Chunks.RemoveAll(bson.D{{Name: "files_id", Value: file.doc.Id}}) } if file.err == nil { index := Index{ @@ -734,7 +755,7 @@ func (file *GridFile) getChunk() (data []byte, err error) { } else { debugf("GridFile %p: Fetching chunk %d", file, file.chunk) var doc gfsChunk - err = file.gfs.Chunks.Find(bson.D{{"files_id", file.doc.Id}, {"n", file.chunk}}).One(&doc) + err = file.gfs.Chunks.Find(bson.D{{Name: "files_id", Value: file.doc.Id}, {Name: "n", Value: file.chunk}}).One(&doc) data = doc.Data } file.chunk++ @@ -750,7 +771,7 @@ func (file *GridFile) getChunk() (data []byte, err error) { defer session.Close() chunks = chunks.With(session) var doc gfsChunk - cache.err = chunks.Find(bson.D{{"files_id", id}, {"n", n}}).One(&doc) + cache.err = chunks.Find(bson.D{{Name: "files_id", Value: id}, {Name: "n", Value: n}}).One(&doc) cache.data = doc.Data cache.wait.Unlock() }(file.doc.Id, file.chunk) diff --git a/vendor/gopkg.in/mgo.v2/internal/json/LICENSE b/vendor/github.com/gedge/mgo/internal/json/LICENSE similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/json/LICENSE rename to vendor/github.com/gedge/mgo/internal/json/LICENSE diff --git a/vendor/gopkg.in/mgo.v2/internal/json/decode.go b/vendor/github.com/gedge/mgo/internal/json/decode.go similarity index 99% rename from vendor/gopkg.in/mgo.v2/internal/json/decode.go rename to vendor/github.com/gedge/mgo/internal/json/decode.go index ce7c7d24..d5ca1f9a 100644 --- a/vendor/gopkg.in/mgo.v2/internal/json/decode.go +++ b/vendor/github.com/gedge/mgo/internal/json/decode.go @@ -773,7 +773,7 @@ func (d *decodeState) isNull(off int) bool { // name consumes a const or function from d.data[d.off-1:], decoding into the value v. // the first byte of the function name has been read already. func (d *decodeState) name(v reflect.Value) { - if d.isNull(d.off-1) { + if d.isNull(d.off - 1) { d.literal(v) return } @@ -899,7 +899,7 @@ func (d *decodeState) name(v reflect.Value) { } // Check for unmarshaler on func field itself. - u, ut, pv = d.indirect(v, false) + u, _, _ = d.indirect(v, false) if u != nil { d.off = nameStart err := u.UnmarshalJSON(d.next()) @@ -1036,7 +1036,7 @@ func (d *decodeState) keyed() (interface{}, bool) { break } - name := d.data[d.off-1+start : d.off-1+end] + name := bytes.Trim(d.data[d.off-1+start:d.off-1+end], " \n\t") var key []byte var ok bool @@ -1076,9 +1076,9 @@ func (d *decodeState) storeKeyed(v reflect.Value) bool { } var ( - trueBytes = []byte("true") + trueBytes = []byte("true") falseBytes = []byte("false") - nullBytes = []byte("null") + nullBytes = []byte("null") ) func (d *decodeState) storeValue(v reflect.Value, from interface{}) { diff --git a/vendor/gopkg.in/mgo.v2/internal/json/encode.go b/vendor/github.com/gedge/mgo/internal/json/encode.go similarity index 99% rename from vendor/gopkg.in/mgo.v2/internal/json/encode.go rename to vendor/github.com/gedge/mgo/internal/json/encode.go index 67a0f006..e4b8f864 100644 --- a/vendor/gopkg.in/mgo.v2/internal/json/encode.go +++ b/vendor/github.com/gedge/mgo/internal/json/encode.go @@ -209,6 +209,8 @@ func (e *UnsupportedTypeError) Error() string { return "json: unsupported type: " + e.Type.String() } +// An UnsupportedValueError is returned by Marshal when attempting +// to encode an unsupported value. type UnsupportedValueError struct { Value reflect.Value Str string @@ -218,7 +220,7 @@ func (e *UnsupportedValueError) Error() string { return "json: unsupported value: " + e.Str } -// Before Go 1.2, an InvalidUTF8Error was returned by Marshal when +// InvalidUTF8Error before Go 1.2, an InvalidUTF8Error was returned by Marshal when // attempting to encode a string value with invalid UTF-8 sequences. // As of Go 1.2, Marshal instead coerces the string to valid UTF-8 by // replacing invalid bytes with the Unicode replacement rune U+FFFD. @@ -232,6 +234,8 @@ func (e *InvalidUTF8Error) Error() string { return "json: invalid UTF-8 in string: " + strconv.Quote(e.S) } +// A MarshalerError is returned by Marshal when attempting +// to marshal an invalid JSON type MarshalerError struct { Type reflect.Type Err error diff --git a/vendor/gopkg.in/mgo.v2/internal/json/extension.go b/vendor/github.com/gedge/mgo/internal/json/extension.go similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/json/extension.go rename to vendor/github.com/gedge/mgo/internal/json/extension.go diff --git a/vendor/gopkg.in/mgo.v2/internal/json/fold.go b/vendor/github.com/gedge/mgo/internal/json/fold.go similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/json/fold.go rename to vendor/github.com/gedge/mgo/internal/json/fold.go diff --git a/vendor/gopkg.in/mgo.v2/internal/json/indent.go b/vendor/github.com/gedge/mgo/internal/json/indent.go similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/json/indent.go rename to vendor/github.com/gedge/mgo/internal/json/indent.go diff --git a/vendor/gopkg.in/mgo.v2/internal/json/scanner.go b/vendor/github.com/gedge/mgo/internal/json/scanner.go similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/json/scanner.go rename to vendor/github.com/gedge/mgo/internal/json/scanner.go diff --git a/vendor/gopkg.in/mgo.v2/internal/json/stream.go b/vendor/github.com/gedge/mgo/internal/json/stream.go similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/json/stream.go rename to vendor/github.com/gedge/mgo/internal/json/stream.go diff --git a/vendor/gopkg.in/mgo.v2/internal/json/tags.go b/vendor/github.com/gedge/mgo/internal/json/tags.go similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/json/tags.go rename to vendor/github.com/gedge/mgo/internal/json/tags.go diff --git a/vendor/gopkg.in/mgo.v2/internal/sasl/sasl.c b/vendor/github.com/gedge/mgo/internal/sasl/sasl.c similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/sasl/sasl.c rename to vendor/github.com/gedge/mgo/internal/sasl/sasl.c diff --git a/vendor/gopkg.in/mgo.v2/internal/sasl/sasl.go b/vendor/github.com/gedge/mgo/internal/sasl/sasl.go similarity index 93% rename from vendor/gopkg.in/mgo.v2/internal/sasl/sasl.go rename to vendor/github.com/gedge/mgo/internal/sasl/sasl.go index 8375dddf..25a53742 100644 --- a/vendor/gopkg.in/mgo.v2/internal/sasl/sasl.go +++ b/vendor/github.com/gedge/mgo/internal/sasl/sasl.go @@ -8,6 +8,7 @@ package sasl // #cgo LDFLAGS: -lsasl2 +// #cgo CFLAGS: -Wno-deprecated-declarations // // struct sasl_conn {}; // @@ -25,7 +26,8 @@ import ( "unsafe" ) -type saslStepper interface { +// Stepper interface for saslSession +type Stepper interface { Step(serverData []byte) (clientData []byte, done bool, err error) Close() } @@ -49,7 +51,8 @@ func initSASL() { } } -func New(username, password, mechanism, service, host string) (saslStepper, error) { +// New creates a new saslSession +func New(username, password, mechanism, service, host string) (Stepper, error) { initOnce.Do(initSASL) if initError != nil { return nil, initError diff --git a/vendor/gopkg.in/mgo.v2/internal/sasl/sasl_windows.c b/vendor/github.com/gedge/mgo/internal/sasl/sasl_windows.c similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/sasl/sasl_windows.c rename to vendor/github.com/gedge/mgo/internal/sasl/sasl_windows.c diff --git a/vendor/gopkg.in/mgo.v2/internal/sasl/sasl_windows.go b/vendor/github.com/gedge/mgo/internal/sasl/sasl_windows.go similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/sasl/sasl_windows.go rename to vendor/github.com/gedge/mgo/internal/sasl/sasl_windows.go diff --git a/vendor/gopkg.in/mgo.v2/internal/sasl/sasl_windows.h b/vendor/github.com/gedge/mgo/internal/sasl/sasl_windows.h similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/sasl/sasl_windows.h rename to vendor/github.com/gedge/mgo/internal/sasl/sasl_windows.h diff --git a/vendor/gopkg.in/mgo.v2/internal/sasl/sspi_windows.c b/vendor/github.com/gedge/mgo/internal/sasl/sspi_windows.c similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/sasl/sspi_windows.c rename to vendor/github.com/gedge/mgo/internal/sasl/sspi_windows.c diff --git a/vendor/gopkg.in/mgo.v2/internal/sasl/sspi_windows.h b/vendor/github.com/gedge/mgo/internal/sasl/sspi_windows.h similarity index 100% rename from vendor/gopkg.in/mgo.v2/internal/sasl/sspi_windows.h rename to vendor/github.com/gedge/mgo/internal/sasl/sspi_windows.h diff --git a/vendor/gopkg.in/mgo.v2/internal/scram/scram.go b/vendor/github.com/gedge/mgo/internal/scram/scram.go similarity index 97% rename from vendor/gopkg.in/mgo.v2/internal/scram/scram.go rename to vendor/github.com/gedge/mgo/internal/scram/scram.go index 80cda913..d3ddd02f 100644 --- a/vendor/gopkg.in/mgo.v2/internal/scram/scram.go +++ b/vendor/github.com/gedge/mgo/internal/scram/scram.go @@ -24,7 +24,7 @@ // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -// Pacakage scram implements a SCRAM-{SHA-1,etc} client per RFC5802. +// Package scram implements a SCRAM-{SHA-1,etc} client per RFC5802. // // http://tools.ietf.org/html/rfc5802 // @@ -96,7 +96,7 @@ func (c *Client) Out() []byte { return c.out.Bytes() } -// Err returns the error that ocurred, or nil if there were no errors. +// Err returns the error that occurred, or nil if there were no errors. func (c *Client) Err() error { return c.err } @@ -133,7 +133,7 @@ func (c *Client) Step(in []byte) bool { func (c *Client) step1(in []byte) error { if len(c.clientNonce) == 0 { const nonceLen = 6 - buf := make([]byte, nonceLen + b64.EncodedLen(nonceLen)) + buf := make([]byte, nonceLen+b64.EncodedLen(nonceLen)) if _, err := rand.Read(buf[:nonceLen]); err != nil { return fmt.Errorf("cannot read random SCRAM-SHA-1 nonce from operating system: %v", err) } diff --git a/vendor/gopkg.in/mgo.v2/log.go b/vendor/github.com/gedge/mgo/log.go similarity index 91% rename from vendor/gopkg.in/mgo.v2/log.go rename to vendor/github.com/gedge/mgo/log.go index 53eb4237..d8377949 100644 --- a/vendor/gopkg.in/mgo.v2/log.go +++ b/vendor/github.com/gedge/mgo/log.go @@ -34,15 +34,15 @@ import ( // --------------------------------------------------------------------------- // Logging integration. -// Avoid importing the log type information unnecessarily. There's a small cost +// LogLogger avoid importing the log type information unnecessarily. There's a small cost // associated with using an interface rather than the type. Depending on how // often the logger is plugged in, it would be worth using the type instead. -type log_Logger interface { +type logLogger interface { Output(calldepth int, s string) error } var ( - globalLogger log_Logger + globalLogger logLogger globalDebug bool globalMutex sync.Mutex ) @@ -53,8 +53,8 @@ var ( // the application starts. Having raceDetector as a constant, the compiler // should elide the locks altogether in actual use. -// Specify the *log.Logger object where log messages should be sent to. -func SetLogger(logger log_Logger) { +// SetLogger specify the *log.Logger object where log messages should be sent to. +func SetLogger(logger logLogger) { if raceDetector { globalMutex.Lock() defer globalMutex.Unlock() @@ -62,7 +62,7 @@ func SetLogger(logger log_Logger) { globalLogger = logger } -// Enable the delivery of debug messages to the logger. Only meaningful +// SetDebug enable the delivery of debug messages to the logger. Only meaningful // if a logger is also set. func SetDebug(debug bool) { if raceDetector { diff --git a/vendor/gopkg.in/mgo.v2/queue.go b/vendor/github.com/gedge/mgo/queue.go similarity index 100% rename from vendor/gopkg.in/mgo.v2/queue.go rename to vendor/github.com/gedge/mgo/queue.go diff --git a/vendor/gopkg.in/mgo.v2/raceoff.go b/vendor/github.com/gedge/mgo/raceoff.go similarity index 100% rename from vendor/gopkg.in/mgo.v2/raceoff.go rename to vendor/github.com/gedge/mgo/raceoff.go diff --git a/vendor/gopkg.in/mgo.v2/raceon.go b/vendor/github.com/gedge/mgo/raceon.go similarity index 100% rename from vendor/gopkg.in/mgo.v2/raceon.go rename to vendor/github.com/gedge/mgo/raceon.go diff --git a/vendor/gopkg.in/mgo.v2/saslimpl.go b/vendor/github.com/gedge/mgo/saslimpl.go similarity index 83% rename from vendor/gopkg.in/mgo.v2/saslimpl.go rename to vendor/github.com/gedge/mgo/saslimpl.go index 0d25f25c..e13f9734 100644 --- a/vendor/gopkg.in/mgo.v2/saslimpl.go +++ b/vendor/github.com/gedge/mgo/saslimpl.go @@ -3,7 +3,7 @@ package mgo import ( - "gopkg.in/mgo.v2/internal/sasl" + "github.com/gedge/mgo/internal/sasl" ) func saslNew(cred Credential, host string) (saslStepper, error) { diff --git a/vendor/gopkg.in/mgo.v2/saslstub.go b/vendor/github.com/gedge/mgo/saslstub.go similarity index 100% rename from vendor/gopkg.in/mgo.v2/saslstub.go rename to vendor/github.com/gedge/mgo/saslstub.go diff --git a/vendor/gopkg.in/mgo.v2/server.go b/vendor/github.com/gedge/mgo/server.go similarity index 65% rename from vendor/gopkg.in/mgo.v2/server.go rename to vendor/github.com/gedge/mgo/server.go index 39259869..f458fa63 100644 --- a/vendor/gopkg.in/mgo.v2/server.go +++ b/vendor/github.com/gedge/mgo/server.go @@ -33,9 +33,21 @@ import ( "sync" "time" - "gopkg.in/mgo.v2/bson" + "github.com/gedge/mgo/bson" ) +// coarseTime is used to amortise the cost of querying the timecounter (possibly +// incurring a syscall too) when setting a socket.lastTimeUsed value which +// happens frequently in the hot-path. +// +// The lastTimeUsed value may be skewed by at least 25ms (see +// coarseTimeProvider). +var coarseTime *coarseTimeProvider + +func init() { + coarseTime = newcoarseTimeProvider(25 * time.Millisecond) +} + // --------------------------------------------------------------------------- // Mongo server encapsulation. @@ -46,15 +58,18 @@ type mongoServer struct { tcpaddr *net.TCPAddr unusedSockets []*mongoSocket liveSockets []*mongoSocket - closed bool - abended bool sync chan bool dial dialer pingValue time.Duration pingIndex int - pingCount uint32 pingWindow [6]time.Duration info *mongoServerInfo + pingCount uint32 + closed bool + abended bool + minPoolSize int + maxIdleTimeMS int + poolWaiter *sync.Cond } type dialer struct { @@ -76,21 +91,28 @@ type mongoServerInfo struct { var defaultServerInfo mongoServerInfo -func newServer(addr string, tcpaddr *net.TCPAddr, sync chan bool, dial dialer) *mongoServer { +func newServer(addr string, tcpaddr *net.TCPAddr, syncChan chan bool, dial dialer, minPoolSize, maxIdleTimeMS int) *mongoServer { server := &mongoServer{ - Addr: addr, - ResolvedAddr: tcpaddr.String(), - tcpaddr: tcpaddr, - sync: sync, - dial: dial, - info: &defaultServerInfo, - pingValue: time.Hour, // Push it back before an actual ping. + Addr: addr, + ResolvedAddr: tcpaddr.String(), + tcpaddr: tcpaddr, + sync: syncChan, + dial: dial, + info: &defaultServerInfo, + pingValue: time.Hour, // Push it back before an actual ping. + minPoolSize: minPoolSize, + maxIdleTimeMS: maxIdleTimeMS, } + server.poolWaiter = sync.NewCond(server) go server.pinger(true) + if maxIdleTimeMS != 0 { + go server.poolShrinker() + } return server } var errPoolLimit = errors.New("per-server connection limit reached") +var errPoolTimeout = errors.New("could not acquire connection within pool timeout") var errServerClosed = errors.New("server was closed") // AcquireSocket returns a socket for communicating with the server. @@ -102,6 +124,21 @@ var errServerClosed = errors.New("server was closed") // use in this server is greater than the provided limit, errPoolLimit is // returned. func (server *mongoServer) AcquireSocket(poolLimit int, timeout time.Duration) (socket *mongoSocket, abended bool, err error) { + return server.acquireSocketInternal(poolLimit, timeout, false, 0*time.Millisecond) +} + +// AcquireSocketWithBlocking wraps AcquireSocket, but if a socket is not available, it will _not_ +// return errPoolLimit. Instead, it will block waiting for a socket to become available. If poolTimeout +// should elapse before a socket is available, it will return errPoolTimeout. +func (server *mongoServer) AcquireSocketWithBlocking( + poolLimit int, socketTimeout time.Duration, poolTimeout time.Duration, +) (socket *mongoSocket, abended bool, err error) { + return server.acquireSocketInternal(poolLimit, socketTimeout, true, poolTimeout) +} + +func (server *mongoServer) acquireSocketInternal( + poolLimit int, timeout time.Duration, shouldBlock bool, poolTimeout time.Duration, +) (socket *mongoSocket, abended bool, err error) { for { server.Lock() abended = server.abended @@ -109,11 +146,58 @@ func (server *mongoServer) AcquireSocket(poolLimit int, timeout time.Duration) ( server.Unlock() return nil, abended, errServerClosed } - n := len(server.unusedSockets) - if poolLimit > 0 && len(server.liveSockets)-n >= poolLimit { - server.Unlock() - return nil, false, errPoolLimit + if poolLimit > 0 { + if shouldBlock { + // Beautiful. Golang conditions don't have a WaitWithTimeout, so I've implemented the timeout + // with a wait + broadcast. The broadcast will cause the loop here to re-check the timeout, + // and fail if it is blown. + // Yes, this is a spurious wakeup, but we can't do a directed signal without having one condition + // variable per waiter, which would involve loop traversal in the RecycleSocket + // method. + // We also can't use the approach of turning a condition variable into a channel outlined in + // https://github.com/golang/go/issues/16620, since the lock needs to be held in _this_ goroutine. + waitDone := make(chan struct{}) + timeoutHit := false + if poolTimeout > 0 { + go func() { + select { + case <-waitDone: + case <-time.After(poolTimeout): + // timeoutHit is part of the wait condition, so needs to be changed under mutex. + server.Lock() + defer server.Unlock() + timeoutHit = true + server.poolWaiter.Broadcast() + } + }() + } + timeSpentWaiting := time.Duration(0) + for len(server.liveSockets)-len(server.unusedSockets) >= poolLimit && !timeoutHit { + // We only count time spent in Wait(), and not time evaluating the entire loop, + // so that in the happy non-blocking path where the condition above evaluates true + // first time, we record a nice round zero wait time. + waitStart := time.Now() + // unlocks server mutex, waits, and locks again. Thus, the condition + // above is evaluated only when the lock is held. + server.poolWaiter.Wait() + timeSpentWaiting += time.Since(waitStart) + } + close(waitDone) + if timeoutHit { + server.Unlock() + stats.noticePoolTimeout(timeSpentWaiting) + return nil, false, errPoolTimeout + } + // Record that we fetched a connection of of a socket list and how long we spent waiting + stats.noticeSocketAcquisition(timeSpentWaiting) + } else { + if len(server.liveSockets)-len(server.unusedSockets) >= poolLimit { + server.Unlock() + return nil, false, errPoolLimit + } + } } + n := len(server.unusedSockets) if n > 0 { socket = server.unusedSockets[n-1] server.unusedSockets[n-1] = nil // Help GC. @@ -143,7 +227,6 @@ func (server *mongoServer) AcquireSocket(poolLimit int, timeout time.Duration) ( } return } - panic("unreachable") } // Connect establishes a new connection to the server. This should @@ -187,6 +270,16 @@ func (server *mongoServer) Connect(timeout time.Duration) (*mongoSocket, error) // Close forces closing all sockets that are alive, whether // they're currently in use or not. func (server *mongoServer) Close() { + server.close(false) +} + +// CloseIdle closing all sockets that are idle, +// sockets currently in use will be closed after idle. +func (server *mongoServer) CloseIdle() { + server.close(true) +} + +func (server *mongoServer) close(waitForIdle bool) { server.Lock() server.closed = true liveSockets := server.liveSockets @@ -196,7 +289,11 @@ func (server *mongoServer) Close() { server.Unlock() logf("Connections to %s closing (%d live sockets).", server.Addr, len(liveSockets)) for i, s := range liveSockets { - s.Close() + if waitForIdle { + s.CloseAfterIdle() + } else { + s.Close() + } liveSockets[i] = nil } for i := range unusedSockets { @@ -208,8 +305,17 @@ func (server *mongoServer) Close() { func (server *mongoServer) RecycleSocket(socket *mongoSocket) { server.Lock() if !server.closed { + socket.lastTimeUsed = coarseTime.Now() // A rough approximation of the current time - see courseTime server.unusedSockets = append(server.unusedSockets, socket) } + // If anybody is waiting for a connection, they should try now. + // Note that this _has_ to be broadcast, not signal; the signature of AcquireSocket + // and AcquireSocketWithBlocking allow the caller to specify the max number of connections, + // rather than that being an intrinsic property of the connection pool (I assume to ensure + // that there is always a connection available for replset topology discovery). Thus, once + // a connection is returned to the pool, _every_ waiter needs to check if the connection count + // is underneath their particular value for poolLimit. + server.poolWaiter.Broadcast() server.Unlock() } @@ -292,7 +398,7 @@ func (server *mongoServer) pinger(loop bool) { } op := queryOp{ collection: "admin.$cmd", - query: bson.D{{"ping", 1}}, + query: bson.D{{Name: "ping", Value: 1}}, flags: flagSlaveOk, limit: -1, } @@ -305,7 +411,7 @@ func (server *mongoServer) pinger(loop bool) { if err == nil { start := time.Now() _, _ = socket.SimpleQuery(&op) - delay := time.Now().Sub(start) + delay := time.Since(start) server.pingWindow[server.pingIndex] = delay server.pingIndex = (server.pingIndex + 1) % len(server.pingWindow) @@ -333,6 +439,53 @@ func (server *mongoServer) pinger(loop bool) { } } +func (server *mongoServer) poolShrinker() { + ticker := time.NewTicker(1 * time.Minute) + for _ = range ticker.C { + if server.closed { + ticker.Stop() + return + } + server.Lock() + unused := len(server.unusedSockets) + if unused < server.minPoolSize { + server.Unlock() + continue + } + now := time.Now() + end := 0 + reclaimMap := map[*mongoSocket]struct{}{} + // Because the acquisition and recycle are done at the tail of array, + // the head is always the oldest unused socket. + for _, s := range server.unusedSockets[:unused-server.minPoolSize] { + if s.lastTimeUsed.Add(time.Duration(server.maxIdleTimeMS) * time.Millisecond).After(now) { + break + } + end++ + reclaimMap[s] = struct{}{} + } + tbr := server.unusedSockets[:end] + if end > 0 { + next := make([]*mongoSocket, unused-end) + copy(next, server.unusedSockets[end:]) + server.unusedSockets = next + remainSockets := []*mongoSocket{} + for _, s := range server.liveSockets { + if _, ok := reclaimMap[s]; !ok { + remainSockets = append(remainSockets, s) + } + } + server.liveSockets = remainSockets + stats.conn(-1*end, server.info.Master) + } + server.Unlock() + + for _, s := range tbr { + s.Close() + } + } +} + type mongoServerSlice []*mongoServer func (s mongoServerSlice) Len() int { diff --git a/vendor/gopkg.in/mgo.v2/session.go b/vendor/github.com/gedge/mgo/session.go similarity index 78% rename from vendor/gopkg.in/mgo.v2/session.go rename to vendor/github.com/gedge/mgo/session.go index 3dccf364..94589da0 100644 --- a/vendor/gopkg.in/mgo.v2/session.go +++ b/vendor/github.com/gedge/mgo/session.go @@ -28,6 +28,9 @@ package mgo import ( "crypto/md5" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" "encoding/hex" "errors" "fmt" @@ -41,26 +44,35 @@ import ( "sync" "time" - "gopkg.in/mgo.v2/bson" + "github.com/gedge/mgo/bson" ) +// Mode read preference mode. See Eventual, Monotonic and Strong for details +// +// Relevant documentation on read preference modes: +// +// http://docs.mongodb.org/manual/reference/read-preference/ +// type Mode int const ( - // Relevant documentation on read preference modes: - // - // http://docs.mongodb.org/manual/reference/read-preference/ - // - Primary Mode = 2 // Default mode. All operations read from the current replica set primary. - PrimaryPreferred Mode = 3 // Read from the primary if available. Read from the secondary otherwise. - Secondary Mode = 4 // Read from one of the nearest secondary members of the replica set. - SecondaryPreferred Mode = 5 // Read from one of the nearest secondaries if available. Read from primary otherwise. - Nearest Mode = 6 // Read from one of the nearest members, irrespective of it being primary or secondary. - - // Read preference modes are specific to mgo: - Eventual Mode = 0 // Same as Nearest, but may change servers between reads. - Monotonic Mode = 1 // Same as SecondaryPreferred before first write. Same as Primary after first write. - Strong Mode = 2 // Same as Primary. + // Primary mode is default mode. All operations read from the current replica set primary. + Primary Mode = 2 + // PrimaryPreferred mode: read from the primary if available. Read from the secondary otherwise. + PrimaryPreferred Mode = 3 + // Secondary mode: read from one of the nearest secondary members of the replica set. + Secondary Mode = 4 + // SecondaryPreferred mode: read from one of the nearest secondaries if available. Read from primary otherwise. + SecondaryPreferred Mode = 5 + // Nearest mode: read from one of the nearest members, irrespective of it being primary or secondary. + Nearest Mode = 6 + + // Eventual mode is specific to mgo, and is same as Nearest, but may change servers between reads. + Eventual Mode = 0 + // Monotonic mode is specifc to mgo, and is same as SecondaryPreferred before first write. Same as Primary after first write. + Monotonic Mode = 1 + // Strong mode is specific to mgo, and is same as Primary. + Strong Mode = 2 ) // mgo.v3: Drop Strong mode, suffix all modes with "Mode". @@ -75,35 +87,49 @@ const ( // multiple goroutines will cause them to share the same underlying socket. // See the documentation on Session.SetMode for more details. type Session struct { - m sync.RWMutex - cluster_ *mongoCluster - slaveSocket *mongoSocket - masterSocket *mongoSocket - slaveOk bool - consistency Mode - queryConfig query - safeOp *queryOp - syncTimeout time.Duration - sockTimeout time.Duration defaultdb string sourcedb string - dialCred *Credential - creds []Credential + syncTimeout time.Duration + sockTimeout time.Duration poolLimit int + poolTimeout time.Duration + consistency Mode + creds []Credential + dialCred *Credential + safeOp *queryOp + mgoCluster *mongoCluster + slaveSocket *mongoSocket + masterSocket *mongoSocket + m sync.RWMutex + queryConfig query bypassValidation bool + slaveOk bool } +// Database holds collections of documents +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/core/databases-and-collections/#databases +// type Database struct { Session *Session Name string } +// Collection stores documents +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/core/databases-and-collections/#collections +// type Collection struct { Database *Database Name string // "collection" FullName string // "db.collection" } +// Query keeps info on the query. type Query struct { m sync.Mutex session *Session @@ -117,13 +143,19 @@ type query struct { } type getLastError struct { - CmdName int "getLastError,omitempty" - W interface{} "w,omitempty" - WTimeout int "wtimeout,omitempty" - FSync bool "fsync,omitempty" - J bool "j,omitempty" + CmdName int `bson:"getLastError,omitempty"` + W interface{} `bson:"w,omitempty"` + WTimeout int `bson:"wtimeout,omitempty"` + FSync bool `bson:"fsync,omitempty"` + J bool `bson:"j,omitempty"` } +// Iter stores informations about a Cursor +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/tutorial/iterate-a-cursor/ +// type Iter struct { m sync.Mutex gotReply sync.Cond @@ -133,17 +165,22 @@ type Iter struct { err error op getMoreOp prefetch float64 - limit int32 docsToReceive int docsBeforeMore int timeout time.Duration + limit int32 timedout bool - findCmd bool + isFindCmd bool + isChangeStream bool + maxTimeMS int64 } var ( + // ErrNotFound error returned when a document could not be found ErrNotFound = errors.New("not found") - ErrCursor = errors.New("invalid cursor") + // ErrCursor error returned when trying to retrieve documents from + // an invalid cursor + ErrCursor = errors.New("invalid cursor") ) const ( @@ -157,9 +194,9 @@ const ( // topology. // // Dial will timeout after 10 seconds if a server isn't reached. The returned -// session will timeout operations after one minute by default if servers -// aren't available. To customize the timeout, see DialWithTimeout, -// SetSyncTimeout, and SetSocketTimeout. +// session will timeout operations after one minute by default if servers aren't +// available. To customize the timeout, see DialWithTimeout, SetSyncTimeout, and +// SetSocketTimeout. // // This method is generally called just once for a given cluster. Further // sessions to the same cluster are then established using the New or Copy @@ -184,8 +221,8 @@ const ( // If the port number is not provided for a server, it defaults to 27017. // // The username and password provided in the URL will be used to authenticate -// into the database named after the slash at the end of the host names, or -// into the "admin" database if none is provided. The authentication information +// into the database named after the slash at the end of the host names, or into +// the "admin" database if none is provided. The authentication information // will persist in sessions obtained through the New method as well. // // The following connection options are supported after the question mark: @@ -235,6 +272,20 @@ const ( // Defines the per-server socket pool limit. Defaults to 4096. // See Session.SetPoolLimit for details. // +// minPoolSize= +// +// Defines the per-server socket pool minium size. Defaults to 0. +// +// maxIdleTimeMS= +// +// The maximum number of milliseconds that a connection can remain idle in the pool +// before being removed and closed. If maxIdleTimeMS is 0, connections will never be +// closed due to inactivity. +// +// appName= +// +// The identifier of this client application. This parameter is used to +// annotate logs / profiler output and cannot exceed 128 bytes. // // Relevant documentation: // @@ -279,45 +330,109 @@ func ParseURL(url string) (*DialInfo, error) { source := "" setName := "" poolLimit := 0 - for k, v := range uinfo.options { - switch k { + appName := "" + readPreferenceMode := Primary + var readPreferenceTagSets []bson.D + minPoolSize := 0 + maxIdleTimeMS := 0 + for _, opt := range uinfo.options { + switch opt.key { case "authSource": - source = v + source = opt.value case "authMechanism": - mechanism = v + mechanism = opt.value case "gssapiServiceName": - service = v + service = opt.value case "replicaSet": - setName = v + setName = opt.value case "maxPoolSize": - poolLimit, err = strconv.Atoi(v) + poolLimit, err = strconv.Atoi(opt.value) if err != nil { - return nil, errors.New("bad value for maxPoolSize: " + v) + return nil, errors.New("bad value for maxPoolSize: " + opt.value) + } + case "appName": + if len(opt.value) > 128 { + return nil, errors.New("appName too long, must be < 128 bytes: " + opt.value) + } + appName = opt.value + case "readPreference": + switch opt.value { + case "nearest": + readPreferenceMode = Nearest + case "primary": + readPreferenceMode = Primary + case "primaryPreferred": + readPreferenceMode = PrimaryPreferred + case "secondary": + readPreferenceMode = Secondary + case "secondaryPreferred": + readPreferenceMode = SecondaryPreferred + default: + return nil, errors.New("bad value for readPreference: " + opt.value) + } + case "readPreferenceTags": + tags := strings.Split(opt.value, ",") + var doc bson.D + for _, tag := range tags { + kvp := strings.Split(tag, ":") + if len(kvp) != 2 { + return nil, errors.New("bad value for readPreferenceTags: " + opt.value) + } + doc = append(doc, bson.DocElem{Name: strings.TrimSpace(kvp[0]), Value: strings.TrimSpace(kvp[1])}) + } + readPreferenceTagSets = append(readPreferenceTagSets, doc) + case "minPoolSize": + minPoolSize, err = strconv.Atoi(opt.value) + if err != nil { + return nil, errors.New("bad value for minPoolSize: " + opt.value) + } + if minPoolSize < 0 { + return nil, errors.New("bad value (negtive) for minPoolSize: " + opt.value) + } + case "maxIdleTimeMS": + maxIdleTimeMS, err = strconv.Atoi(opt.value) + if err != nil { + return nil, errors.New("bad value for maxIdleTimeMS: " + opt.value) + } + if maxIdleTimeMS < 0 { + return nil, errors.New("bad value (negtive) for maxIdleTimeMS: " + opt.value) } case "connect": - if v == "direct" { + if opt.value == "direct" { direct = true break } - if v == "replicaSet" { + if opt.value == "replicaSet" { break } fallthrough default: - return nil, errors.New("unsupported connection URL option: " + k + "=" + v) + return nil, errors.New("unsupported connection URL option: " + opt.key + "=" + opt.value) } } + + if readPreferenceMode == Primary && len(readPreferenceTagSets) > 0 { + return nil, errors.New("readPreferenceTagSet may not be specified when readPreference is primary") + } + info := DialInfo{ - Addrs: uinfo.addrs, - Direct: direct, - Database: uinfo.db, - Username: uinfo.user, - Password: uinfo.pass, - Mechanism: mechanism, - Service: service, - Source: source, - PoolLimit: poolLimit, + Addrs: uinfo.addrs, + Direct: direct, + Database: uinfo.db, + Username: uinfo.user, + Password: uinfo.pass, + Mechanism: mechanism, + Service: service, + Source: source, + PoolLimit: poolLimit, + AppName: appName, + ReadPreference: &ReadPreference{ + Mode: readPreferenceMode, + TagSets: readPreferenceTagSets, + }, ReplicaSetName: setName, + MinPoolSize: minPoolSize, + MaxIdleTimeMS: maxIdleTimeMS, } return &info, nil } @@ -328,24 +443,12 @@ type DialInfo struct { // Addrs holds the addresses for the seed servers. Addrs []string - // Direct informs whether to establish connections only with the - // specified seed servers, or to obtain information for the whole - // cluster and establish connections with further servers too. - Direct bool - // Timeout is the amount of time to wait for a server to respond when // first connecting and on follow up operations in the session. If // timeout is zero, the call may block forever waiting for a connection // to be established. Timeout does not affect logic in DialServer. Timeout time.Duration - // FailFast will cause connection and query attempts to fail faster when - // the server is unavailable, instead of retrying until the configured - // timeout period. Note that an unavailable server may silently drop - // packets instead of rejecting them, in which case it's impossible to - // distinguish it from a slow server, so the timeout stays relevant. - FailFast bool - // Database is the default database name used when the Session.DB method // is called with an empty name, and is also used during the initial // authentication if Source is unset. @@ -384,6 +487,38 @@ type DialInfo struct { // See Session.SetPoolLimit for details. PoolLimit int + // PoolTimeout defines max time to wait for a connection to become available + // if the pool limit is reaqched. Defaults to zero, which means forever. + // See Session.SetPoolTimeout for details + PoolTimeout time.Duration + + // The identifier of the client application which ran the operation. + AppName string + + // ReadPreference defines the manner in which servers are chosen. See + // Session.SetMode and Session.SelectServers. + ReadPreference *ReadPreference + + // FailFast will cause connection and query attempts to fail faster when + // the server is unavailable, instead of retrying until the configured + // timeout period. Note that an unavailable server may silently drop + // packets instead of rejecting them, in which case it's impossible to + // distinguish it from a slow server, so the timeout stays relevant. + FailFast bool + + // Direct informs whether to establish connections only with the + // specified seed servers, or to obtain information for the whole + // cluster and establish connections with further servers too. + Direct bool + + // MinPoolSize defines The minimum number of connections in the connection pool. + // Defaults to 0. + MinPoolSize int + + //The maximum number of milliseconds that a connection can remain idle in the pool + // before being removed and closed. + MaxIdleTimeMS int + // DialServer optionally specifies the dial function for establishing // connections with the MongoDB servers. DialServer func(addr *ServerAddr) (net.Conn, error) @@ -392,6 +527,15 @@ type DialInfo struct { Dial func(addr net.Addr) (net.Conn, error) } +// ReadPreference defines the manner in which servers are chosen. +type ReadPreference struct { + // Mode determines the consistency of results. See Session.SetMode. + Mode Mode + + // TagSets indicates which servers are allowed to be used. See Session.SelectServers. + TagSets []bson.D +} + // mgo.v3: Drop DialInfo.Dial. // ServerAddr represents the address for establishing a connection to an @@ -422,7 +566,7 @@ func DialWithInfo(info *DialInfo) (*Session, error) { } addrs[i] = addr } - cluster := newCluster(addrs, info.Direct, info.FailFast, dialer{info.Dial, info.DialServer}, info.ReplicaSetName) + cluster := newCluster(addrs, info.Direct, info.FailFast, dialer{info.Dial, info.DialServer}, info.ReplicaSetName, info.AppName) session := newSession(Eventual, cluster, info.Timeout) session.defaultdb = info.Database if session.defaultdb == "" { @@ -454,6 +598,14 @@ func DialWithInfo(info *DialInfo) (*Session, error) { if info.PoolLimit > 0 { session.poolLimit = info.PoolLimit } + + cluster.minPoolSize = info.MinPoolSize + cluster.maxIdleTimeMS = info.MaxIdleTimeMS + + if info.PoolTimeout > 0 { + session.poolTimeout = info.PoolTimeout + } + cluster.Release() // People get confused when we return a session that is not actually @@ -464,7 +616,14 @@ func DialWithInfo(info *DialInfo) (*Session, error) { session.Close() return nil, err } - session.SetMode(Strong, true) + + if info.ReadPreference != nil { + session.SelectServers(info.ReadPreference.TagSets...) + session.SetMode(info.ReadPreference.Mode, true) + } else { + session.SetMode(Strong, true) + } + return session, nil } @@ -477,21 +636,25 @@ type urlInfo struct { user string pass string db string - options map[string]string + options []urlInfoOption +} + +type urlInfoOption struct { + key string + value string } func extractURL(s string) (*urlInfo, error) { - if strings.HasPrefix(s, "mongodb://") { - s = s[10:] - } - info := &urlInfo{options: make(map[string]string)} + s = strings.TrimPrefix(s, "mongodb://") + info := &urlInfo{options: []urlInfoOption{}} + if c := strings.Index(s, "?"); c != -1 { for _, pair := range strings.FieldsFunc(s[c+1:], isOptSep) { l := strings.SplitN(pair, "=", 2) if len(l) != 2 || l[0] == "" || l[1] == "" { return nil, errors.New("connection option must be key=value: " + pair) } - info.options[l[0]] = l[1] + info.options = append(info.options, urlInfoOption{key: l[0], value: l[1]}) } s = s[:c] } @@ -524,7 +687,7 @@ func extractURL(s string) (*urlInfo, error) { func newSession(consistency Mode, cluster *mongoCluster, timeout time.Duration) (session *Session) { cluster.Acquire() session = &Session{ - cluster_: cluster, + mgoCluster: cluster, syncTimeout: timeout, sockTimeout: timeout, poolLimit: 4096, @@ -552,9 +715,25 @@ func copySession(session *Session, keepCreds bool) (s *Session) { } else if session.dialCred != nil { creds = []Credential{*session.dialCred} } - scopy := *session - scopy.m = sync.RWMutex{} - scopy.creds = creds + scopy := Session{ + defaultdb: session.defaultdb, + sourcedb: session.sourcedb, + syncTimeout: session.syncTimeout, + sockTimeout: session.sockTimeout, + poolLimit: session.poolLimit, + poolTimeout: session.poolTimeout, + consistency: session.consistency, + creds: creds, + dialCred: session.dialCred, + safeOp: session.safeOp, + mgoCluster: session.mgoCluster, + slaveSocket: session.slaveSocket, + masterSocket: session.masterSocket, + m: sync.RWMutex{}, + queryConfig: session.queryConfig, + bypassValidation: session.bypassValidation, + slaveOk: session.slaveOk, + } s = &scopy debugf("New session %p on cluster %p (copy from %p)", s, cluster, session) return s @@ -591,6 +770,30 @@ func (db *Database) C(name string) *Collection { return &Collection{db, name, db.Name + "." + name} } +// CreateView creates a view as the result of the applying the specified +// aggregation pipeline to the source collection or view. Views act as +// read-only collections, and are computed on demand during read operations. +// MongoDB executes read operations on views as part of the underlying aggregation pipeline. +// +// For example: +// +// db := session.DB("mydb") +// db.CreateView("myview", "mycoll", []bson.M{{"$match": bson.M{"c": 1}}}, nil) +// view := db.C("myview") +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/core/views/ +// https://docs.mongodb.com/manual/reference/method/db.createView/ +// +func (db *Database) CreateView(view string, source string, pipeline interface{}, collation *Collation) error { + command := bson.D{{Name: "create", Value: view}, {Name: "viewOn", Value: source}, {Name: "pipeline", Value: pipeline}} + if collation != nil { + command = append(command, bson.DocElem{Name: "collation", Value: collation}) + } + return db.Run(command, nil) +} + // With returns a copy of db that uses session s. func (db *Database) With(s *Session) *Database { newdb := *db @@ -656,6 +859,15 @@ func (db *Database) Run(cmd interface{}, result interface{}) error { return db.run(socket, cmd, result) } +// runOnSocket does the same as Run, but guarantees that your command will be run +// on the provided socket instance; if it's unhealthy, you will receive the error +// from it. +func (db *Database) runOnSocket(socket *mongoSocket, cmd interface{}, result interface{}) error { + socket.Acquire() + defer socket.Release() + return db.run(socket, cmd, result) +} + // Credential holds details to authenticate with a MongoDB server. type Credential struct { // Username and Password hold the basic details for authentication. @@ -680,6 +892,14 @@ type Credential struct { // Mechanism defines the protocol for credential negotiation. // Defaults to "MONGODB-CR". Mechanism string + + // Certificate sets the x509 certificate for authentication, see: + // + // https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/ + // + // If using certificate authentication the Username, Mechanism and Source + // fields should not be set. + Certificate *x509.Certificate } // Login authenticates with MongoDB using the provided credential. The @@ -702,6 +922,19 @@ func (s *Session) Login(cred *Credential) error { defer socket.Release() credCopy := *cred + if cred.Certificate != nil && cred.Username != "" { + return errors.New("failed to login, both certificate and credentials are given") + } + + if cred.Certificate != nil { + credCopy.Username, err = getRFC2253NameStringFromCert(cred.Certificate) + if err != nil { + return err + } + credCopy.Mechanism = "MONGODB-X509" + credCopy.Source = "$external" + } + if cred.Source == "" { if cred.Mechanism == "GSSAPI" { credCopy.Source = "$external" @@ -811,23 +1044,51 @@ type User struct { UserSource string `bson:"userSource,omitempty"` } +// Role available role for users +// +// Relevant documentation: +// +// http://docs.mongodb.org/manual/reference/user-privileges/ +// type Role string const ( - // Relevant documentation: - // - // http://docs.mongodb.org/manual/reference/user-privileges/ - // - RoleRoot Role = "root" - RoleRead Role = "read" - RoleReadAny Role = "readAnyDatabase" - RoleReadWrite Role = "readWrite" + // RoleRoot provides access to the operations and all the resources + // of the readWriteAnyDatabase, dbAdminAnyDatabase, userAdminAnyDatabase, + // clusterAdmin roles, restore, and backup roles combined. + RoleRoot Role = "root" + // RoleRead provides the ability to read data on all non-system collections + // and on the following system collections: system.indexes, system.js, and + // system.namespaces collections on a specific database. + RoleRead Role = "read" + // RoleReadAny provides the same read-only permissions as read, except it + // applies to it applies to all but the local and config databases in the cluster. + // The role also provides the listDatabases action on the cluster as a whole. + RoleReadAny Role = "readAnyDatabase" + //RoleReadWrite provides all the privileges of the read role plus ability to modify data on + //all non-system collections and the system.js collection on a specific database. + RoleReadWrite Role = "readWrite" + // RoleReadWriteAny provides the same read and write permissions as readWrite, except it + // applies to all but the local and config databases in the cluster. The role also provides + // the listDatabases action on the cluster as a whole. RoleReadWriteAny Role = "readWriteAnyDatabase" - RoleDBAdmin Role = "dbAdmin" - RoleDBAdminAny Role = "dbAdminAnyDatabase" - RoleUserAdmin Role = "userAdmin" + // RoleDBAdmin provides all the privileges of the dbAdmin role on a specific database + RoleDBAdmin Role = "dbAdmin" + // RoleDBAdminAny provides all the privileges of the dbAdmin role on all databases + RoleDBAdminAny Role = "dbAdminAnyDatabase" + // RoleUserAdmin Provides the ability to create and modify roles and users on the + // current database. This role also indirectly provides superuser access to either + // the database or, if scoped to the admin database, the cluster. The userAdmin role + // allows users to grant any user any privilege, including themselves. + RoleUserAdmin Role = "userAdmin" + // RoleUserAdminAny provides the same access to user administration operations as userAdmin, + // except it applies to all but the local and config databases in the cluster RoleUserAdminAny Role = "userAdminAnyDatabase" + // RoleClusterAdmin Provides the greatest cluster-management access. This role combines + // the privileges granted by the clusterManager, clusterMonitor, and hostManager roles. + // Additionally, the role provides the dropDatabase action. RoleClusterAdmin Role = "clusterAdmin" + // TODO some roles are missing: dbOwner/clusterManager/clusterMonitor/hostManager/backup/restore ) // UpsertUser updates the authentication credentials and the roles for @@ -873,32 +1134,32 @@ func (db *Database) UpsertUser(user *User) error { if user.Password != "" { psum := md5.New() psum.Write([]byte(user.Username + ":mongo:" + user.Password)) - set = append(set, bson.DocElem{"pwd", hex.EncodeToString(psum.Sum(nil))}) - unset = append(unset, bson.DocElem{"userSource", 1}) + set = append(set, bson.DocElem{Name: "pwd", Value: hex.EncodeToString(psum.Sum(nil))}) + unset = append(unset, bson.DocElem{Name: "userSource", Value: 1}) } else if user.PasswordHash != "" { - set = append(set, bson.DocElem{"pwd", user.PasswordHash}) - unset = append(unset, bson.DocElem{"userSource", 1}) + set = append(set, bson.DocElem{Name: "pwd", Value: user.PasswordHash}) + unset = append(unset, bson.DocElem{Name: "userSource", Value: 1}) } if user.UserSource != "" { - set = append(set, bson.DocElem{"userSource", user.UserSource}) - unset = append(unset, bson.DocElem{"pwd", 1}) + set = append(set, bson.DocElem{Name: "userSource", Value: user.UserSource}) + unset = append(unset, bson.DocElem{Name: "pwd", Value: 1}) } if user.Roles != nil || user.OtherDBRoles != nil { - set = append(set, bson.DocElem{"roles", user.Roles}) + set = append(set, bson.DocElem{Name: "roles", Value: user.Roles}) if len(user.OtherDBRoles) > 0 { - set = append(set, bson.DocElem{"otherDBRoles", user.OtherDBRoles}) + set = append(set, bson.DocElem{Name: "otherDBRoles", Value: user.OtherDBRoles}) } else { - unset = append(unset, bson.DocElem{"otherDBRoles", 1}) + unset = append(unset, bson.DocElem{Name: "otherDBRoles", Value: 1}) } } users := db.C("system.users") - err = users.Update(bson.D{{"user", user.Username}}, bson.D{{"$unset", unset}, {"$set", set}}) + err = users.Update(bson.D{{Name: "user", Value: user.Username}}, bson.D{{Name: "$unset", Value: unset}, {Name: "$set", Value: set}}) if err == ErrNotFound { - set = append(set, bson.DocElem{"user", user.Username}) + set = append(set, bson.DocElem{Name: "user", Value: user.Username}) if user.Roles == nil && user.OtherDBRoles == nil { // Roles must be sent, as it's the way MongoDB distinguishes // old-style documents from new-style documents in pre-2.6. - set = append(set, bson.DocElem{"roles", user.Roles}) + set = append(set, bson.DocElem{Name: "roles", Value: user.Roles}) } err = users.Insert(set) } @@ -920,11 +1181,16 @@ func isAuthError(err error) bool { return ok && e.Code == 13 } +func isNotMasterError(err error) bool { + e, ok := err.(*QueryError) + return ok && strings.Contains(e.Message, "not master") +} + func (db *Database) runUserCmd(cmdName string, user *User) error { cmd := make(bson.D, 0, 16) - cmd = append(cmd, bson.DocElem{cmdName, user.Username}) + cmd = append(cmd, bson.DocElem{Name: cmdName, Value: user.Username}) if user.Password != "" { - cmd = append(cmd, bson.DocElem{"pwd", user.Password}) + cmd = append(cmd, bson.DocElem{Name: "pwd", Value: user.Password}) } var roles []interface{} for _, role := range user.Roles { @@ -932,11 +1198,11 @@ func (db *Database) runUserCmd(cmdName string, user *User) error { } for db, dbroles := range user.OtherDBRoles { for _, role := range dbroles { - roles = append(roles, bson.D{{"role", role}, {"db", db}}) + roles = append(roles, bson.D{{Name: "role", Value: role}, {Name: "db", Value: db}}) } } if roles != nil || user.Roles != nil || cmdName == "createUser" { - cmd = append(cmd, bson.DocElem{"roles", roles}) + cmd = append(cmd, bson.DocElem{Name: "roles", Value: roles}) } err := db.Run(cmd, nil) if !isNoCmd(err) && user.UserSource != "" && (user.UserSource != "$external" || db.Name != "$external") { @@ -985,7 +1251,7 @@ func (db *Database) AddUser(username, password string, readOnly bool) error { // RemoveUser removes the authentication credentials of user from the database. func (db *Database) RemoveUser(user string) error { - err := db.Run(bson.D{{"dropUser", user}}, nil) + err := db.Run(bson.D{{Name: "dropUser", Value: user}}, nil) if isNoCmd(err) { users := db.C("system.users") return users.Remove(bson.M{"user": user}) @@ -997,30 +1263,37 @@ func (db *Database) RemoveUser(user string) error { } type indexSpec struct { - Name, NS string - Key bson.D - Unique bool ",omitempty" - DropDups bool "dropDups,omitempty" - Background bool ",omitempty" - Sparse bool ",omitempty" - Bits int ",omitempty" - Min, Max float64 ",omitempty" - BucketSize float64 "bucketSize,omitempty" - ExpireAfter int "expireAfterSeconds,omitempty" - Weights bson.D ",omitempty" - DefaultLanguage string "default_language,omitempty" - LanguageOverride string "language_override,omitempty" - TextIndexVersion int "textIndexVersion,omitempty" - - Collation *Collation "collation,omitempty" -} - + Name, NS string + Key bson.D + Unique bool `bson:",omitempty"` + DropDups bool `bson:"dropDups,omitempty"` + Background bool `bson:",omitempty"` + Sparse bool `bson:",omitempty"` + Bits int `bson:",omitempty"` + Min, Max float64 `bson:",omitempty"` + BucketSize float64 `bson:"bucketSize,omitempty"` + ExpireAfter int `bson:"expireAfterSeconds,omitempty"` + Weights bson.D `bson:",omitempty"` + DefaultLanguage string `bson:"default_language,omitempty"` + LanguageOverride string `bson:"language_override,omitempty"` + TextIndexVersion int `bson:"textIndexVersion,omitempty"` + PartialFilterExpression bson.M `bson:"partialFilterExpression,omitempty"` + + Collation *Collation `bson:"collation,omitempty"` +} + +// Index are special data structures that store a small portion of the collection’s +// data set in an easy to traverse form. The index stores the value of a specific +// field or set of fields, ordered by the value of the field. The ordering of the +// index entries supports efficient equality matches and range-based query operations. +// In addition, MongoDB can return sorted results by using the ordering in the index. type Index struct { - Key []string // Index key fields; prefix name with dash (-) for descending order - Unique bool // Prevent two documents from having the same index key - DropDups bool // Drop documents with the same index key as a previously indexed one - Background bool // Build index in background and return immediately - Sparse bool // Only index documents containing the Key fields + Key []string // Index key fields; prefix name with dash (-) for descending order + Unique bool // Prevent two documents from having the same index key + DropDups bool // Drop documents with the same index key as a previously indexed one + Background bool // Build index in background and return immediately + Sparse bool // Only index documents containing the Key fields + PartialFilter bson.M // Partial index filter expression // If ExpireAfter is defined the server will periodically delete // documents with indexed time.Time older than the provided delta. @@ -1056,14 +1329,13 @@ type Index struct { Collation *Collation } +// Collation allows users to specify language-specific rules for string comparison, +// such as rules for lettercase and accent marks. type Collation struct { // Locale defines the collation locale. Locale string `bson:"locale"` - // CaseLevel defines whether to turn case sensitivity on at strength 1 or 2. - CaseLevel bool `bson:"caseLevel,omitempty"` - // CaseFirst may be set to "upper" or "lower" to define whether // to have uppercase or lowercase items first. Default is "off". CaseFirst string `bson:"caseFirst,omitempty"` @@ -1086,16 +1358,27 @@ type Collation struct { // Strength defaults to 3. Strength int `bson:"strength,omitempty"` - // NumericOrdering defines whether to order numbers based on numerical - // order and not collation order. - NumericOrdering bool `bson:"numericOrdering,omitempty"` - // Alternate controls whether spaces and punctuation are considered base characters. // May be set to "non-ignorable" (spaces and punctuation considered base characters) // or "shifted" (spaces and punctuation not considered base characters, and only // distinguished at strength > 3). Defaults to "non-ignorable". Alternate string `bson:"alternate,omitempty"` + // MaxVariable defines which characters are affected when the value for Alternate is + // "shifted". It may be set to "punct" to affect punctuation or spaces, or "space" to + // affect only spaces. + MaxVariable string `bson:"maxVariable,omitempty"` + + // Normalization defines whether text is normalized into Unicode NFD. + Normalization bool `bson:"normalization,omitempty"` + + // CaseLevel defines whether to turn case sensitivity on at strength 1 or 2. + CaseLevel bool `bson:"caseLevel,omitempty"` + + // NumericOrdering defines whether to order numbers based on numerical + // order and not collation order. + NumericOrdering bool `bson:"numericOrdering,omitempty"` + // Backwards defines whether to have secondary differences considered in reverse order, // as done in the French language. Backwards bool `bson:"backwards,omitempty"` @@ -1162,12 +1445,12 @@ func parseIndexKey(key []string) (*indexKeyInfo, error) { } if kind == "text" { if !isText { - keyInfo.key = append(keyInfo.key, bson.DocElem{"_fts", "text"}, bson.DocElem{"_ftsx", 1}) + keyInfo.key = append(keyInfo.key, bson.DocElem{Name: "_fts", Value: "text"}, bson.DocElem{Name: "_ftsx", Value: 1}) isText = true } - keyInfo.weights = append(keyInfo.weights, bson.DocElem{field, 1}) + keyInfo.weights = append(keyInfo.weights, bson.DocElem{Name: field, Value: 1}) } else { - keyInfo.key = append(keyInfo.key, bson.DocElem{field, order}) + keyInfo.key = append(keyInfo.key, bson.DocElem{Name: field, Value: order}) } } if keyInfo.name == "" { @@ -1265,6 +1548,10 @@ func (c *Collection) EnsureIndexKey(key ...string) error { // http://www.mongodb.org/display/DOCS/Multikeys // func (c *Collection) EnsureIndex(index Index) error { + if index.Sparse && index.PartialFilter != nil { + return errors.New("cannot mix sparse and partial indexes") + } + keyInfo, err := parseIndexKey(index.Key) if err != nil { return err @@ -1277,22 +1564,23 @@ func (c *Collection) EnsureIndex(index Index) error { } spec := indexSpec{ - Name: keyInfo.name, - NS: c.FullName, - Key: keyInfo.key, - Unique: index.Unique, - DropDups: index.DropDups, - Background: index.Background, - Sparse: index.Sparse, - Bits: index.Bits, - Min: index.Minf, - Max: index.Maxf, - BucketSize: index.BucketSize, - ExpireAfter: int(index.ExpireAfter / time.Second), - Weights: keyInfo.weights, - DefaultLanguage: index.DefaultLanguage, - LanguageOverride: index.LanguageOverride, - Collation: index.Collation, + Name: keyInfo.name, + NS: c.FullName, + Key: keyInfo.key, + Unique: index.Unique, + DropDups: index.DropDups, + Background: index.Background, + Sparse: index.Sparse, + Bits: index.Bits, + Min: index.Minf, + Max: index.Maxf, + BucketSize: index.BucketSize, + ExpireAfter: int(index.ExpireAfter / time.Second), + Weights: keyInfo.weights, + DefaultLanguage: index.DefaultLanguage, + LanguageOverride: index.LanguageOverride, + Collation: index.Collation, + PartialFilterExpression: index.PartialFilter, } if spec.Min == 0 && spec.Max == 0 { @@ -1322,7 +1610,7 @@ NextField: db := c.Database.With(cloned) // Try with a command first. - err = db.Run(bson.D{{"createIndexes", c.Name}, {"indexes", []indexSpec{spec}}}, nil) + err = db.Run(bson.D{{Name: "createIndexes", Value: c.Name}, {Name: "indexes", Value: []indexSpec{spec}}}, nil) if isNoCmd(err) { // Command not yet supported. Insert into the indexes collection instead. err = db.C("system.indexes").Insert(&spec) @@ -1361,7 +1649,7 @@ func (c *Collection) DropIndex(key ...string) error { ErrMsg string Ok bool }{} - err = db.Run(bson.D{{"dropIndexes", c.Name}, {"index", keyInfo.name}}, &result) + err = db.Run(bson.D{{Name: "dropIndexes", Value: c.Name}, {Name: "index", Value: keyInfo.name}}, &result) if err != nil { return err } @@ -1413,7 +1701,30 @@ func (c *Collection) DropIndexName(name string) error { ErrMsg string Ok bool }{} - err = c.Database.Run(bson.D{{"dropIndexes", c.Name}, {"index", name}}, &result) + err = c.Database.Run(bson.D{{Name: "dropIndexes", Value: c.Name}, {Name: "index", Value: name}}, &result) + if err != nil { + return err + } + if !result.Ok { + return errors.New(result.ErrMsg) + } + return nil +} + +// DropAllIndexes drops all the indexes from the c collection +func (c *Collection) DropAllIndexes() error { + session := c.Database.Session + session.ResetIndexCache() + + session = session.Clone() + defer session.Close() + + db := c.Database.With(session) + result := struct { + ErrMsg string + Ok bool + }{} + err := db.Run(bson.D{{Name: "dropIndexes", Value: c.Name}, {Name: "index", Value: "*"}}, &result) if err != nil { return err } @@ -1426,8 +1737,8 @@ func (c *Collection) DropIndexName(name string) error { // nonEventual returns a clone of session and ensures it is not Eventual. // This guarantees that the server that is used for queries may be reused // afterwards when a cursor is received. -func (session *Session) nonEventual() *Session { - cloned := session.Clone() +func (s *Session) nonEventual() *Session { + cloned := s.Clone() if cloned.consistency == Eventual { cloned.SetMode(Monotonic, false) } @@ -1436,19 +1747,6 @@ func (session *Session) nonEventual() *Session { // Indexes returns a list of all indexes for the collection. // -// For example, this snippet would drop all available indexes: -// -// indexes, err := collection.Indexes() -// if err != nil { -// return err -// } -// for _, index := range indexes { -// err = collection.DropIndex(index.Key...) -// if err != nil { -// return err -// } -// } -// // See the EnsureIndex method for more details on indexes. func (c *Collection) Indexes() (indexes []Index, err error) { cloned := c.Database.Session.nonEventual() @@ -1462,7 +1760,7 @@ func (c *Collection) Indexes() (indexes []Index, err error) { Cursor cursorData } var iter *Iter - err = c.Database.With(cloned).Run(bson.D{{"listIndexes", c.Name}, {"cursor", bson.D{{"batchSize", batchSize}}}}, &result) + err = c.Database.With(cloned).Run(bson.D{{Name: "listIndexes", Value: c.Name}, {Name: "cursor", Value: bson.D{{Name: "batchSize", Value: batchSize}}}}, &result) if err == nil { firstBatch := result.Indexes if firstBatch == nil { @@ -1508,6 +1806,7 @@ func indexFromSpec(spec indexSpec) Index { LanguageOverride: spec.LanguageOverride, ExpireAfter: time.Duration(spec.ExpireAfter) * time.Second, Collation: spec.Collation, + PartialFilter: spec.PartialFilterExpression, } if float64(int(spec.Min)) == spec.Min && float64(int(spec.Max)) == spec.Max { index.Min = int(spec.Min) @@ -1534,12 +1833,25 @@ func (idxs indexSlice) Swap(i, j int) { idxs[i], idxs[j] = idxs[j], idxs[i] func simpleIndexKey(realKey bson.D) (key []string) { for i := range realKey { + var vi int field := realKey[i].Name - vi, ok := realKey[i].Value.(int) - if !ok { + + switch realKey[i].Value.(type) { + case int64: + vf, _ := realKey[i].Value.(int64) + vi = int(vf) + case float64: vf, _ := realKey[i].Value.(float64) vi = int(vf) + case string: + if vs, ok := realKey[i].Value.(string); ok { + key = append(key, "$"+vs+":"+field) + continue + } + case int: + vi = realKey[i].Value.(int) } + if vi == 1 { key = append(key, field) continue @@ -1548,16 +1860,12 @@ func simpleIndexKey(realKey bson.D) (key []string) { key = append(key, "-"+field) continue } - if vs, ok := realKey[i].Value.(string); ok { - key = append(key, "$"+vs+":"+field) - continue - } panic("Got unknown index key type for field " + field) } return } -// ResetIndexCache() clears the cache of previously ensured indexes. +// ResetIndexCache clears the cache of previously ensured indexes. // Following requests to EnsureIndex will contact the server. func (s *Session) ResetIndexCache() { s.cluster().ResetIndexCache() @@ -1610,20 +1918,20 @@ func (s *Session) Clone() *Session { // after it has been closed. func (s *Session) Close() { s.m.Lock() - if s.cluster_ != nil { + if s.mgoCluster != nil { debugf("Closing session %p", s) s.unsetSocket() - s.cluster_.Release() - s.cluster_ = nil + s.mgoCluster.Release() + s.mgoCluster = nil } s.m.Unlock() } func (s *Session) cluster() *mongoCluster { - if s.cluster_ == nil { + if s.mgoCluster == nil { panic("Session already closed") } - return s.cluster_ + return s.mgoCluster } // Refresh puts back any reserved sockets in use and restarts the consistency @@ -1754,6 +2062,16 @@ func (s *Session) SetPoolLimit(limit int) { s.m.Unlock() } +// SetPoolTimeout sets the maxinum time connection attempts will wait to reuse +// an existing connection from the pool if the PoolLimit has been reached. If +// the value is exceeded, the attempt to use a session will fail with an error. +// The default value is zero, which means to wait forever with no timeout. +func (s *Session) SetPoolTimeout(timeout time.Duration) { + s.m.Lock() + s.poolTimeout = timeout + s.m.Unlock() +} + // SetBypassValidation sets whether the server should bypass the registered // validation expressions executed when documents are inserted or modified, // in the interest of preserving invariants in the collection being modified. @@ -1808,10 +2126,11 @@ func (s *Session) SetPrefetch(p float64) { s.m.Unlock() } -// See SetSafe for details on the Safe type. +// Safe session safety mode. See SetSafe for details on the Safe type. type Safe struct { W int // Min # of servers to ack before success WMode string // Write mode for MongoDB 2.0+ (e.g. "majority") + RMode string // Read mode for MonogDB 3.2+ ("majority", "local", "linearizable") WTimeout int // Milliseconds to wait for W before timing out FSync bool // Sync via the journal if present, or via data files sync otherwise J bool // Sync via the journal if present @@ -1823,7 +2142,7 @@ func (s *Session) Safe() (safe *Safe) { defer s.m.Unlock() if s.safeOp != nil { cmd := s.safeOp.query.(*getLastError) - safe = &Safe{WTimeout: cmd.WTimeout, FSync: cmd.FSync, J: cmd.J} + safe = &Safe{WTimeout: cmd.WTimeout, FSync: cmd.FSync, J: cmd.J, RMode: s.queryConfig.op.readConcern} switch w := cmd.W.(type) { case string: safe.WMode = w @@ -1903,6 +2222,7 @@ func (s *Session) Safe() (safe *Safe) { // // Relevant documentation: // +// https://docs.mongodb.com/manual/reference/read-concern/ // http://www.mongodb.org/display/DOCS/getLastError+Command // http://www.mongodb.org/display/DOCS/Verifying+Propagation+of+Writes+with+getLastError // http://www.mongodb.org/display/DOCS/Data+Center+Awareness @@ -1921,6 +2241,7 @@ func (s *Session) SetSafe(safe *Safe) { // That is: // // - safe.WMode is always used if set. +// - safe.RMode is always used if set. // - safe.W is used if larger than the current W and WMode is empty. // - safe.FSync is always used if true. // - safe.J is used if FSync is false. @@ -1959,6 +2280,13 @@ func (s *Session) ensureSafe(safe *Safe) { w = safe.W } + // Set the read concern + switch safe.RMode { + case "majority", "local", "linearizable": + s.queryConfig.op.readConcern = safe.RMode + default: + } + var cmd getLastError if s.safeOp == nil { cmd = getLastError{1, w, safe.WTimeout, safe.FSync, safe.J} @@ -2014,6 +2342,13 @@ func (s *Session) Run(cmd interface{}, result interface{}) error { return s.DB("admin").Run(cmd, result) } +// runOnSocket does the same as Run, but guarantees that your command will be run +// on the provided socket instance; if it's unhealthy, you will receive the error +// from it. +func (s *Session) runOnSocket(socket *mongoSocket, cmd interface{}, result interface{}) error { + return s.DB("admin").runOnSocket(socket, cmd, result) +} + // SelectServers restricts communication to servers configured with the // given tags. For example, the following statement restricts servers // used for reading operations to those with both tag "disk" set to @@ -2047,7 +2382,7 @@ func (s *Session) Ping() error { // is established with. If async is true, the call returns immediately, // otherwise it returns after the flush has been made. func (s *Session) Fsync(async bool) error { - return s.Run(bson.D{{"fsync", 1}, {"async", async}}, nil) + return s.Run(bson.D{{Name: "fsync", Value: 1}, {Name: "async", Value: async}}, nil) } // FsyncLock locks all writes in the specific server the session is @@ -2076,12 +2411,12 @@ func (s *Session) Fsync(async bool) error { // http://www.mongodb.org/display/DOCS/Backups // func (s *Session) FsyncLock() error { - return s.Run(bson.D{{"fsync", 1}, {"lock", true}}, nil) + return s.Run(bson.D{{Name: "fsync", Value: 1}, {Name: "lock", Value: true}}, nil) } // FsyncUnlock releases the server for writes. See FsyncLock for details. func (s *Session) FsyncUnlock() error { - err := s.Run(bson.D{{"fsyncUnlock", 1}}, nil) + err := s.Run(bson.D{{Name: "fsyncUnlock", Value: 1}}, nil) if isNoCmd(err) { err = s.DB("admin").C("$cmd.sys.unlock").Find(nil).One(nil) // WTF? } @@ -2122,7 +2457,7 @@ func (c *Collection) Find(query interface{}) *Query { type repairCmd struct { RepairCursor string `bson:"repairCursor"` - Cursor *repairCmdCursor ",omitempty" + Cursor *repairCmdCursor `bson:",omitempty"` } type repairCmdCursor struct { @@ -2163,23 +2498,29 @@ func (c *Collection) Repair() *Iter { // // See the Find method for more details. func (c *Collection) FindId(id interface{}) *Query { - return c.Find(bson.D{{"_id", id}}) + return c.Find(bson.D{{Name: "_id", Value: id}}) } +// Pipe is used to run aggregation queries against a +// collection. type Pipe struct { session *Session collection *Collection pipeline interface{} allowDisk bool batchSize int + maxTimeMS int64 + collation *Collation } type pipeCmd struct { Aggregate string Pipeline interface{} - Cursor *pipeCmdCursor ",omitempty" - Explain bool ",omitempty" - AllowDisk bool "allowDiskUse,omitempty" + Cursor *pipeCmdCursor `bson:",omitempty"` + Explain bool `bson:",omitempty"` + AllowDisk bool `bson:"allowDiskUse,omitempty"` + MaxTimeMS int64 `bson:"maxTimeMS,omitempty"` + Collation *Collation `bson:"collation,omitempty"` } type pipeCmdCursor struct { @@ -2200,6 +2541,7 @@ type pipeCmdCursor struct { // http://docs.mongodb.org/manual/applications/aggregation // http://docs.mongodb.org/manual/tutorial/aggregation-examples // + func (c *Collection) Pipe(pipeline interface{}) *Pipe { session := c.Database.Session session.m.RLock() @@ -2233,6 +2575,10 @@ func (p *Pipe) Iter() *Iter { Pipeline: p.pipeline, AllowDisk: p.allowDisk, Cursor: &pipeCmdCursor{p.batchSize}, + Collation: p.collation, + } + if p.maxTimeMS > 0 { + cmd.MaxTimeMS = p.maxTimeMS } err := c.Database.Run(cmd, &result) if e, ok := err.(*QueryError); ok && e.Message == `unrecognized field "cursor` { @@ -2244,29 +2590,38 @@ func (p *Pipe) Iter() *Iter { if firstBatch == nil { firstBatch = result.Cursor.FirstBatch } - return c.NewIter(p.session, firstBatch, result.Cursor.Id, err) + it := c.NewIter(p.session, firstBatch, result.Cursor.Id, err) + if p.maxTimeMS > 0 { + it.maxTimeMS = p.maxTimeMS + } + return it } -// NewIter returns a newly created iterator with the provided parameters. -// Using this method is not recommended unless the desired functionality -// is not yet exposed via a more convenient interface (Find, Pipe, etc). +// NewIter returns a newly created iterator with the provided parameters. Using +// this method is not recommended unless the desired functionality is not yet +// exposed via a more convenient interface (Find, Pipe, etc). // // The optional session parameter associates the lifetime of the returned -// iterator to an arbitrary session. If nil, the iterator will be bound to -// c's session. +// iterator to an arbitrary session. If nil, the iterator will be bound to c's +// session. // // Documents in firstBatch will be individually provided by the returned -// iterator before documents from cursorId are made available. If cursorId -// is zero, only the documents in firstBatch are provided. +// iterator before documents from cursorId are made available. If cursorId is +// zero, only the documents in firstBatch are provided. // -// If err is not nil, the iterator's Err method will report it after -// exhausting documents in firstBatch. +// If err is not nil, the iterator's Err method will report it after exhausting +// documents in firstBatch. // -// NewIter must be called right after the cursor id is obtained, and must not -// be called on a collection in Eventual mode, because the cursor id is -// associated with the specific server that returned it. The provided session -// parameter may be in any mode or state, though. +// NewIter must not be called on a collection in Eventual mode, because the +// cursor id is associated with the specific server that returned it. The +// provided session parameter may be in any mode or state, though. // +// The new Iter fetches documents in batches of the server defined default, +// however this can be changed by setting the session Batch method. +// +// When using MongoDB 3.2+ NewIter supports re-using an existing cursor on the +// server. Ensure the connection has been established (i.e. by calling +// session.Ping()) before calling NewIter. func (c *Collection) NewIter(session *Session, firstBatch []bson.Raw, cursorId int64, err error) *Iter { var server *mongoServer csession := c.Database.Session @@ -2299,11 +2654,19 @@ func (c *Collection) NewIter(session *Session, firstBatch []bson.Raw, cursorId i timeout: -1, err: err, } + + if socket.ServerInfo().MaxWireVersion >= 4 && c.FullName != "admin.$cmd" { + iter.isFindCmd = true + } + iter.gotReply.L = &iter.m for _, doc := range firstBatch { iter.docData.Push(doc.Data) } if cursorId != 0 { + if socket != nil && socket.ServerInfo().MaxWireVersion >= 4 { + iter.docsBeforeMore = len(firstBatch) + } iter.op.cursorId = cursorId iter.op.collection = c.FullName iter.op.replyFunc = iter.replyFunc() @@ -2311,6 +2674,30 @@ func (c *Collection) NewIter(session *Session, firstBatch []bson.Raw, cursorId i return iter } +// State returns the current state of Iter. When combined with NewIter an +// existing cursor can be reused on Mongo 3.2+. Like NewIter, this method should +// be avoided if the desired functionality is exposed via a more convenient +// interface. +// +// Care must be taken to resume using Iter only when connected directly to the +// same server that the cursor was created on (with a Monotonic connection or +// with the connect=direct connection option). +func (iter *Iter) State() (int64, []bson.Raw) { + // Make a copy of the docData to avoid changing iter state + iter.m.Lock() + data := iter.docData + iter.m.Unlock() + + batch := make([]bson.Raw, 0, data.Len()) + for data.Len() > 0 { + batch = append(batch, bson.Raw{ + Kind: 0x00, + Data: data.Pop().([]byte), + }) + } + return iter.op.cursorId, batch +} + // All works like Iter.All. func (p *Pipe) All(result interface{}) error { return p.Iter().All(result) @@ -2371,8 +2758,36 @@ func (p *Pipe) Batch(n int) *Pipe { return p } -// mgo.v3: Use a single user-visible error type. +// SetMaxTime sets the maximum amount of time to allow the query to run. +// +func (p *Pipe) SetMaxTime(d time.Duration) *Pipe { + p.maxTimeMS = int64(d / time.Millisecond) + return p +} +// Collation allows to specify language-specific rules for string comparison, +// such as rules for lettercase and accent marks. +// When specifying collation, the locale field is mandatory; all other collation +// fields are optional +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/reference/collation/ +// +func (p *Pipe) Collation(collation *Collation) *Pipe { + if collation != nil { + p.collation = collation + } + return p +} + +// LastError the error status of the preceding write operation on the current connection. +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/reference/command/getLastError/ +// +// mgo.v3: Use a single user-visible error type. type LastError struct { Err string Code, N, Waited int @@ -2390,13 +2805,14 @@ func (err *LastError) Error() string { } type queryError struct { - Err string "$err" + Err string `bson:"$err"` ErrMsg string Assertion string Code int - AssertionCode int "assertionCode" + AssertionCode int `bson:"assertionCode"` } +// QueryError is returned when a query fails type QueryError struct { Code int Message string @@ -2471,7 +2887,7 @@ func (c *Collection) Update(selector interface{}, update interface{}) error { // // See the Update method for more details. func (c *Collection) UpdateId(id interface{}, update interface{}) error { - return c.Update(bson.D{{"_id", id}}, update) + return c.Update(bson.D{{Name: "_id", Value: id}}, update) } // ChangeInfo holds details about the outcome of an update operation. @@ -2566,7 +2982,7 @@ func (c *Collection) Upsert(selector interface{}, update interface{}) (info *Cha // // See the Upsert method for more details. func (c *Collection) UpsertId(id interface{}, update interface{}) (info *ChangeInfo, err error) { - return c.Upsert(bson.D{{"_id", id}}, update) + return c.Upsert(bson.D{{Name: "_id", Value: id}}, update) } // Remove finds a single document matching the provided selector document @@ -2596,7 +3012,7 @@ func (c *Collection) Remove(selector interface{}) error { // // See the Remove method for more details. func (c *Collection) RemoveId(id interface{}) error { - return c.Remove(bson.D{{"_id", id}}) + return c.Remove(bson.D{{Name: "_id", Value: id}}) } // RemoveAll finds all documents matching the provided selector document @@ -2621,12 +3037,12 @@ func (c *Collection) RemoveAll(selector interface{}) (info *ChangeInfo, err erro // DropDatabase removes the entire database including all of its collections. func (db *Database) DropDatabase() error { - return db.Run(bson.D{{"dropDatabase", 1}}, nil) + return db.Run(bson.D{{Name: "dropDatabase", Value: 1}}, nil) } // DropCollection removes the entire collection including all of its documents. func (c *Collection) DropCollection() error { - return c.Database.Run(bson.D{{"drop", c.Name}}, nil) + return c.Database.Run(bson.D{{Name: "drop", Value: c.Name}}, nil) } // The CollectionInfo type holds metadata about a collection. @@ -2676,6 +3092,10 @@ type CollectionInfo struct { // storage engine in use. The map keys must hold the storage engine // name for which options are being specified. StorageEngine interface{} + // Specifies the default collation for the collection. + // Collation allows users to specify language-specific rules for string + // comparison, such as rules for lettercase and accent marks. + Collation *Collation } // Create explicitly creates the c collection with details of info. @@ -2690,42 +3110,46 @@ type CollectionInfo struct { // func (c *Collection) Create(info *CollectionInfo) error { cmd := make(bson.D, 0, 4) - cmd = append(cmd, bson.DocElem{"create", c.Name}) + cmd = append(cmd, bson.DocElem{Name: "create", Value: c.Name}) if info.Capped { if info.MaxBytes < 1 { return fmt.Errorf("Collection.Create: with Capped, MaxBytes must also be set") } - cmd = append(cmd, bson.DocElem{"capped", true}) - cmd = append(cmd, bson.DocElem{"size", info.MaxBytes}) + cmd = append(cmd, bson.DocElem{Name: "capped", Value: true}) + cmd = append(cmd, bson.DocElem{Name: "size", Value: info.MaxBytes}) if info.MaxDocs > 0 { - cmd = append(cmd, bson.DocElem{"max", info.MaxDocs}) + cmd = append(cmd, bson.DocElem{Name: "max", Value: info.MaxDocs}) } } if info.DisableIdIndex { - cmd = append(cmd, bson.DocElem{"autoIndexId", false}) + cmd = append(cmd, bson.DocElem{Name: "autoIndexId", Value: false}) } if info.ForceIdIndex { - cmd = append(cmd, bson.DocElem{"autoIndexId", true}) + cmd = append(cmd, bson.DocElem{Name: "autoIndexId", Value: true}) } if info.Validator != nil { - cmd = append(cmd, bson.DocElem{"validator", info.Validator}) + cmd = append(cmd, bson.DocElem{Name: "validator", Value: info.Validator}) } if info.ValidationLevel != "" { - cmd = append(cmd, bson.DocElem{"validationLevel", info.ValidationLevel}) + cmd = append(cmd, bson.DocElem{Name: "validationLevel", Value: info.ValidationLevel}) } if info.ValidationAction != "" { - cmd = append(cmd, bson.DocElem{"validationAction", info.ValidationAction}) + cmd = append(cmd, bson.DocElem{Name: "validationAction", Value: info.ValidationAction}) } if info.StorageEngine != nil { - cmd = append(cmd, bson.DocElem{"storageEngine", info.StorageEngine}) + cmd = append(cmd, bson.DocElem{Name: "storageEngine", Value: info.StorageEngine}) + } + if info.Collation != nil { + cmd = append(cmd, bson.DocElem{Name: "collation", Value: info.Collation}) } + return c.Database.Run(cmd, nil) } // Batch sets the batch size used when fetching documents from the database. // It's possible to change this setting on a per-session basis as well, using // the Batch method of Session. - +// // The default batch size is defined by the database itself. As of this // writing, MongoDB will use an initial size of min(100 docs, 4MB) on the // first batch, and 4MB on remaining ones. @@ -2847,9 +3271,9 @@ func (q *Query) Sort(fields ...string) *Query { panic("Sort: empty field name") } if kind == "textScore" { - order = append(order, bson.DocElem{field, bson.M{"$meta": kind}}) + order = append(order, bson.DocElem{Name: field, Value: bson.M{"$meta": kind}}) } else { - order = append(order, bson.DocElem{field, n}) + order = append(order, bson.DocElem{Name: field, Value: n}) } } q.op.options.OrderBy = order @@ -2858,6 +3282,38 @@ func (q *Query) Sort(fields ...string) *Query { return q } +// Collation allows to specify language-specific rules for string comparison, +// such as rules for lettercase and accent marks. +// When specifying collation, the locale field is mandatory; all other collation +// fields are optional +// +// For example, to perform a case and diacritic insensitive query: +// +// var res []bson.M +// collation := &mgo.Collation{Locale: "en", Strength: 1} +// err = db.C("mycoll").Find(bson.M{"a": "a"}).Collation(collation).All(&res) +// if err != nil { +// return err +// } +// +// This query will match following documents: +// +// {"a": "a"} +// {"a": "A"} +// {"a": "â"} +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/reference/collation/ +// +func (q *Query) Collation(collation *Collation) *Query { + q.m.Lock() + q.op.options.Collation = collation + q.op.hasOptions = true + q.m.Unlock() + return q +} + // Explain returns a number of details about how the MongoDB server would // execute the requested query, such as the number of objects examined, // the number of times the read lock was yielded to allow writes to go in, @@ -3149,19 +3605,25 @@ func prepareFindOp(socket *mongoSocket, op *queryOp, limit int32) bool { } find := findCmd{ - Collection: op.collection[nameDot+1:], - Filter: op.query, - Projection: op.selector, - Sort: op.options.OrderBy, - Skip: op.skip, - Limit: limit, - MaxTimeMS: op.options.MaxTimeMS, - MaxScan: op.options.MaxScan, - Hint: op.options.Hint, - Comment: op.options.Comment, - Snapshot: op.options.Snapshot, - OplogReplay: op.flags&flagLogReplay != 0, + Collection: op.collection[nameDot+1:], + Filter: op.query, + Projection: op.selector, + Sort: op.options.OrderBy, + Skip: op.skip, + Limit: limit, + MaxTimeMS: op.options.MaxTimeMS, + MaxScan: op.options.MaxScan, + Hint: op.options.Hint, + Comment: op.options.Comment, + Snapshot: op.options.Snapshot, + Collation: op.options.Collation, + Tailable: op.flags&flagTailable != 0, + AwaitData: op.flags&flagAwaitData != 0, + OplogReplay: op.flags&flagLogReplay != 0, + NoCursorTimeout: op.flags&flagNoCursorTimeout != 0, + ReadConcern: readLevel{level: op.readConcern}, } + if op.limit < 0 { find.BatchSize = -op.limit find.SingleBatch = true @@ -3179,15 +3641,15 @@ func prepareFindOp(socket *mongoSocket, op *queryOp, limit int32) bool { op.hasOptions = false if explain { - op.query = bson.D{{"explain", op.query}} + op.query = bson.D{{Name: "explain", Value: op.query}} return false } return true } type cursorData struct { - FirstBatch []bson.Raw "firstBatch" - NextBatch []bson.Raw "nextBatch" + FirstBatch []bson.Raw `bson:"firstBatch"` + NextBatch []bson.Raw `bson:"nextBatch"` NS string Id int64 } @@ -3211,7 +3673,7 @@ type findCmd struct { Comment string `bson:"comment,omitempty"` MaxScan int `bson:"maxScan,omitempty"` MaxTimeMS int `bson:"maxTimeMS,omitempty"` - ReadConcern interface{} `bson:"readConcern,omitempty"` + ReadConcern readLevel `bson:"readConcern,omitempty"` Max interface{} `bson:"max,omitempty"` Min interface{} `bson:"min,omitempty"` ReturnKey bool `bson:"returnKey,omitempty"` @@ -3222,6 +3684,13 @@ type findCmd struct { OplogReplay bool `bson:"oplogReplay,omitempty"` NoCursorTimeout bool `bson:"noCursorTimeout,omitempty"` AllowPartialResults bool `bson:"allowPartialResults,omitempty"` + Collation *Collation `bson:"collation,omitempty"` +} + +// readLevel provides the nested "level: majority" serialisation needed for the +// query read concern. +type readLevel struct { + level string `bson:"level,omitempty"` } // getMoreCmd holds the command used for requesting more query results on MongoDB 3.2+. @@ -3243,7 +3712,7 @@ type getMoreCmd struct { func (db *Database) run(socket *mongoSocket, cmd, result interface{}) (err error) { // Database.Run: if name, ok := cmd.(string); ok { - cmd = bson.D{{name, 1}} + cmd = bson.D{{Name: name, Value: 1}} } // Collection.Find: @@ -3332,7 +3801,7 @@ func (db *Database) FindRef(ref *DBRef) *Query { // func (s *Session) FindRef(ref *DBRef) *Query { if ref.Database == "" { - panic(errors.New(fmt.Sprintf("Can't resolve database for %#v", ref))) + panic(fmt.Errorf("Can't resolve database for %#v", ref)) } c := s.DB(ref.Database).C(ref.Collection) return c.FindId(ref.Id) @@ -3353,7 +3822,7 @@ func (db *Database) CollectionNames() (names []string, err error) { Collections []bson.Raw Cursor cursorData } - err = db.With(cloned).Run(bson.D{{"listCollections", 1}, {"cursor", bson.D{{"batchSize", batchSize}}}}, &result) + err = db.With(cloned).Run(bson.D{{Name: "listCollections", Value: 1}, {Name: "cursor", Value: bson.D{{Name: "batchSize", Value: batchSize}}}}, &result) if err == nil { firstBatch := result.Collections if firstBatch == nil { @@ -3454,7 +3923,7 @@ func (q *Query) Iter() *Iter { op.replyFunc = iter.op.replyFunc if prepareFindOp(socket, &op, limit) { - iter.findCmd = true + iter.isFindCmd = true } iter.server = socket.Server() @@ -3668,7 +4137,8 @@ func (iter *Iter) Timeout() bool { // Next returns true if a document was successfully unmarshalled onto result, // and false at the end of the result set or if an error happened. // When Next returns false, the Err method should be called to verify if -// there was an error during iteration. +// there was an error during iteration, and the Timeout method to verify if the +// false return value was caused by a timeout (no available results). // // For example: // @@ -3684,7 +4154,16 @@ func (iter *Iter) Next(result interface{}) bool { iter.m.Lock() iter.timedout = false timeout := time.Time{} + // for a ChangeStream iterator we have to call getMore before the loop otherwise + // we'll always return false + if iter.isChangeStream { + iter.getMore() + } + // check should we expect more data. for iter.err == nil && iter.docData.Len() == 0 && (iter.docsToReceive > 0 || iter.op.cursorId != 0) { + // we should expect more data. + + // If we have yet to receive data, increment the timer until we timeout. if iter.docsToReceive == 0 { if iter.timeout >= 0 { if timeout.IsZero() { @@ -3696,6 +4175,13 @@ func (iter *Iter) Next(result interface{}) bool { return false } } + // for a ChangeStream one loop i enought to declare the timeout + if iter.isChangeStream { + iter.timedout = true + iter.m.Unlock() + return false + } + // run a getmore to fetch more data. iter.getMore() if iter.err != nil { break @@ -3703,7 +4189,7 @@ func (iter *Iter) Next(result interface{}) bool { } iter.gotReply.Wait() } - + // We have data from the getMore. // Exhaust available data before reporting any errors. if docData, ok := iter.docData.Pop().([]byte); ok { close := false @@ -3719,6 +4205,7 @@ func (iter *Iter) Next(result interface{}) bool { } } if iter.op.cursorId != 0 && iter.err == nil { + // we still have a live cursor and currently expect data. iter.docsBeforeMore-- if iter.docsBeforeMore == -1 { iter.getMore() @@ -3818,13 +4305,13 @@ func (q *Query) All(result interface{}) error { return q.Iter().All(result) } -// The For method is obsolete and will be removed in a future release. +// For method is obsolete and will be removed in a future release. // See Iter as an elegant replacement. func (q *Query) For(result interface{}, f func() error) error { return q.Iter().For(result, f) } -// The For method is obsolete and will be removed in a future release. +// For method is obsolete and will be removed in a future release. // See Iter as an elegant replacement. func (iter *Iter) For(result interface{}, f func() error) (err error) { valid := false @@ -3908,7 +4395,7 @@ func (iter *Iter) getMore() { } } var op interface{} - if iter.findCmd { + if iter.isFindCmd || iter.isChangeStream { op = iter.getMoreCmd() } else { op = &iter.op @@ -3931,6 +4418,9 @@ func (iter *Iter) getMoreCmd() *queryOp { Collection: iter.op.collection[nameDot+1:], BatchSize: iter.op.limit, } + if iter.maxTimeMS > 0 { + getMore.MaxTimeMS = iter.maxTimeMS + } var op queryOp op.collection = iter.op.collection[:nameDot] + ".$cmd" @@ -3941,10 +4431,12 @@ func (iter *Iter) getMoreCmd() *queryOp { } type countCmd struct { - Count string - Query interface{} - Limit int32 ",omitempty" - Skip int32 ",omitempty" + Count string + Query interface{} + Limit int32 `bson:",omitempty"` + Skip int32 `bson:",omitempty"` + Hint bson.D `bson:"hint,omitempty"` + MaxTimeMS int `bson:"maxTimeMS,omitempty"` } // Count returns the total number of documents in the result set. @@ -3966,8 +4458,12 @@ func (q *Query) Count() (n int, err error) { if query == nil { query = bson.D{} } + // not checking the error because if type assertion fails, we + // simply want a Zero bson.D + hint, _ := q.op.options.Hint.(bson.D) result := struct{ N int }{} - err = session.DB(dbname).Run(countCmd{cname, query, limit, op.skip}, &result) + err = session.DB(dbname).Run(countCmd{cname, query, limit, op.skip, hint, op.options.MaxTimeMS}, &result) + return result.N, err } @@ -3977,9 +4473,9 @@ func (c *Collection) Count() (n int, err error) { } type distinctCmd struct { - Collection string "distinct" + Collection string `bson:"distinct"` Key string - Query interface{} ",omitempty" + Query interface{} `bson:",omitempty"` } // Distinct unmarshals into result the list of distinct values for the given key. @@ -4016,28 +4512,34 @@ func (q *Query) Distinct(key string, result interface{}) error { } type mapReduceCmd struct { - Collection string "mapreduce" - Map string ",omitempty" - Reduce string ",omitempty" - Finalize string ",omitempty" - Limit int32 ",omitempty" + Collection string `bson:"mapreduce"` + Map string `bson:",omitempty"` + Reduce string `bson:",omitempty"` + Finalize string `bson:",omitempty"` Out interface{} - Query interface{} ",omitempty" - Sort interface{} ",omitempty" - Scope interface{} ",omitempty" - Verbose bool ",omitempty" + Query interface{} `bson:",omitempty"` + Sort interface{} `bson:",omitempty"` + Scope interface{} `bson:",omitempty"` + Limit int32 `bson:",omitempty"` + Verbose bool `bson:",omitempty"` } type mapReduceResult struct { Results bson.Raw Result bson.Raw - TimeMillis int64 "timeMillis" + TimeMillis int64 `bson:"timeMillis"` Counts struct{ Input, Emit, Output int } Ok bool Err string Timing *MapReduceTime } +// MapReduce used to perform Map Reduce operations +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/core/map-reduce/ +// type MapReduce struct { Map string // Map Javascript function code (required) Reduce string // Reduce Javascript function code (required) @@ -4047,6 +4549,7 @@ type MapReduce struct { Verbose bool } +// MapReduceInfo stores informations on a MapReduce operation type MapReduceInfo struct { InputCount int // Number of documents mapped EmitCount int // Number of times reduce called emit @@ -4057,10 +4560,11 @@ type MapReduceInfo struct { VerboseTime *MapReduceTime // Only defined if Verbose was true } +// MapReduceTime stores execution time of a MapReduce operation type MapReduceTime struct { Total int64 // Total time, in nanoseconds - Map int64 "mapTime" // Time within map function, in nanoseconds - EmitLoop int64 "emitLoop" // Time within the emit/map loop, in nanoseconds + Map int64 `bson:"mapTime"` // Time within map function, in nanoseconds + EmitLoop int64 `bson:"emitLoop"` // Time within the emit/map loop, in nanoseconds } // MapReduce executes a map/reduce job for documents covered by the query. @@ -4151,7 +4655,7 @@ func (q *Query) MapReduce(job *MapReduce, result interface{}) (info *MapReduceIn } if cmd.Out == nil { - cmd.Out = bson.D{{"inline", 1}} + cmd.Out = bson.D{{Name: "inline", Value: 1}} } var doc mapReduceResult @@ -4236,14 +4740,14 @@ type Change struct { } type findModifyCmd struct { - Collection string "findAndModify" - Query, Update, Sort, Fields interface{} ",omitempty" - Upsert, Remove, New bool ",omitempty" + Collection string `bson:"findAndModify"` + Query, Update, Sort, Fields interface{} `bson:",omitempty"` + Upsert, Remove, New bool `bson:",omitempty"` } type valueResult struct { Value bson.Raw - LastError LastError "lastErrorObject" + LastError LastError `bson:"lastErrorObject"` } // Apply runs the findAndModify MongoDB command, which allows updating, upserting @@ -4375,7 +4879,7 @@ func (bi *BuildInfo) VersionAtLeast(version ...int) bool { // BuildInfo retrieves the version and other details about the // running MongoDB server. func (s *Session) BuildInfo() (info BuildInfo, err error) { - err = s.Run(bson.D{{"buildInfo", "1"}}, &info) + err = s.Run(bson.D{{Name: "buildInfo", Value: "1"}}, &info) if len(info.VersionArray) == 0 { for _, a := range strings.Split(info.Version, ".") { i, err := strconv.Atoi(a) @@ -4408,13 +4912,13 @@ func (s *Session) acquireSocket(slaveOk bool) (*mongoSocket, error) { s.m.RLock() // If there is a slave socket reserved and its use is acceptable, take it as long // as there isn't a master socket which would be preferred by the read preference mode. - if s.slaveSocket != nil && s.slaveOk && slaveOk && (s.masterSocket == nil || s.consistency != PrimaryPreferred && s.consistency != Monotonic) { + if s.slaveSocket != nil && s.slaveSocket.dead == nil && s.slaveOk && slaveOk && (s.masterSocket == nil || s.consistency != PrimaryPreferred && s.consistency != Monotonic) { socket := s.slaveSocket socket.Acquire() s.m.RUnlock() return socket, nil } - if s.masterSocket != nil { + if s.masterSocket != nil && s.masterSocket.dead == nil { socket := s.masterSocket socket.Acquire() s.m.RUnlock() @@ -4428,16 +4932,26 @@ func (s *Session) acquireSocket(slaveOk bool) (*mongoSocket, error) { defer s.m.Unlock() if s.slaveSocket != nil && s.slaveOk && slaveOk && (s.masterSocket == nil || s.consistency != PrimaryPreferred && s.consistency != Monotonic) { - s.slaveSocket.Acquire() - return s.slaveSocket, nil + if s.slaveSocket.dead == nil { + s.slaveSocket.Acquire() + return s.slaveSocket, nil + } else { + s.unsetSocket() + } } if s.masterSocket != nil { - s.masterSocket.Acquire() - return s.masterSocket, nil + if s.masterSocket.dead == nil { + s.masterSocket.Acquire() + return s.masterSocket, nil + } else { + s.unsetSocket() + } } // Still not good. We need a new socket. - sock, err := s.cluster().AcquireSocket(s.consistency, slaveOk && s.slaveOk, s.syncTimeout, s.sockTimeout, s.queryConfig.op.serverTags, s.poolLimit) + sock, err := s.cluster().AcquireSocketWithPoolTimeout( + s.consistency, slaveOk && s.slaveOk, s.syncTimeout, s.sockTimeout, s.queryConfig.op.serverTags, s.poolLimit, s.poolTimeout, + ) if err != nil { return nil, err } @@ -4484,9 +4998,11 @@ func (s *Session) setSocket(socket *mongoSocket) { // unsetSocket releases any slave and/or master sockets reserved. func (s *Session) unsetSocket() { if s.masterSocket != nil { + debugf("unset master socket from session %p", s) s.masterSocket.Release() } if s.slaveSocket != nil { + debugf("unset slave socket from session %p", s) s.slaveSocket.Release() } s.masterSocket = nil @@ -4511,7 +5027,7 @@ func (iter *Iter) replyFunc() replyFunc { } else { iter.err = ErrNotFound } - } else if iter.findCmd { + } else if iter.isFindCmd { debugf("Iter %p received reply document %d/%d (cursor=%d)", iter, docNum+1, int(op.replyDocs), op.cursorId) var findReply struct { Ok bool @@ -4523,7 +5039,7 @@ func (iter *Iter) replyFunc() replyFunc { iter.err = err } else if !findReply.Ok && findReply.Errmsg != "" { iter.err = &QueryError{Code: findReply.Code, Message: findReply.Errmsg} - } else if len(findReply.Cursor.FirstBatch) == 0 && len(findReply.Cursor.NextBatch) == 0 { + } else if !iter.isChangeStream && len(findReply.Cursor.FirstBatch) == 0 && len(findReply.Cursor.NextBatch) == 0 { iter.err = ErrNotFound } else { batch := findReply.Cursor.FirstBatch @@ -4569,7 +5085,7 @@ type writeCmdResult struct { NModified int `bson:"nModified"` Upserted []struct { Index int - Id interface{} `_id` + Id interface{} `bson:"_id"` } ConcernError writeConcernError `bson:"writeConcernError"` Errors []writeCmdError `bson:"writeErrors"` @@ -4642,6 +5158,58 @@ func (c *Collection) writeOp(op interface{}, ordered bool) (lerr *LastError, err } return &lerr, nil } + if updateOp, ok := op.(bulkUpdateOp); ok && len(updateOp) > 1000 { + var lerr LastError + + // Maximum batch size is 1000. Must split out in separate operations for compatibility. + for i := 0; i < len(updateOp); i += 1000 { + l := i + 1000 + if l > len(updateOp) { + l = len(updateOp) + } + + oplerr, err := c.writeOpCommand(socket, safeOp, updateOp[i:l], ordered, bypassValidation) + + lerr.N += oplerr.N + lerr.modified += oplerr.modified + if err != nil { + lerr.ecases = append(lerr.ecases, BulkErrorCase{i, err}) + if ordered { + break + } + } + } + if len(lerr.ecases) != 0 { + return &lerr, lerr.ecases[0].Err + } + return &lerr, nil + } + if deleteOps, ok := op.(bulkDeleteOp); ok && len(deleteOps) > 1000 { + var lerr LastError + + // Maximum batch size is 1000. Must split out in separate operations for compatibility. + for i := 0; i < len(deleteOps); i += 1000 { + l := i + 1000 + if l > len(deleteOps) { + l = len(deleteOps) + } + + oplerr, err := c.writeOpCommand(socket, safeOp, deleteOps[i:l], ordered, bypassValidation) + + lerr.N += oplerr.N + lerr.modified += oplerr.modified + if err != nil { + lerr.ecases = append(lerr.ecases, BulkErrorCase{i, err}) + if ordered { + break + } + } + } + if len(lerr.ecases) != 0 { + return &lerr, lerr.ecases[0].Err + } + return &lerr, nil + } return c.writeOpCommand(socket, safeOp, op, ordered, bypassValidation) } else if updateOps, ok := op.(bulkUpdateOp); ok { var lerr LastError @@ -4730,7 +5298,7 @@ func (c *Collection) writeOpQuery(socket *mongoSocket, safeOp *queryOp, op inter func (c *Collection) writeOpCommand(socket *mongoSocket, safeOp *queryOp, op interface{}, ordered, bypassValidation bool) (lerr *LastError, err error) { var writeConcern interface{} if safeOp == nil { - writeConcern = bson.D{{"w", 0}} + writeConcern = bson.D{{Name: "w", Value: 0}} } else { writeConcern = safeOp.query.(*getLastError) } @@ -4740,46 +5308,46 @@ func (c *Collection) writeOpCommand(socket *mongoSocket, safeOp *queryOp, op int case *insertOp: // http://docs.mongodb.org/manual/reference/command/insert cmd = bson.D{ - {"insert", c.Name}, - {"documents", op.documents}, - {"writeConcern", writeConcern}, - {"ordered", op.flags&1 == 0}, + {Name: "insert", Value: c.Name}, + {Name: "documents", Value: op.documents}, + {Name: "writeConcern", Value: writeConcern}, + {Name: "ordered", Value: op.flags&1 == 0}, } case *updateOp: // http://docs.mongodb.org/manual/reference/command/update cmd = bson.D{ - {"update", c.Name}, - {"updates", []interface{}{op}}, - {"writeConcern", writeConcern}, - {"ordered", ordered}, + {Name: "update", Value: c.Name}, + {Name: "updates", Value: []interface{}{op}}, + {Name: "writeConcern", Value: writeConcern}, + {Name: "ordered", Value: ordered}, } case bulkUpdateOp: // http://docs.mongodb.org/manual/reference/command/update cmd = bson.D{ - {"update", c.Name}, - {"updates", op}, - {"writeConcern", writeConcern}, - {"ordered", ordered}, + {Name: "update", Value: c.Name}, + {Name: "updates", Value: op}, + {Name: "writeConcern", Value: writeConcern}, + {Name: "ordered", Value: ordered}, } case *deleteOp: // http://docs.mongodb.org/manual/reference/command/delete cmd = bson.D{ - {"delete", c.Name}, - {"deletes", []interface{}{op}}, - {"writeConcern", writeConcern}, - {"ordered", ordered}, + {Name: "delete", Value: c.Name}, + {Name: "deletes", Value: []interface{}{op}}, + {Name: "writeConcern", Value: writeConcern}, + {Name: "ordered", Value: ordered}, } case bulkDeleteOp: // http://docs.mongodb.org/manual/reference/command/delete cmd = bson.D{ - {"delete", c.Name}, - {"deletes", op}, - {"writeConcern", writeConcern}, - {"ordered", ordered}, + {Name: "delete", Value: c.Name}, + {Name: "deletes", Value: op}, + {Name: "writeConcern", Value: writeConcern}, + {Name: "ordered", Value: ordered}, } } if bypassValidation { - cmd = append(cmd, bson.DocElem{"bypassDocumentValidation", true}) + cmd = append(cmd, bson.DocElem{Name: "bypassDocumentValidation", Value: true}) } var result writeCmdResult @@ -4823,3 +5391,73 @@ func hasErrMsg(d []byte) bool { } return false } + +// getRFC2253NameStringFromCert converts from an ASN.1 structured representation of the certificate +// to a UTF-8 string representation(RDN) and returns it. +func getRFC2253NameStringFromCert(certificate *x509.Certificate) (string, error) { + var RDNElements = pkix.RDNSequence{} + _, err := asn1.Unmarshal(certificate.RawSubject, &RDNElements) + return getRFC2253NameString(&RDNElements), err +} + +// getRFC2253NameString converts from an ASN.1 structured representation of the RDNSequence +// from the certificate to a UTF-8 string representation(RDN) and returns it. +func getRFC2253NameString(RDNElements *pkix.RDNSequence) string { + var RDNElementsString = []string{} + var replacer = strings.NewReplacer(",", "\\,", "=", "\\=", "+", "\\+", "<", "\\<", ">", "\\>", ";", "\\;") + //The elements in the sequence needs to be reversed when converting them + for i := len(*RDNElements) - 1; i >= 0; i-- { + var nameAndValueList = make([]string, len((*RDNElements)[i])) + for j, attribute := range (*RDNElements)[i] { + var shortAttributeName = rdnOIDToShortName(attribute.Type) + if len(shortAttributeName) <= 0 { + nameAndValueList[j] = fmt.Sprintf("%s=%X", attribute.Type.String(), attribute.Value.([]byte)) + continue + } + var attributeValueString = attribute.Value.(string) + // escape leading space or # + if strings.HasPrefix(attributeValueString, " ") || strings.HasPrefix(attributeValueString, "#") { + attributeValueString = "\\" + attributeValueString + } + // escape trailing space, unless it's already escaped + if strings.HasSuffix(attributeValueString, " ") && !strings.HasSuffix(attributeValueString, "\\ ") { + attributeValueString = attributeValueString[:len(attributeValueString)-1] + "\\ " + } + + // escape , = + < > # ; + attributeValueString = replacer.Replace(attributeValueString) + nameAndValueList[j] = fmt.Sprintf("%s=%s", shortAttributeName, attributeValueString) + } + + RDNElementsString = append(RDNElementsString, strings.Join(nameAndValueList, "+")) + } + + return strings.Join(RDNElementsString, ",") +} + +var oidsToShortNames = []struct { + oid asn1.ObjectIdentifier + shortName string +}{ + {asn1.ObjectIdentifier{2, 5, 4, 3}, "CN"}, + {asn1.ObjectIdentifier{2, 5, 4, 6}, "C"}, + {asn1.ObjectIdentifier{2, 5, 4, 7}, "L"}, + {asn1.ObjectIdentifier{2, 5, 4, 8}, "ST"}, + {asn1.ObjectIdentifier{2, 5, 4, 10}, "O"}, + {asn1.ObjectIdentifier{2, 5, 4, 11}, "OU"}, + {asn1.ObjectIdentifier{2, 5, 4, 9}, "STREET"}, + {asn1.ObjectIdentifier{0, 9, 2342, 19200300, 100, 1, 25}, "DC"}, + {asn1.ObjectIdentifier{0, 9, 2342, 19200300, 100, 1, 1}, "UID"}, +} + +// rdnOIDToShortName returns an short name of the given RDN OID. If the OID does not have a short +// name, the function returns an empty string +func rdnOIDToShortName(oid asn1.ObjectIdentifier) string { + for i := range oidsToShortNames { + if oidsToShortNames[i].oid.Equal(oid) { + return oidsToShortNames[i].shortName + } + } + + return "" +} diff --git a/vendor/gopkg.in/mgo.v2/socket.go b/vendor/github.com/gedge/mgo/socket.go similarity index 88% rename from vendor/gopkg.in/mgo.v2/socket.go rename to vendor/github.com/gedge/mgo/socket.go index 8891dd5d..4a891306 100644 --- a/vendor/gopkg.in/mgo.v2/socket.go +++ b/vendor/github.com/gedge/mgo/socket.go @@ -33,26 +33,29 @@ import ( "sync" "time" - "gopkg.in/mgo.v2/bson" + "github.com/gedge/mgo/bson" ) type replyFunc func(err error, reply *replyOp, docNum int, docData []byte) type mongoSocket struct { sync.Mutex - server *mongoServer // nil when cached - conn net.Conn - timeout time.Duration - addr string // For debugging only. - nextRequestId uint32 - replyFuncs map[uint32]replyFunc - references int - creds []Credential - logout []Credential - cachedNonce string - gotNonce sync.Cond - dead error - serverInfo *mongoServerInfo + server *mongoServer // nil when cached + conn net.Conn + timeout time.Duration + addr string // For debugging only. + nextRequestId uint32 + replyFuncs map[uint32]replyFunc + references int + creds []Credential + logout []Credential + cachedNonce string + gotNonce sync.Cond + dead error + serverInfo *mongoServerInfo + closeAfterIdle bool + lastTimeUsed time.Time // for time based idle socket release + sendMeta sync.Once } type queryOpFlags uint32 @@ -67,30 +70,31 @@ const ( ) type queryOp struct { - collection string - query interface{} - skip int32 - limit int32 - selector interface{} - flags queryOpFlags - replyFunc replyFunc - - mode Mode - options queryWrapper - hasOptions bool - serverTags []bson.D + query interface{} + collection string + serverTags []bson.D + selector interface{} + replyFunc replyFunc + mode Mode + skip int32 + limit int32 + options queryWrapper + hasOptions bool + flags queryOpFlags + readConcern string } type queryWrapper struct { - Query interface{} "$query" - OrderBy interface{} "$orderby,omitempty" - Hint interface{} "$hint,omitempty" - Explain bool "$explain,omitempty" - Snapshot bool "$snapshot,omitempty" - ReadPreference bson.D "$readPreference,omitempty" - MaxScan int "$maxScan,omitempty" - MaxTimeMS int "$maxTimeMS,omitempty" - Comment string "$comment,omitempty" + Query interface{} `bson:"$query"` + OrderBy interface{} `bson:"$orderby,omitempty"` + Hint interface{} `bson:"$hint,omitempty"` + Explain bool `bson:"$explain,omitempty"` + Snapshot bool `bson:"$snapshot,omitempty"` + ReadPreference bson.D `bson:"$readPreference,omitempty"` + MaxScan int `bson:"$maxScan,omitempty"` + MaxTimeMS int `bson:"$maxTimeMS,omitempty"` + Comment string `bson:"$comment,omitempty"` + Collation *Collation `bson:"$collation,omitempty"` } func (op *queryOp) finalQuery(socket *mongoSocket) interface{} { @@ -114,9 +118,9 @@ func (op *queryOp) finalQuery(socket *mongoSocket) interface{} { } op.hasOptions = true op.options.ReadPreference = make(bson.D, 0, 2) - op.options.ReadPreference = append(op.options.ReadPreference, bson.DocElem{"mode", modeName}) + op.options.ReadPreference = append(op.options.ReadPreference, bson.DocElem{Name: "mode", Value: modeName}) if len(op.serverTags) > 0 { - op.options.ReadPreference = append(op.options.ReadPreference, bson.DocElem{"tags", op.serverTags}) + op.options.ReadPreference = append(op.options.ReadPreference, bson.DocElem{Name: "tags", Value: op.serverTags}) } } if op.hasOptions { @@ -207,6 +211,9 @@ func (socket *mongoSocket) Server() *mongoServer { // ServerInfo returns details for the server at the time the socket // was initially acquired. func (socket *mongoSocket) ServerInfo() *mongoServerInfo { + if socket == nil { + return &mongoServerInfo{} + } socket.Lock() serverInfo := socket.serverInfo socket.Unlock() @@ -264,10 +271,13 @@ func (socket *mongoSocket) Release() { if socket.references == 0 { stats.socketsInUse(-1) server := socket.server + closeAfterIdle := socket.closeAfterIdle socket.Unlock() socket.LogoutAll() - // If the socket is dead server is nil. - if server != nil { + if closeAfterIdle { + socket.Close() + } else if server != nil { + // If the socket is dead server is nil. server.RecycleSocket(socket) } } else { @@ -316,6 +326,21 @@ func (socket *mongoSocket) Close() { socket.kill(errors.New("Closed explicitly"), false) } +// CloseAfterIdle terminates an idle socket, which has a zero +// reference, or marks the socket to be terminate after idle. +func (socket *mongoSocket) CloseAfterIdle() { + socket.Lock() + if socket.references == 0 { + socket.Unlock() + socket.Close() + logf("Socket %p to %s: idle and close.", socket, socket.addr) + return + } + socket.closeAfterIdle = true + socket.Unlock() + logf("Socket %p to %s: close after idle.", socket, socket.addr) +} + func (socket *mongoSocket) kill(err error, abend bool) { socket.Lock() if socket.dead != nil { @@ -372,13 +397,22 @@ func (socket *mongoSocket) SimpleQuery(op *queryOp) (data []byte, err error) { return data, err } +var bytesBufferPool = sync.Pool{ + New: func() interface{} { + return make([]byte, 0, 256) + }, +} + func (socket *mongoSocket) Query(ops ...interface{}) (err error) { if lops := socket.flushLogout(); len(lops) > 0 { ops = append(lops, ops...) } - buf := make([]byte, 0, 256) + buf := bytesBufferPool.Get().([]byte) + defer func() { + bytesBufferPool.Put(buf[:0]) + }() // Serialize operations synchronously to avoid interrupting // other goroutines while we can't really be sending data. @@ -517,16 +551,15 @@ func (socket *mongoSocket) Query(ops ...interface{}) (err error) { socket.replyFuncs[requestId] = request.replyFunc requestId++ } - + socket.Unlock() debugf("Socket %p to %s: sending %d op(s) (%d bytes)", socket, socket.addr, len(ops), len(buf)) - stats.sentOps(len(ops)) + stats.sentOps(len(ops)) socket.updateDeadline(writeDeadline) _, err = socket.conn.Write(buf) if !wasWaiting && requestCount > 0 { socket.updateDeadline(readDeadline) } - socket.Unlock() return err } @@ -674,11 +707,11 @@ func addBSON(b []byte, doc interface{}) ([]byte, error) { if doc == nil { return append(b, 5, 0, 0, 0, 0), nil } - data, err := bson.Marshal(doc) + data, err := bson.MarshalBuffer(doc, b) if err != nil { return b, err } - return append(b, data...), nil + return data, nil } func setInt32(b []byte, pos int, i int32) { diff --git a/vendor/gopkg.in/mgo.v2/stats.go b/vendor/github.com/gedge/mgo/stats.go similarity index 74% rename from vendor/gopkg.in/mgo.v2/stats.go rename to vendor/github.com/gedge/mgo/stats.go index 59723e60..8cf4ecec 100644 --- a/vendor/gopkg.in/mgo.v2/stats.go +++ b/vendor/github.com/gedge/mgo/stats.go @@ -28,11 +28,13 @@ package mgo import ( "sync" + "time" ) var stats *Stats var statsMutex sync.Mutex +// SetStats enable database state monitoring func SetStats(enabled bool) { statsMutex.Lock() if enabled { @@ -45,6 +47,7 @@ func SetStats(enabled bool) { statsMutex.Unlock() } +// GetStats return the current database state func GetStats() (snapshot Stats) { statsMutex.Lock() snapshot = *stats @@ -52,6 +55,7 @@ func GetStats() (snapshot Stats) { return } +// ResetStats reset Stats to the previous database state func ResetStats() { statsMutex.Lock() debug("Resetting stats") @@ -66,16 +70,27 @@ func ResetStats() { return } +// Stats holds info on the database state +// +// Relevant documentation: +// +// https://docs.mongodb.com/manual/reference/command/serverStatus/ +// +// TODO outdated fields ? type Stats struct { - Clusters int - MasterConns int - SlaveConns int - SentOps int - ReceivedOps int - ReceivedDocs int - SocketsAlive int - SocketsInUse int - SocketRefs int + Clusters int + MasterConns int + SlaveConns int + SentOps int + ReceivedOps int + ReceivedDocs int + SocketsAlive int + SocketsInUse int + SocketRefs int + TimesSocketAcquired int + TimesWaitedForPool int + TotalPoolWaitTime time.Duration + PoolTimeouts int } func (stats *Stats) cluster(delta int) { @@ -145,3 +160,25 @@ func (stats *Stats) socketRefs(delta int) { statsMutex.Unlock() } } + +func (stats *Stats) noticeSocketAcquisition(waitTime time.Duration) { + if stats != nil { + statsMutex.Lock() + stats.TimesSocketAcquired++ + stats.TotalPoolWaitTime += waitTime + if waitTime > 0 { + stats.TimesWaitedForPool++ + } + statsMutex.Unlock() + } +} + +func (stats *Stats) noticePoolTimeout(waitTime time.Duration) { + if stats != nil { + statsMutex.Lock() + stats.TimesWaitedForPool++ + stats.PoolTimeouts++ + stats.TotalPoolWaitTime += waitTime + statsMutex.Unlock() + } +} diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/LICENSE b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/LICENSE similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/LICENSE rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/LICENSE diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/README.md b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/README.md similarity index 94% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/README.md rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/README.md index 2ded4a9c..0bdf10e7 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/README.md +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/README.md @@ -1,12 +1,13 @@ # Golang Neo4J Bolt Driver -[![Build Status](https://travis-ci.org/ONSdigital/golang-neo4j-bolt-driver.svg?branch=master)](https://travis-ci.org/ONSdigital/golang-neo4j-bolt-driver) *Tested against Golang 1.4.3 and up* +[![Build Status](https://travis-ci.org/johnnadratowski/golang-neo4j-bolt-driver.svg?branch=master)](https://travis-ci.org/johnnadratowski/golang-neo4j-bolt-driver) +[![GoDoc](https://godoc.org/github.com/johnnadratowski/golang-neo4j-bolt-driver?status.svg)](https://godoc.org/github.com/johnnadratowski/golang-neo4j-bolt-driver) Implements the Neo4J Bolt Protocol specification: As of the time of writing this, the current version is v3.1.0-M02 ``` -go get github.com/ONSdigital/golang-neo4j-bolt-driver +go get github.com/johnnadratowski/golang-neo4j-bolt-driver ``` ## Features @@ -194,7 +195,7 @@ func slowNClean() { ``` ## API -*_There is much more detailed information in [the godoc](http://godoc.org/github.com/ONSdigital/golang-neo4j-bolt-driver)_* +*_There is much more detailed information in [the godoc](http://godoc.org/github.com/johnnadratowski/golang-neo4j-bolt-driver)_* This implementation attempts to follow the best practices as per the Bolt specification, but also implements compatibility with Golang's `sql.driver` interface. diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/conn.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/conn.go similarity index 98% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/conn.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/conn.go index c23ca8a4..d7e55bc3 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/conn.go +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/conn.go @@ -17,10 +17,10 @@ import ( "crypto/x509" "strconv" - "github.com/ONSdigital/golang-neo4j-bolt-driver/encoding" - "github.com/ONSdigital/golang-neo4j-bolt-driver/errors" - "github.com/ONSdigital/golang-neo4j-bolt-driver/log" - "github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/log" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages" ) // Conn represents a connection to Neo4J @@ -356,12 +356,6 @@ func (c *boltConn) Close() error { return nil } - if c.transaction != nil { - if err := c.transaction.Rollback(); err != nil { - return err - } - } - if c.statement != nil { if err := c.statement.Close(); err != nil { return err diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/doc.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/doc.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/doc.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/doc.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/driver.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/driver.go similarity index 98% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/driver.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/driver.go index 293cce8d..9703b9bf 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/driver.go +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/driver.go @@ -3,8 +3,8 @@ package golangNeo4jBoltDriver import ( "database/sql" "database/sql/driver" - "github.com/ONSdigital/golang-neo4j-bolt-driver/errors" "sync" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors" ) var ( diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/encoding/decoder.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding/decoder.go similarity index 98% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/encoding/decoder.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding/decoder.go index ba7469e6..7d88f549 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/encoding/decoder.go +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding/decoder.go @@ -5,9 +5,9 @@ import ( "encoding/binary" "io" - "github.com/ONSdigital/golang-neo4j-bolt-driver/errors" - "github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph" - "github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages" ) // Decoder decodes a message from the bolt protocol stream diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/encoding/doc.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding/doc.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/encoding/doc.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding/doc.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/encoding/encoder.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding/encoder.go similarity index 99% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/encoding/encoder.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding/encoder.go index 16138b04..56752c8e 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/encoding/encoder.go +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding/encoder.go @@ -7,8 +7,8 @@ import ( "bytes" - "github.com/ONSdigital/golang-neo4j-bolt-driver/errors" - "github.com/ONSdigital/golang-neo4j-bolt-driver/structures" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures" ) const ( diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/encoding/util.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding/util.go similarity index 91% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/encoding/util.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding/util.go index 4ea43c38..1b219325 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/encoding/util.go +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding/util.go @@ -1,8 +1,8 @@ package encoding import ( - "github.com/ONSdigital/golang-neo4j-bolt-driver/errors" - "github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph" ) func sliceInterfaceToString(from []interface{}) ([]string, error) { diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/errors/doc.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/errors/doc.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/errors/doc.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/errors/doc.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/errors/errors.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/errors/errors.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/errors/errors.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/errors/errors.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/log/doc.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/log/doc.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/log/doc.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/log/doc.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/log/log.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/log/log.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/log/log.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/log/log.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/recorder.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/recorder.go similarity index 97% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/recorder.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/recorder.go index 17e77211..41ba2ca8 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/recorder.go +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/recorder.go @@ -8,9 +8,9 @@ import ( "os" "time" - "github.com/ONSdigital/golang-neo4j-bolt-driver/encoding" - "github.com/ONSdigital/golang-neo4j-bolt-driver/errors" - "github.com/ONSdigital/golang-neo4j-bolt-driver/log" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/log" ) // recorder records a given session with Neo4j. diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/result.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/result.go similarity index 96% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/result.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/result.go index 41417a99..c37a9212 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/result.go +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/result.go @@ -1,6 +1,6 @@ package golangNeo4jBoltDriver -import "github.com/ONSdigital/golang-neo4j-bolt-driver/errors" +import "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors" // Result represents a result from a query that returns no data type Result interface { diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/rows.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/rows.go similarity index 96% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/rows.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/rows.go index 90e9c7eb..4eb67822 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/rows.go +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/rows.go @@ -4,11 +4,11 @@ import ( "database/sql/driver" "io" - "github.com/ONSdigital/golang-neo4j-bolt-driver/encoding" - "github.com/ONSdigital/golang-neo4j-bolt-driver/errors" - "github.com/ONSdigital/golang-neo4j-bolt-driver/log" - "github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph" - "github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/log" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages" ) // Rows represents results of rows from the DB diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/stmt.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/stmt.go similarity index 97% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/stmt.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/stmt.go index 7d66a11d..42a6f321 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/stmt.go +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/stmt.go @@ -3,9 +3,9 @@ package golangNeo4jBoltDriver import ( "database/sql/driver" - "github.com/ONSdigital/golang-neo4j-bolt-driver/errors" - "github.com/ONSdigital/golang-neo4j-bolt-driver/log" - "github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/log" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages" ) // Stmt represents a statement to run against the database diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/doc.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/doc.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/doc.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/doc.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph/doc.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph/doc.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph/doc.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph/doc.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph/node.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph/node.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph/node.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph/node.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph/path.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph/path.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph/path.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph/path.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph/relationship.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph/relationship.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph/relationship.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph/relationship.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph/unbound_relationship.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph/unbound_relationship.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph/unbound_relationship.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph/unbound_relationship.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/ack_failure.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/ack_failure.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/ack_failure.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/ack_failure.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/discard_all.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/discard_all.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/discard_all.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/discard_all.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/doc.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/doc.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/doc.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/doc.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/failure.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/failure.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/failure.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/failure.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/ignored.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/ignored.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/ignored.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/ignored.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/init.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/init.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/init.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/init.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/pull_all.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/pull_all.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/pull_all.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/pull_all.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/record.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/record.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/record.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/record.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/reset.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/reset.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/reset.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/reset.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/run.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/run.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/run.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/run.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/success.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/success.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages/success.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages/success.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/structures.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/structures.go similarity index 100% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/structures/structures.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/structures.go diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/tx.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/tx.go similarity index 92% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/tx.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/tx.go index 1792bee0..e70bbefb 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/tx.go +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/tx.go @@ -1,9 +1,9 @@ package golangNeo4jBoltDriver import ( - "github.com/ONSdigital/golang-neo4j-bolt-driver/errors" - "github.com/ONSdigital/golang-neo4j-bolt-driver/log" - "github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/log" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages" ) // Tx represents a transaction diff --git a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/util.go b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/util.go similarity index 94% rename from vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/util.go rename to vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/util.go index 904b538b..4e16ab73 100644 --- a/vendor/github.com/ONSdigital/golang-neo4j-bolt-driver/util.go +++ b/vendor/github.com/johnnadratowski/golang-neo4j-bolt-driver/util.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" - "github.com/ONSdigital/golang-neo4j-bolt-driver/encoding" + "github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding" ) // sprintByteHex returns a formatted string of the byte array in hexadecimal diff --git a/vendor/gopkg.in/mgo.v2/README.md b/vendor/gopkg.in/mgo.v2/README.md deleted file mode 100644 index f4e452c0..00000000 --- a/vendor/gopkg.in/mgo.v2/README.md +++ /dev/null @@ -1,4 +0,0 @@ -The MongoDB driver for Go -------------------------- - -Please go to [http://labix.org/mgo](http://labix.org/mgo) for all project details. diff --git a/vendor/vendor.json b/vendor/vendor.json index 42483f2e..c265217d 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -3,130 +3,94 @@ "ignore": "test", "package": [ { - "checksumSHA1": "w4WYJOG6u77qJORANUwGKdxkOYY=", + "checksumSHA1": "qhFKyZRRovlrhBNLfxRZPG2IlWE=", "path": "github.com/ONSdigital/dp-filter/observation", - "revision": "a3b43eb05f07f1ca05f19427dad23e4648516cb8", - "revisionTime": "2018-04-13T11:23:02Z" + "revision": "bfd100adc17e907c2f0824c848b229adc29ffbe5", + "revisionTime": "2018-06-04T12:37:22Z" }, { - "checksumSHA1": "YBSQ8Wb0hEsy+Qrm9QPRNZV9qkY=", + "checksumSHA1": "VUBOs+JwgkDk/g7qgUgfMS7K4jc=", "path": "github.com/ONSdigital/go-ns/audit", - "revision": "52af9538feaa8af4ef4702416cca4e640b4f2300", - "revisionTime": "2018-05-02T12:53:10Z" + "revision": "e89bb271c005948b66da47ec485f1fb786753612", + "revisionTime": "2018-06-07T08:14:26Z" + }, + { + "checksumSHA1": "q34mbe+JfzJZQ1Uocz4LTtM73Ds=", + "path": "github.com/ONSdigital/go-ns/audit/audit_mock", + "revision": "be5712823158df77dac2833e1b7f289def8325d0", + "revisionTime": "2018-06-28T08:50:39Z" }, { "checksumSHA1": "vyzH2Rju6G4A/uK4zqWIidfk3dA=", "path": "github.com/ONSdigital/go-ns/avro", - "revision": "ca71da72d1cc91a0897057170895294c864cf66a", - "revisionTime": "2018-04-30T16:40:16Z" + "revision": "e89bb271c005948b66da47ec485f1fb786753612", + "revisionTime": "2018-06-07T08:14:26Z" }, { - "checksumSHA1": "k68VR3oRFV5ppeXFt7tLDbRNTek=", + "checksumSHA1": "69YldJlpMZgzWrnUEEFQeuFF5Ro=", "path": "github.com/ONSdigital/go-ns/clients/identity", - "revision": "2a4ed4b9091e2042ae489ab492711ec1a866c94b", - "revisionTime": "2018-05-02T09:37:52Z" + "revision": "e89bb271c005948b66da47ec485f1fb786753612", + "revisionTime": "2018-06-07T08:14:26Z" }, { - "checksumSHA1": "fDVkILVJU1lePJ2WXYrCxkqPxGU=", + "checksumSHA1": "PqFwPuxh85nRWHSiSD8b83p4F7g=", "path": "github.com/ONSdigital/go-ns/common", - "revision": "2a4ed4b9091e2042ae489ab492711ec1a866c94b", - "revisionTime": "2018-05-02T09:37:52Z" + "revision": "e89bb271c005948b66da47ec485f1fb786753612", + "revisionTime": "2018-06-07T08:14:26Z" }, { - "checksumSHA1": "5JuTuAuidImdCdyGDKeAx2PLghA=", + "checksumSHA1": "Vjyl/gfY0qx2lW9te3Ieb5HV3/s=", "path": "github.com/ONSdigital/go-ns/handlers/requestID", - "revision": "2a4ed4b9091e2042ae489ab492711ec1a866c94b", - "revisionTime": "2018-05-02T09:37:52Z" + "revision": "e89bb271c005948b66da47ec485f1fb786753612", + "revisionTime": "2018-06-07T08:14:26Z" }, { "checksumSHA1": "p1fQ6Gqk1X/bTmhSqOnLDPOSD10=", "path": "github.com/ONSdigital/go-ns/healthcheck", - "revision": "01c38ac76f9fdd25871a39b9e008d335ac251499", - "revisionTime": "2018-05-04T09:27:22Z" + "revision": "e89bb271c005948b66da47ec485f1fb786753612", + "revisionTime": "2018-06-07T08:14:26Z" }, { "checksumSHA1": "tbpmGqkqN79SspYfFUgob5rxKik=", "path": "github.com/ONSdigital/go-ns/identity", - "revision": "2a4ed4b9091e2042ae489ab492711ec1a866c94b", - "revisionTime": "2018-05-02T09:37:52Z" + "revision": "e89bb271c005948b66da47ec485f1fb786753612", + "revisionTime": "2018-06-07T08:14:26Z" }, { - "checksumSHA1": "d0xkjPw9SKVWK1vDJqvD3HoFbnA=", + "checksumSHA1": "qqaN4hllMTNueffB5bcz8xQnwX0=", "path": "github.com/ONSdigital/go-ns/kafka", - "revision": "6920413b753350672215a083e0f9d5c270a21075", - "revisionTime": "2017-11-28T09:28:02Z" + "revision": "e89bb271c005948b66da47ec485f1fb786753612", + "revisionTime": "2018-06-07T08:14:26Z" }, { - "checksumSHA1": "AjbjbhFVAOP/1NU5HL+uy+X/yJo=", + "checksumSHA1": "g9flYl+PBs6cYflFY3EpG6IcJGM=", "path": "github.com/ONSdigital/go-ns/log", - "revision": "6920413b753350672215a083e0f9d5c270a21075", - "revisionTime": "2017-11-28T09:28:02Z" + "revision": "3fdd13a161b0d6221b2405cce8bbe5ec69168ab7", + "revisionTime": "2018-07-05T11:25:05Z" }, { - "checksumSHA1": "7qnU2KaLcrNltdsVw4zWEh4vmTM=", + "checksumSHA1": "/I1BHzejhYoOtk9JrLHAqmt5HTs=", "path": "github.com/ONSdigital/go-ns/mongo", - "revision": "7d241b721903e1b6a9a1f5c916cdbfa661d230f1", - "revisionTime": "2018-05-08T08:56:58Z" + "revision": "491470a66fbb2471ff3fd02f8159b7158d9281ab", + "revisionTime": "2018-06-07T13:26:14Z" }, { - "checksumSHA1": "rFpFke/Hl7rcH3ND5b4M0idiL0c=", + "checksumSHA1": "ZXfEqZ/3ekHfv7QKSUO+av7SEWA=", "path": "github.com/ONSdigital/go-ns/neo4j", - "revision": "7d241b721903e1b6a9a1f5c916cdbfa661d230f1", - "revisionTime": "2018-05-08T08:56:58Z" + "revision": "5cee3f2646232f0706d6ab2447ef617a51fc94cf", + "revisionTime": "2018-06-05T08:47:20Z" }, { - "checksumSHA1": "qIw/Zr/SVxTRbznJGR4upRI4x8M=", + "checksumSHA1": "DCXeIpFKcAMDgRMCQlrQ3qwM4Hg=", "path": "github.com/ONSdigital/go-ns/rchttp", - "revision": "2a4ed4b9091e2042ae489ab492711ec1a866c94b", - "revisionTime": "2018-05-02T09:37:52Z" + "revision": "e89bb271c005948b66da47ec485f1fb786753612", + "revisionTime": "2018-06-07T08:14:26Z" }, { "checksumSHA1": "GQdzMpAMb42KQQ/GsJFSRU5dj1Y=", "path": "github.com/ONSdigital/go-ns/server", - "revision": "6920413b753350672215a083e0f9d5c270a21075", - "revisionTime": "2017-11-28T09:28:02Z" - }, - { - "checksumSHA1": "WjSWGeNopWuQ0sPZUtw2NmnlHLo=", - "path": "github.com/ONSdigital/golang-neo4j-bolt-driver", - "revision": "8758b21e0c8b3c79bc0eca4db0fc188906e89410", - "revisionTime": "2017-12-08T13:48:35Z" - }, - { - "checksumSHA1": "FTqnhFuBw9Vh06jwKTQF2c7rml0=", - "path": "github.com/ONSdigital/golang-neo4j-bolt-driver/encoding", - "revision": "8758b21e0c8b3c79bc0eca4db0fc188906e89410", - "revisionTime": "2017-12-08T13:48:35Z" - }, - { - "checksumSHA1": "A4Cj2p77p0Q0iJYTPfMg3+ymbDs=", - "path": "github.com/ONSdigital/golang-neo4j-bolt-driver/errors", - "revision": "8758b21e0c8b3c79bc0eca4db0fc188906e89410", - "revisionTime": "2017-12-08T13:48:35Z" - }, - { - "checksumSHA1": "cmsdmQr891JU29w8APk0Yg1Swas=", - "path": "github.com/ONSdigital/golang-neo4j-bolt-driver/log", - "revision": "8758b21e0c8b3c79bc0eca4db0fc188906e89410", - "revisionTime": "2017-12-08T13:48:35Z" - }, - { - "checksumSHA1": "WCUVztJBtOykxNRmk/Qa+mbJBlk=", - "path": "github.com/ONSdigital/golang-neo4j-bolt-driver/structures", - "revision": "8758b21e0c8b3c79bc0eca4db0fc188906e89410", - "revisionTime": "2017-12-08T13:48:35Z" - }, - { - "checksumSHA1": "Jz9ShYvpqcvgORATMVBRVgEMskY=", - "path": "github.com/ONSdigital/golang-neo4j-bolt-driver/structures/graph", - "revision": "8758b21e0c8b3c79bc0eca4db0fc188906e89410", - "revisionTime": "2017-12-08T13:48:35Z" - }, - { - "checksumSHA1": "L8Ub/QM9+d1CC/csXwahYw85F3w=", - "path": "github.com/ONSdigital/golang-neo4j-bolt-driver/structures/messages", - "revision": "8758b21e0c8b3c79bc0eca4db0fc188906e89410", - "revisionTime": "2017-12-08T13:48:35Z" + "revision": "e89bb271c005948b66da47ec485f1fb786753612", + "revisionTime": "2018-06-07T08:14:26Z" }, { "checksumSHA1": "+Jp0tVXfQ1TM8T+oun82oJtME5U=", @@ -170,6 +134,36 @@ "revision": "6920413b753350672215a083e0f9d5c270a21075", "revisionTime": "2017-11-28T09:28:02Z" }, + { + "checksumSHA1": "Tsc4M+xTb2NgaCq3y8dbIcgBJ0A=", + "path": "github.com/gedge/mgo", + "revision": "a170908eafcff6f64b9ad83db3943fe6332d52a1", + "revisionTime": "2018-05-23T09:46:01Z" + }, + { + "checksumSHA1": "SAegSFbBQ3lKgxV3opxcqTPwz+o=", + "path": "github.com/gedge/mgo/bson", + "revision": "a170908eafcff6f64b9ad83db3943fe6332d52a1", + "revisionTime": "2018-05-23T09:46:01Z" + }, + { + "checksumSHA1": "+soDoin5Apt/Dyk73uC1trzHx5M=", + "path": "github.com/gedge/mgo/internal/json", + "revision": "a170908eafcff6f64b9ad83db3943fe6332d52a1", + "revisionTime": "2018-05-23T09:46:01Z" + }, + { + "checksumSHA1": "PZU+CtYXLjDXogLC2nNePsLBlkU=", + "path": "github.com/gedge/mgo/internal/sasl", + "revision": "a170908eafcff6f64b9ad83db3943fe6332d52a1", + "revisionTime": "2018-05-23T09:46:01Z" + }, + { + "checksumSHA1": "Uoi1jh9qrlyApl2Q5JgW7OW9wy4=", + "path": "github.com/gedge/mgo/internal/scram", + "revision": "a170908eafcff6f64b9ad83db3943fe6332d52a1", + "revisionTime": "2018-05-23T09:46:01Z" + }, { "checksumSHA1": "I6MnUzkLJt+sh73CodD0NKswFrs=", "origin": "github.com/ONSdigital/go-ns/vendor/github.com/go-avro/avro", @@ -202,6 +196,48 @@ "revision": "ac112f7d75a0714af1bd86ab17749b31f7809640", "revisionTime": "2017-07-03T15:07:09Z" }, + { + "checksumSHA1": "TsX+LuxHhV9GFmua8C1nxflFcTA=", + "path": "github.com/johnnadratowski/golang-neo4j-bolt-driver", + "revision": "1108d6e66ccf2c8e68ab26b5f64e6c0a2ad00899", + "revisionTime": "2017-12-18T14:36:11Z" + }, + { + "checksumSHA1": "GYtNDxyckMgJew8cMZggWD9xfhg=", + "path": "github.com/johnnadratowski/golang-neo4j-bolt-driver/encoding", + "revision": "2387cc1f01254d3a0055e034f5716278a1f420c7", + "revisionTime": "2016-12-20T21:52:15Z" + }, + { + "checksumSHA1": "9lgMFoaIFZe75vv7ln+IbPKHasE=", + "path": "github.com/johnnadratowski/golang-neo4j-bolt-driver/errors", + "revision": "2387cc1f01254d3a0055e034f5716278a1f420c7", + "revisionTime": "2016-12-20T21:52:15Z" + }, + { + "checksumSHA1": "MeB74aEJl/Vif3wfI/yyskLXgQ8=", + "path": "github.com/johnnadratowski/golang-neo4j-bolt-driver/log", + "revision": "2387cc1f01254d3a0055e034f5716278a1f420c7", + "revisionTime": "2016-12-20T21:52:15Z" + }, + { + "checksumSHA1": "QBWn/ajykCeEt1W21Ufl9RxLeX4=", + "path": "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures", + "revision": "2387cc1f01254d3a0055e034f5716278a1f420c7", + "revisionTime": "2016-12-20T21:52:15Z" + }, + { + "checksumSHA1": "vLBWZ/5wRQ4PIyIOW3dbXlKbd3s=", + "path": "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/graph", + "revision": "2387cc1f01254d3a0055e034f5716278a1f420c7", + "revisionTime": "2016-12-20T21:52:15Z" + }, + { + "checksumSHA1": "WSdNJOxRdPV0gCzq7mX6cqIf3/c=", + "path": "github.com/johnnadratowski/golang-neo4j-bolt-driver/structures/messages", + "revision": "2387cc1f01254d3a0055e034f5716278a1f420c7", + "revisionTime": "2016-12-20T21:52:15Z" + }, { "checksumSHA1": "Js/yx9fZ3+wH1wZpHNIxSTMIaCg=", "path": "github.com/jtolds/gls", @@ -310,36 +346,6 @@ "path": "golang.org/x/net/context/ctxhttp", "revision": "98b5dee4d793d5a0547b7bc7d18aceb353c88a54", "revisionTime": "2018-03-18T14:57:33Z" - }, - { - "checksumSHA1": "1D8GzeoFGUs5FZOoyC2DpQg8c5Y=", - "path": "gopkg.in/mgo.v2", - "revision": "3f83fa5005286a7fe593b055f0d7771a7dce4655", - "revisionTime": "2016-08-18T02:01:20Z" - }, - { - "checksumSHA1": "YsB2DChSV9HxdzHaKATllAUKWSI=", - "path": "gopkg.in/mgo.v2/bson", - "revision": "3f83fa5005286a7fe593b055f0d7771a7dce4655", - "revisionTime": "2016-08-18T02:01:20Z" - }, - { - "checksumSHA1": "XQsrqoNT1U0KzLxOFcAZVvqhLfk=", - "path": "gopkg.in/mgo.v2/internal/json", - "revision": "3f83fa5005286a7fe593b055f0d7771a7dce4655", - "revisionTime": "2016-08-18T02:01:20Z" - }, - { - "checksumSHA1": "LEvMCnprte47qdAxWvQ/zRxVF1U=", - "path": "gopkg.in/mgo.v2/internal/sasl", - "revision": "3f83fa5005286a7fe593b055f0d7771a7dce4655", - "revisionTime": "2016-08-18T02:01:20Z" - }, - { - "checksumSHA1": "+1WDRPaOphSCmRMxVPIPBV4aubc=", - "path": "gopkg.in/mgo.v2/internal/scram", - "revision": "3f83fa5005286a7fe593b055f0d7771a7dce4655", - "revisionTime": "2016-08-18T02:01:20Z" } ], "rootPath": "github.com/ONSdigital/dp-dataset-api"