diff --git a/README.md b/README.md index 97bf8fda..403882be 100644 --- a/README.md +++ b/README.md @@ -90,4 +90,4 @@ See [CONTRIBUTING](CONTRIBUTING.md) for details. Copyright © 2016-2021, Office for National Statistics (https://www.ons.gov.uk) -Released under MIT license, see [LICENSE](LICENSE.md) for details +Released under MIT license, see [LICENSE](LICENSE.md) for details. diff --git a/api/api.go b/api/api.go index bd00fa02..8997ea47 100644 --- a/api/api.go +++ b/api/api.go @@ -51,46 +51,44 @@ type AuthHandler interface { // DatasetAPI manages importing filters against a dataset type DatasetAPI struct { - Router *mux.Router - dataStore store.DataStore - urlBuilder *url.Builder - host string - downloadServiceToken string - EnablePrePublishView bool - downloadGenerator DownloadsGenerator - enablePrivateEndpoints bool - enableDetachDataset bool - enableObservationEndpoint bool - datasetPermissions AuthHandler - permissions AuthHandler - instancePublishedChecker *instance.PublishCheck - versionPublishedChecker *PublishCheck - defaultLimit int - defaultOffset int - maxLimit int + Router *mux.Router + dataStore store.DataStore + urlBuilder *url.Builder + host string + downloadServiceToken string + EnablePrePublishView bool + downloadGenerator DownloadsGenerator + enablePrivateEndpoints bool + enableDetachDataset bool + datasetPermissions AuthHandler + permissions AuthHandler + instancePublishedChecker *instance.PublishCheck + versionPublishedChecker *PublishCheck + defaultLimit int + defaultOffset int + maxLimit int } // Setup creates a new Dataset API instance and register the API routes based on the application configuration. func Setup(ctx context.Context, cfg *config.Configuration, router *mux.Router, dataStore store.DataStore, urlBuilder *url.Builder, downloadGenerator DownloadsGenerator, datasetPermissions AuthHandler, permissions AuthHandler) *DatasetAPI { api := &DatasetAPI{ - dataStore: dataStore, - host: cfg.DatasetAPIURL, - downloadServiceToken: cfg.DownloadServiceSecretKey, - EnablePrePublishView: cfg.EnablePrivateEndpoints, - Router: router, - urlBuilder: urlBuilder, - downloadGenerator: downloadGenerator, - enablePrivateEndpoints: cfg.EnablePrivateEndpoints, - enableDetachDataset: cfg.EnableDetachDataset, - enableObservationEndpoint: cfg.EnableObservationEndpoint, - datasetPermissions: datasetPermissions, - permissions: permissions, - versionPublishedChecker: nil, - instancePublishedChecker: nil, - defaultLimit: cfg.DefaultLimit, - defaultOffset: cfg.DefaultOffset, - maxLimit: cfg.DefaultMaxLimit, + dataStore: dataStore, + host: cfg.DatasetAPIURL, + downloadServiceToken: cfg.DownloadServiceSecretKey, + EnablePrePublishView: cfg.EnablePrivateEndpoints, + Router: router, + urlBuilder: urlBuilder, + downloadGenerator: downloadGenerator, + enablePrivateEndpoints: cfg.EnablePrivateEndpoints, + enableDetachDataset: cfg.EnableDetachDataset, + datasetPermissions: datasetPermissions, + permissions: permissions, + versionPublishedChecker: nil, + instancePublishedChecker: nil, + defaultLimit: cfg.DefaultLimit, + defaultOffset: cfg.DefaultOffset, + maxLimit: cfg.DefaultMaxLimit, } if api.enablePrivateEndpoints { @@ -138,10 +136,6 @@ func (api *DatasetAPI) enablePublicEndpoints(ctx context.Context) { api.get("/datasets/{dataset_id}/editions/{edition}/versions/{version}/dimensions", api.getDimensions) api.get("/datasets/{dataset_id}/editions/{edition}/versions/{version}/dimensions/{dimension}/options", api.getDimensionOptions) - if api.enableObservationEndpoint { - log.Event(ctx, "enabling observations endpoint", log.INFO) - api.get("/datasets/{dataset_id}/editions/{edition}/versions/{version}/observations", api.getObservations) - } } // enablePrivateDatasetEndpoints register the datasets endpoints with the appropriate authentication and authorisation @@ -187,15 +181,6 @@ func (api *DatasetAPI) enablePrivateDatasetEndpoints(ctx context.Context) { api.getMetadata), ) - if api.enableObservationEndpoint { - log.Event(ctx, "enabling observations endpoint", log.INFO) - api.get( - "/datasets/{dataset_id}/editions/{edition}/versions/{version}/observations", - api.isAuthorisedForDatasets(readPermission, - api.getObservations), - ) - } - api.get( "/datasets/{dataset_id}/editions/{edition}/versions/{version}/dimensions", api.isAuthorisedForDatasets(readPermission, diff --git a/api/dataset.go b/api/dataset.go index e60e992b..d5b4e73a 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -327,6 +327,14 @@ func (api *DatasetAPI) putDataset(w http.ResponseWriter, r *http.Request) { return err } } else { + if err := models.CleanDataset(dataset); err != nil { + log.Event(ctx, "could not clean dataset", log.ERROR, log.Error(err)) + return nil + } + if err := models.ValidateDataset(dataset); err != nil { + log.Event(ctx, "failed validation check to update dataset", log.ERROR, log.Error(err)) + return nil + } if err := api.dataStore.Backend.UpdateDataset(ctx, datasetID, dataset, currentDataset.Next.State); err != nil { log.Event(ctx, "putDataset endpoint: failed to update dataset resource", log.ERROR, log.Error(err), data) return err diff --git a/api/observation.go b/api/observation.go deleted file mode 100644 index 2927c9c5..00000000 --- a/api/observation.go +++ /dev/null @@ -1,397 +0,0 @@ -package api - -import ( - "context" - "encoding/csv" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "strings" - - errs "github.com/ONSdigital/dp-dataset-api/apierrors" - "github.com/ONSdigital/dp-dataset-api/models" - "github.com/ONSdigital/dp-graph/v2/observation" - "github.com/ONSdigital/log.go/log" - "github.com/gorilla/mux" - "github.com/pkg/errors" -) - -// Upper limit, if this is not big enough, we may need to consider increasing value -// and then if this has a performance hit then consider paging -const ( - defaultObservationLimit = 10000 - defaultOffset = 0 -) - -var ( - observationNotFound = map[error]bool{ - errs.ErrDatasetNotFound: true, - errs.ErrEditionNotFound: true, - errs.ErrVersionNotFound: true, - errs.ErrObservationsNotFound: true, - } - - observationBadRequest = map[error]bool{ - errs.ErrTooManyWildcards: true, - } -) - -type observationQueryError struct { - message string -} - -func (e observationQueryError) Error() string { - return e.message -} - -func errorIncorrectQueryParameters(params []string) error { - return observationQueryError{ - message: fmt.Sprintf("incorrect selection of query parameters: %v, these dimensions do not exist for this version of the dataset", params), - } -} - -func errorMissingQueryParameters(params []string) error { - return observationQueryError{ - message: fmt.Sprintf("missing query parameters for the following dimensions: %v", params), - } -} - -func errorMultivaluedQueryParameters(params []string) error { - return observationQueryError{ - message: fmt.Sprintf("multi-valued query parameters for the following dimensions: %v", params), - } -} - -func (api *DatasetAPI) getObservations(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() - vars := mux.Vars(r) - datasetID := vars["dataset_id"] - edition := vars["edition"] - version := vars["version"] - logData := log.Data{"dataset_id": datasetID, "edition": edition, "version": version} - - observationsDoc, err := func() (*models.ObservationsDoc, error) { - // get dataset document - datasetDoc, err := api.dataStore.Backend.GetDataset(datasetID) - if err != nil { - log.Event(ctx, "get observations: datastore.GetDataset returned an error", log.ERROR, log.Error(err), logData) - return nil, err - } - - authorised := api.authenticate(r, logData) - - var ( - state string - dataset *models.Dataset - ) - - // if request is not authenticated then only access resources of state published - if !authorised { - // Check for current sub document - if datasetDoc.Current == nil || datasetDoc.Current.State != models.PublishedState { - logData["dataset_doc"] = datasetDoc.Current - log.Event(ctx, "get observations: found no published dataset", log.ERROR, log.Error(errs.ErrDatasetNotFound), logData) - return nil, errs.ErrDatasetNotFound - } - - dataset = datasetDoc.Current - state = dataset.State - } else { - dataset = datasetDoc.Next - } - - if err = api.dataStore.Backend.CheckEditionExists(datasetID, edition, state); err != nil { - log.Event(ctx, "get observations: failed to find edition for dataset", log.ERROR, log.Error(err), logData) - return nil, err - } - - versionDoc, err := api.dataStore.Backend.GetVersion(datasetID, edition, version, state) - if err != nil { - log.Event(ctx, "get observations: failed to find version for dataset edition", log.ERROR, log.Error(err), logData) - return nil, err - } - - if err = models.CheckState("version", versionDoc.State); err != nil { - logData["state"] = versionDoc.State - log.Event(ctx, "get observations: unpublished version has an invalid state", log.ERROR, log.Error(err), logData) - return nil, err - } - - if versionDoc.Headers == nil || versionDoc.Dimensions == nil { - logData["version_doc"] = versionDoc - log.Event(ctx, "get observations", log.ERROR, log.Error(errs.ErrMissingVersionHeadersOrDimensions), logData) - return nil, errs.ErrMissingVersionHeadersOrDimensions - } - - // loop through version dimensions to retrieve list of dimension names - validDimensionNames := getListOfValidDimensionNames(versionDoc.Dimensions) - logData["version_dimensions"] = validDimensionNames - - dimensionOffset, err := getDimensionOffsetInHeaderRow(versionDoc.Headers) - if err != nil { - log.Event(ctx, "get observations: unable to distinguish headers from version document", log.ERROR, log.Error(err), logData) - return nil, err - } - - // check query parameters match the version headers - queryParameters, err := extractQueryParameters(r.URL.Query(), validDimensionNames) - if err != nil { - log.Event(ctx, "get observations: error extracting query parameters", log.ERROR, log.Error(err), logData) - return nil, err - } - logData["query_parameters"] = queryParameters - - // retrieve observations - observations, err := api.getObservationList(ctx, versionDoc, queryParameters, defaultObservationLimit, dimensionOffset, logData) - if err != nil { - log.Event(ctx, "get observations: unable to retrieve observations", log.ERROR, log.Error(err), logData) - return nil, err - } - - return models.CreateObservationsDoc(r.URL.RawQuery, versionDoc, dataset, observations, queryParameters, defaultOffset, defaultObservationLimit), nil - }() - - if err != nil { - handleObservationsErrorType(ctx, w, err, logData) - return - } - - setJSONContentType(w) - - // The ampersand "&" is escaped to "\u0026" to keep some browsers from - // misinterpreting JSON output as HTML. This escaping can be disabled using - // an Encoder that had SetEscapeHTML(false) called on it. - enc := json.NewEncoder(w) - enc.SetEscapeHTML(false) - - if err = enc.Encode(observationsDoc); err != nil { - handleObservationsErrorType(ctx, w, errors.WithMessage(err, "failed to marshal metadata resource into bytes"), logData) - return - } - - log.Event(ctx, "get observations endpoint: successfully retrieved observations relative to a selected set of dimension options for a version", log.INFO, logData) -} - -func getDimensionOffsetInHeaderRow(headerRow []string) (int, error) { - metaData := strings.Split(headerRow[0], "_") - - if len(metaData) < 2 { - return 0, errs.ErrIndexOutOfRange - } - - dimensionOffset, err := strconv.Atoi(metaData[1]) - if err != nil { - return 0, err - } - - return dimensionOffset, nil -} - -func getListOfValidDimensionNames(dimensions []models.Dimension) []string { - - var dimensionNames []string - for _, dimension := range dimensions { - dimensionNames = append(dimensionNames, dimension.Name) - } - - return dimensionNames -} - -func extractQueryParameters(urlQuery url.Values, validDimensions []string) (map[string]string, error) { - queryParameters := make(map[string]string) - var incorrectQueryParameters, missingQueryParameters, multivaluedQueryParameters []string - - // Determine if any request query parameters are invalid dimensions - // and map the valid dimensions with their equivalent values in map - for rawDimension, option := range urlQuery { - // Ignore case sensitivity - dimension := strings.ToLower(rawDimension) - - queryParamExists := false - for _, validDimension := range validDimensions { - if dimension == validDimension { - queryParamExists = true - queryParameters[dimension] = option[0] - if len(option) != 1 { - multivaluedQueryParameters = append(multivaluedQueryParameters, rawDimension) - } - break - } - } - if !queryParamExists { - incorrectQueryParameters = append(incorrectQueryParameters, rawDimension) - } - } - - if len(incorrectQueryParameters) > 0 { - return nil, errorIncorrectQueryParameters(incorrectQueryParameters) - } - - if len(multivaluedQueryParameters) > 0 { - return nil, errorMultivaluedQueryParameters(multivaluedQueryParameters) - } - - // Determine if any dimensions have not been set in request query parameters - if len(queryParameters) != len(validDimensions) { - for _, validDimension := range validDimensions { - if queryParameters[validDimension] == "" { - missingQueryParameters = append(missingQueryParameters, validDimension) - } - } - return nil, errorMissingQueryParameters(missingQueryParameters) - } - - return queryParameters, nil -} - -func (api *DatasetAPI) getObservationList(ctx context.Context, versionDoc *models.Version, queryParameters map[string]string, limit, dimensionOffset int, logData log.Data) ([]models.Observation, error) { - - // Build query (observation.Filter type) - var dimensionFilters []*observation.Dimension - - // Unable to have more than one wildcard parameter per query - var wildcardParameter string - - // Build dimension filter object to create queryObject for neo4j - for dimension, option := range queryParameters { - if option == "*" { - if wildcardParameter != "" { - return nil, errs.ErrTooManyWildcards - } - - wildcardParameter = dimension - continue - } - - dimensionFilter := &observation.Dimension{ - Name: dimension, - Options: []string{option}, - } - - dimensionFilters = append(dimensionFilters, dimensionFilter) - } - - queryObject := observation.DimensionFilters{ - Dimensions: dimensionFilters, - } - - logData["query_object"] = queryObject - - log.Event(ctx, "query object built to retrieve observations from db", log.INFO, logData) - - csvRowReader, err := api.dataStore.Backend.StreamCSVRows(ctx, versionDoc.ID, "", &queryObject, &limit) - if err != nil { - return nil, err - } - - headerRow, err := csvRowReader.Read() - if err != nil { - return nil, err - } - - defer csvRowReader.Close(ctx) - - headerRowReader := csv.NewReader(strings.NewReader(headerRow)) - headerRowArray, err := headerRowReader.Read() - if err != nil { - return nil, err - } - - var observationRow string - var observations []models.Observation - // Iterate over observation row reader - for observationRow, err = csvRowReader.Read(); err != io.EOF; observationRow, err = csvRowReader.Read() { - - if err != nil { - if strings.Contains(err.Error(), "the filter options created no results") { - return nil, errs.ErrObservationsNotFound - } - return nil, err - } - - observationRowReader := csv.NewReader(strings.NewReader(observationRow)) - observationRowArray, err := observationRowReader.Read() - if err != nil { - return nil, err - } - - // TODO for the below maybe put this in a seperate function? - observation := models.Observation{ - Observation: observationRowArray[0], - } - - // add observation metadata - if dimensionOffset != 0 { - observationMetaData := make(map[string]string) - - for i := 1; i < dimensionOffset+1; i++ { - observationMetaData[headerRowArray[i]] = observationRowArray[i] - } - - observation.Metadata = observationMetaData - } - - if wildcardParameter != "" { - dimensions := make(map[string]*models.DimensionObject) - - for i := dimensionOffset + 2; i < len(observationRowArray); i += 2 { - - if strings.ToLower(headerRowArray[i]) == wildcardParameter { - for _, versionDimension := range versionDoc.Dimensions { - if versionDimension.Name == wildcardParameter { - - dimensions[headerRowArray[i]] = &models.DimensionObject{ - ID: observationRowArray[i-1], - HRef: versionDimension.HRef + "/codes/" + observationRowArray[i-1], - Label: observationRowArray[i], - } - - break - } - } - - break - } - } - observation.Dimensions = dimensions - } - - observations = append(observations, observation) - } - - // neo4j will always return the same list of observations in the same - // order as it is deterministic for static data, but this does not - // necessarily mean we won't want to return observations in a particular - // order (which may be costly on the services performance) - - return observations, nil -} - -func handleObservationsErrorType(ctx context.Context, w http.ResponseWriter, err error, data log.Data) { - _, isObservationErr := err.(observationQueryError) - var status int - resErrMsg := err.Error() - - switch { - case isObservationErr: - status = http.StatusBadRequest - case observationNotFound[err]: - status = http.StatusNotFound - case observationBadRequest[err]: - status = http.StatusBadRequest - default: - resErrMsg = errs.ErrInternalServer.Error() - status = http.StatusInternalServerError - } - - if data == nil { - data = log.Data{} - } - - data["responseStatus"] = status - log.Event(ctx, "get observation endpoint: request unsuccessful", log.ERROR, log.Error(err), data) - http.Error(w, resErrMsg, status) -} diff --git a/api/observation_test.go b/api/observation_test.go deleted file mode 100644 index 43578c91..00000000 --- a/api/observation_test.go +++ /dev/null @@ -1,876 +0,0 @@ -package api - -import ( - "bytes" - "context" - "encoding/json" - "io" - "io/ioutil" - "net/http" - "net/http/httptest" - "os" - "testing" - - errs "github.com/ONSdigital/dp-dataset-api/apierrors" - "github.com/ONSdigital/dp-dataset-api/mocks" - "github.com/ONSdigital/dp-dataset-api/models" - storetest "github.com/ONSdigital/dp-dataset-api/store/datastoretest" - "github.com/ONSdigital/dp-graph/v2/observation" - observationtest "github.com/ONSdigital/dp-graph/v2/observation/observationtest" - "github.com/ONSdigital/log.go/log" - . "github.com/smartystreets/goconvey/convey" -) - -var ( - dimension1 = models.Dimension{Name: "aggregate"} - dimension2 = models.Dimension{Name: "geography"} - dimension3 = models.Dimension{Name: "time"} - dimension4 = models.Dimension{Name: "age"} -) - -func TestGetObservationsReturnsOK(t *testing.T) { - ctx := context.Background() - - t.Parallel() - Convey("Given a request to get a single observation for a version of a dataset returns 200 OK response", t, func() { - - dimensions := []models.Dimension{ - { - Name: "aggregate", - HRef: "http://localhost:8081/code-lists/cpih1dim1aggid", - }, - { - Name: "geography", - HRef: "http://localhost:8081/code-lists/uk-only", - }, - { - Name: "time", - HRef: "http://localhost:8081/code-lists/time", - }, - } - usagesNotes := &[]models.UsageNote{{Title: "data_marking", Note: "this marks the obsevation with a special character"}} - - count := 0 - mockRowReader := &observationtest.StreamRowReaderMock{ - ReadFunc: func() (string, error) { - count++ - if count == 1 { - return "v4_2,data_marking,confidence_interval,time,time,geography_code,geography,aggregate_code,aggregate", nil - } else if count == 2 { - return "146.3,p,2,Month,Aug-16,K02000001,,cpi1dim1G10100,01.1 Food", nil - } - return "", io.EOF - }, - CloseFunc: func(context.Context) error { - return nil - }, - } - - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(string, string, string, string) (*models.Version, error) { - return &models.Version{ - Dimensions: dimensions, - Headers: []string{"v4_2", "data_marking", "confidence_interval", "aggregate_code", "aggregate", "geography_code", "geography", "time", "time"}, - Links: &models.VersionLinks{ - Version: &models.LinkObject{ - HRef: "http://localhost:8080/datasets/cpih012/editions/2017/versions/1", - ID: "1", - }, - }, - State: models.PublishedState, - UsageNotes: usagesNotes, - }, nil - }, - StreamCSVRowsFunc: func(context.Context, string, string, *observation.DimensionFilters, *int) (observation.StreamRowReader, error) { - return mockRowReader, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - - Convey("When request contains query parameters where the dimension name is in lower casing", func() { - r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusOK) - So(w.Body.String(), ShouldContainSubstring, getTestData(ctx, "expectedDocWithSingleObservation")) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.StreamCSVRowsCalls()), ShouldEqual, 1) - So(len(mockRowReader.ReadCalls()), ShouldEqual, 3) - }) - - Convey("When request contains query parameters where the dimension name is in upper casing", func() { - r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&AggregaTe=cpi1dim1S40403&GEOGRAPHY=K02000001", nil) - w := httptest.NewRecorder() - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusOK) - So(w.Body.String(), ShouldContainSubstring, getTestData(ctx, "expectedSecondDocWithSingleObservation")) - - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.StreamCSVRowsCalls()), ShouldEqual, 1) - So(len(mockRowReader.ReadCalls()), ShouldEqual, 3) - }) - }) - - Convey("A successful request to get multiple observations via a wildcard for a version of a dataset returns 200 OK response", t, func() { - r := httptest.NewRequest("GET", "http://localhost:8080/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=*&geography=K02000001", nil) - w := httptest.NewRecorder() - - dimensions := []models.Dimension{ - { - Name: "aggregate", - HRef: "http://localhost:8081/code-lists/cpih1dim1aggid", - }, - { - Name: "geography", - HRef: "http://localhost:8081/code-lists/uk-only", - }, - { - Name: "time", - HRef: "http://localhost:8081/code-lists/time", - }, - } - usagesNotes := &[]models.UsageNote{{Title: "data_marking", Note: "this marks the observation with a special character"}} - - count := 0 - mockRowReader := &observationtest.StreamRowReaderMock{ - ReadFunc: func() (string, error) { - count++ - if count == 1 { - return "v4_2,data_marking,confidence_interval,time,time,geography_code,geography,aggregate_code,aggregate", nil - } else if count == 2 { - return "146.3,p,2,Month,Aug-16,K02000001,,cpi1dim1G10100,01.1 Food", nil - } else if count == 3 { - return "112.1,,,Month,Aug-16,K02000001,,cpi1dim1G10101,01.2 Waste", nil - } - return "", io.EOF - }, - CloseFunc: func(context.Context) error { - return nil - }, - } - - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(string, string, string, string) (*models.Version, error) { - return &models.Version{ - Dimensions: dimensions, - Headers: []string{"v4_2", "data_marking", "confidence_interval", "aggregate_code", "aggregate", "geography_code", "geography", "time", "time"}, - Links: &models.VersionLinks{ - Version: &models.LinkObject{ - HRef: "http://localhost:8080/datasets/cpih012/editions/2017/versions/1", - ID: "1", - }, - }, - State: models.PublishedState, - UsageNotes: usagesNotes, - }, nil - }, - StreamCSVRowsFunc: func(context.Context, string, string, *observation.DimensionFilters, *int) (observation.StreamRowReader, error) { - return mockRowReader, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusOK) - So(w.Body.String(), ShouldContainSubstring, getTestData(ctx, "expectedDocWithMultipleObservations")) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.StreamCSVRowsCalls()), ShouldEqual, 1) - So(len(mockRowReader.ReadCalls()), ShouldEqual, 4) - }) -} - -func TestGetObservationsReturnsError(t *testing.T) { - t.Parallel() - Convey("When the api cannot connect to mongo datastore return an internal server error", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return nil, errs.ErrInternalServer - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - So(w.Body.String(), ShouldResemble, "internal error\n") - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) - }) - - Convey("When the dataset does not exist return status not found", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return nil, errs.ErrDatasetNotFound - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) - }) - - Convey("When the dataset exists but is unpublished return status not found", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{}, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldContainSubstring, errs.ErrDatasetNotFound.Error()) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 0) - }) - - Convey("When the edition of a dataset does not exist return status not found", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return errs.ErrEditionNotFound - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldContainSubstring, errs.ErrEditionNotFound.Error()) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionsCalls()), ShouldEqual, 0) - }) - - Convey("When version does not exist for an edition of a dataset returns status not found", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return nil, errs.ErrVersionNotFound - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldContainSubstring, errs.ErrVersionNotFound.Error()) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When an unpublished version has an incorrect state for an edition of a dataset return an internal error", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{State: "gobbly-gook"}, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - assertInternalServerErr(w) - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When a version document has not got a headers field return an internal server error", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When a version document has not got any dimensions field return an internal server error", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Headers: []string{"v4_0", "time_code", "time", "aggregate_code", "aggregate"}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusInternalServerError) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When the first header in array does not describe the header row correctly return internal error", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3}, - Headers: []string{"v4"}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - assertInternalServerErr(w) - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When an invalid query parameter is set in request return 400 bad request with an error message containing a list of incorrect query parameters", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension3}, - Headers: []string{"v4_0", "time_code", "time", "aggregate_code", "aggregate"}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "incorrect selection of query parameters: [geography], these dimensions do not exist for this version of the dataset\n") - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When there is a missing query parameter that is expected to be set in request return 400 bad request with an error message containing a list of missing query parameters", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3, dimension4}, - Headers: []string{"v4_0", "time_code", "time", "aggregate_code", "aggregate", "geography_code", "geography", "age_code", "age"}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "missing query parameters for the following dimensions: [age]\n") - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When there are too many query parameters that are set to wildcard (*) value request returns 400 bad request", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=*&aggregate=*&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3}, - Headers: []string{"v4_0", "time_code", "time", "aggregate_code", "aggregate", "geography_code", "geography"}, - State: models.PublishedState, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "only one wildcard (*) is allowed as a value in selected query parameters\n") - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - }) - - Convey("When requested query does not find a unique observation return no observations found", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001", nil) - w := httptest.NewRecorder() - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(datasetID string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(datasetID, editionID, version, state string) (*models.Version, error) { - return &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3}, - Headers: []string{"v4_0", "time_code", "time", "aggregate_code", "aggregate", "geography_code", "geography"}, - State: models.PublishedState, - }, - nil - }, - StreamCSVRowsFunc: func(context.Context, string, string, *observation.DimensionFilters, *int) (observation.StreamRowReader, error) { - return nil, errs.ErrObservationsNotFound - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusNotFound) - So(w.Body.String(), ShouldContainSubstring, errs.ErrObservationsNotFound.Error()) - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.StreamCSVRowsCalls()), ShouldEqual, 1) - }) - - Convey("When requested query has a multi-valued dimension return bad request", t, func() { - r := httptest.NewRequest("GET", "http://localhost:22000/datasets/cpih012/editions/2017/versions/1/observations?time=16-Aug&aggregate=cpi1dim1S40403&geography=K02000001&geography=K02000002", nil) - w := httptest.NewRecorder() - - dimensions := []models.Dimension{ - { - Name: "aggregate", - HRef: "http://localhost:8081/code-lists/cpih1dim1aggid", - }, - { - Name: "geography", - HRef: "http://localhost:8081/code-lists/uk-only", - }, - { - Name: "time", - HRef: "http://localhost:8081/code-lists/time", - }, - } - usagesNotes := &[]models.UsageNote{{Title: "data_marking", Note: "this marks the obsevation with a special character"}} - - mockedDataStore := &storetest.StorerMock{ - GetDatasetFunc: func(string) (*models.DatasetUpdate, error) { - return &models.DatasetUpdate{Current: &models.Dataset{State: models.PublishedState}}, nil - }, - CheckEditionExistsFunc: func(datasetID, editionID, state string) error { - return nil - }, - GetVersionFunc: func(string, string, string, string) (*models.Version, error) { - return &models.Version{ - Dimensions: dimensions, - Headers: []string{"v4_2", "data_marking", "confidence_interval", "aggregate_code", "aggregate", "geography_code", "geography", "time", "time"}, - Links: &models.VersionLinks{ - Version: &models.LinkObject{ - HRef: "http://localhost:8080/datasets/cpih012/editions/2017/versions/1", - ID: "1", - }, - }, - State: models.PublishedState, - UsageNotes: usagesNotes, - }, nil - }, - } - - datasetPermissions := getAuthorisationHandlerMock() - permissions := getAuthorisationHandlerMock() - api := GetAPIWithMocks(mockedDataStore, &mocks.DownloadsGeneratorMock{}, datasetPermissions, permissions) - api.Router.ServeHTTP(w, r) - - So(w.Code, ShouldEqual, http.StatusBadRequest) - So(w.Body.String(), ShouldResemble, "multi-valued query parameters for the following dimensions: [geography]\n") - - So(datasetPermissions.Required.Calls, ShouldEqual, 1) - So(permissions.Required.Calls, ShouldEqual, 0) - So(len(mockedDataStore.GetDatasetCalls()), ShouldEqual, 1) - So(len(mockedDataStore.CheckEditionExistsCalls()), ShouldEqual, 1) - So(len(mockedDataStore.GetVersionCalls()), ShouldEqual, 1) - So(len(mockedDataStore.StreamCSVRowsCalls()), ShouldEqual, 0) - }) -} - -func TestGetListOfValidDimensionNames(t *testing.T) { - t.Parallel() - Convey("Given a list of valid dimension codelist objects", t, func() { - Convey("When getListOfValidDimensionNames is called", func() { - dimension1 := models.Dimension{ - Name: "time", - } - - dimension2 := models.Dimension{ - Name: "aggregate", - } - - dimension3 := models.Dimension{ - Name: "geography", - } - - version := &models.Version{ - Dimensions: []models.Dimension{dimension1, dimension2, dimension3}, - } - - Convey("Then func returns the correct number of dimensions", func() { - validDimensions := getListOfValidDimensionNames(version.Dimensions) - - So(len(validDimensions), ShouldEqual, 3) - So(validDimensions[0], ShouldEqual, "time") - So(validDimensions[1], ShouldEqual, "aggregate") - So(validDimensions[2], ShouldEqual, "geography") - }) - }) - }) -} - -func TestGetDimensionOffsetInHeaderRow(t *testing.T) { - t.Parallel() - Convey("Given the version headers are valid", t, func() { - Convey("When the version has no metadata headers", func() { - version := &models.Version{ - Headers: []string{ - "v4_0", - "time_codelist", - "time", - "aggregate_codelist", - "Aggregate", - "geography_codelist", - "geography", - }, - } - - Convey("Then getListOfValidDimensionNames func returns the correct number of headers", func() { - dimensionOffset, err := getDimensionOffsetInHeaderRow(version.Headers) - - So(err, ShouldBeNil) - So(dimensionOffset, ShouldEqual, 0) - }) - }) - - Convey("When the version has metadata headers", func() { - version := &models.Version{ - Headers: []string{ - "V4_2", - "data_marking", - "confidence_interval", - "time_codelist", - "time", - }, - } - - Convey("Then getListOfValidDimensionNames func returns the correct number of headers", func() { - dimensionOffset, err := getDimensionOffsetInHeaderRow(version.Headers) - - So(err, ShouldBeNil) - So(dimensionOffset, ShouldEqual, 2) - }) - }) - }) - - Convey("Given the first value in the header does not have an underscore `_` in value", t, func() { - Convey("When the getListOfValidDimensionNames func is called", func() { - version := &models.Version{ - Headers: []string{ - "v4", - "time_codelist", - "time", - "aggregate_codelist", - "aggregate", - "geography_codelist", - "geography", - }, - } - Convey("Then function returns error, `index out of range`", func() { - dimensionOffset, err := getDimensionOffsetInHeaderRow(version.Headers) - - So(err, ShouldNotBeNil) - So(err.Error(), ShouldResemble, "index out of range") - So(dimensionOffset, ShouldEqual, 0) - }) - }) - }) - - Convey("Given the first value in the header does not follow the format `v4_1`", t, func() { - Convey("When the getListOfValidDimensionNames func is called", func() { - version := &models.Version{ - Headers: []string{ - "v4_one", - "time_codelist", - "time", - "aggregate_codelist", - "aggregate", - "geography_codelist", - "geography", - }, - } - Convey("Then function returns error, `index out of range`", func() { - dimensionOffset, err := getDimensionOffsetInHeaderRow(version.Headers) - - So(err, ShouldNotBeNil) - So(err.Error(), ShouldResemble, "strconv.Atoi: parsing \"one\": invalid syntax") - So(dimensionOffset, ShouldEqual, 0) - }) - }) - }) -} - -func TestExtractQueryParameters(t *testing.T) { - t.Parallel() - Convey("Given a list of valid dimension headers for version", t, func() { - headers := []string{ - "time", - "aggregate", - "geography", - } - - Convey("When a request is made containing query parameters for each dimension/header", func() { - r, err := http.NewRequest("GET", - "http://localhost:22000/datasets/123/editions/2017/versions/1/observations?time=JAN08&aggregate=Overall Index&geography=wales", - nil, - ) - So(err, ShouldBeNil) - - Convey("Then extractQueryParameters func returns a list of query parameters and their corresponding value", func() { - queryParameters, err := extractQueryParameters(r.URL.Query(), headers) - So(err, ShouldBeNil) - So(len(queryParameters), ShouldEqual, 3) - So(queryParameters["time"], ShouldEqual, "JAN08") - So(queryParameters["aggregate"], ShouldEqual, "Overall Index") - So(queryParameters["geography"], ShouldEqual, "wales") - }) - }) - - Convey("When a request is made containing query parameters for 2/3 dimensions/headers", func() { - r, err := http.NewRequest("GET", - "http://localhost:22000/datasets/123/editions/2017/versions/1/observations?time=JAN08&geography=wales", - nil, - ) - So(err, ShouldBeNil) - - Convey("Then extractQueryParameters func returns an error", func() { - queryParameters, err := extractQueryParameters(r.URL.Query(), headers) - So(err, ShouldNotBeNil) - So(err, ShouldResemble, errorMissingQueryParameters([]string{"aggregate"})) - So(queryParameters, ShouldBeNil) - }) - }) - - Convey("When a request is made containing all query parameters for each dimensions/headers but also an invalid one", func() { - r, err := http.NewRequest("GET", - "http://localhost:22000/datasets/123/editions/2017/versions/1/observations?time=JAN08&aggregate=Food&geography=wales&age=52", - nil, - ) - So(err, ShouldBeNil) - - Convey("Then extractQueryParameters func returns an error", func() { - queryParameters, err := extractQueryParameters(r.URL.Query(), headers) - So(err, ShouldNotBeNil) - So(err, ShouldResemble, errorIncorrectQueryParameters([]string{"age"})) - So(queryParameters, ShouldBeNil) - }) - }) - - Convey("When a request is made containing all query parameters for each dimensions/headers but there is a duplicate", func() { - r, err := http.NewRequest("GET", - "http://localhost:22000/datasets/123/editions/2017/versions/1/observations?time=JAN08&aggregate=Food&geography=wales&time=JAN0", - nil, - ) - So(err, ShouldBeNil) - - Convey("Then extractQueryParameters func returns an error", func() { - queryParameters, err := extractQueryParameters(r.URL.Query(), headers) - So(err, ShouldNotBeNil) - So(err, ShouldResemble, errorMultivaluedQueryParameters([]string{"time"})) - So(queryParameters, ShouldBeNil) - }) - }) - }) -} - -func getTestData(ctx context.Context, filename string) string { - jsonBytes, err := ioutil.ReadFile("./observation_test_data/" + filename + ".json") - if err != nil { - log.Event(ctx, "unable to read json file into bytes", log.ERROR, log.Error(err), log.Data{"filename": filename}) - os.Exit(1) - } - buffer := new(bytes.Buffer) - if err := json.Compact(buffer, jsonBytes); err != nil { - log.Event(ctx, "unable to remove whitespace from json bytes", log.ERROR, log.Error(err), log.Data{"filename": filename}) - os.Exit(1) - } - - return buffer.String() -} diff --git a/config/config_test.go b/config/config_test.go index d03f5919..8d424296 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -38,7 +38,6 @@ func TestSpec(t *testing.T) { So(cfg.EnablePermissionsAuth, ShouldBeFalse) So(cfg.HealthCheckCriticalTimeout, ShouldEqual, 90*time.Second) So(cfg.HealthCheckInterval, ShouldEqual, 30*time.Second) - So(cfg.EnableObservationEndpoint, ShouldEqual, true) }) }) }) diff --git a/features/steps/dataset_feature.go b/features/steps/dataset_feature.go index 178e56b0..2dd359e2 100644 --- a/features/steps/dataset_feature.go +++ b/features/steps/dataset_feature.go @@ -6,6 +6,7 @@ import ( "net/http" "time" + componenttest "github.com/ONSdigital/dp-component-test" "github.com/ONSdigital/dp-dataset-api/config" "github.com/ONSdigital/dp-dataset-api/models" "github.com/ONSdigital/dp-dataset-api/mongo" @@ -16,7 +17,6 @@ import ( "github.com/ONSdigital/dp-healthcheck/healthcheck" kafka "github.com/ONSdigital/dp-kafka/v2" "github.com/ONSdigital/dp-kafka/v2/kafkatest" - featuretest "github.com/armakuni/dp-go-featuretest" "github.com/benweissmann/memongo" "github.com/cucumber/godog" "github.com/globalsign/mgo" @@ -25,7 +25,7 @@ import ( ) type DatasetFeature struct { - ErrorFeature featuretest.ErrorFeature + ErrorFeature componenttest.ErrorFeature svc *service.Service errorChan chan error Datasets []*models.Dataset @@ -35,7 +35,7 @@ type DatasetFeature struct { ServiceRunning bool } -func NewDatasetFeature(mongoFeature *featuretest.MongoFeature, zebedeeURL string) (*DatasetFeature, error) { +func NewDatasetFeature(mongoFeature *componenttest.MongoFeature, zebedeeURL string) (*DatasetFeature, error) { f := &DatasetFeature{ HTTPServer: &http.Server{}, diff --git a/go.mod b/go.mod index 7384ecf3..e9b2979c 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,12 @@ module github.com/ONSdigital/dp-dataset-api go 1.15 +replace github.com/coreos/etcd => github.com/coreos/etcd v3.3.24+incompatible + require ( - github.com/ONSdigital/dp-api-clients-go v1.32.5 + github.com/ONSdigital/dp-api-clients-go v1.33.6 github.com/ONSdigital/dp-authorisation v0.1.0 + github.com/ONSdigital/dp-component-test v0.2.0 github.com/ONSdigital/dp-graph/v2 v2.7.1 github.com/ONSdigital/dp-healthcheck v1.0.5 github.com/ONSdigital/dp-kafka/v2 v2.1.2 @@ -12,23 +15,25 @@ require ( github.com/ONSdigital/dp-net v1.0.11 github.com/ONSdigital/go-ns v0.0.0-20200902154605-290c8b5ba5eb github.com/ONSdigital/log.go v1.0.1 - github.com/armakuni/dp-go-featuretest v0.0.0-20210225121618-47aebd5e92b6 github.com/benweissmann/memongo v0.1.1 - github.com/cucumber/godog v0.10.0 + github.com/cucumber/godog v0.11.0 + github.com/fatih/color v1.10.0 // indirect github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 github.com/gofrs/uuid v4.0.0+incompatible // indirect - github.com/golang/snappy v0.0.2 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/gorilla/mux v1.8.0 - github.com/gorilla/websocket v1.4.2 // indirect + github.com/hashicorp/go-memdb v1.3.1 // indirect + github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 // indirect github.com/justinas/alice v1.2.0 github.com/kelseyhightower/envconfig v1.4.0 - github.com/klauspost/compress v1.11.7 // indirect - github.com/mattn/go-isatty v0.0.12 // indirect github.com/pkg/errors v0.9.1 - github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/satori/go.uuid v1.2.0 github.com/smartystreets/goconvey v1.6.4 github.com/stretchr/testify v1.6.1 go.mongodb.org/mongo-driver v1.4.6 + golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect + golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect + golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43 // indirect gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183 // indirect ) diff --git a/go.sum b/go.sum index 52e7b7b6..7196db71 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,27 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ONSdigital/dp-api-clients-go v1.1.0/go.mod h1:9lqor0I7caCnRWr04gU/r7x5dqxgoODob8L48q+cE4E= github.com/ONSdigital/dp-api-clients-go v1.9.0/go.mod h1:SM0b/NXDWndJ9EulmAGdfDY4DxPxK+pNsP8eZlIWiqM= github.com/ONSdigital/dp-api-clients-go v1.28.0/go.mod h1:iyJy6uRL4B6OYOJA0XMr5UHt6+Q8XmN9uwmURO+9Oj4= -github.com/ONSdigital/dp-api-clients-go v1.32.5 h1:/Fg2KBjxdaLkgnXBqnxMUg+Y/vWhyk2WsOsVphuCQzU= -github.com/ONSdigital/dp-api-clients-go v1.32.5/go.mod h1:0pUK3MN1v7DTjq0JSAD+DqbsZ8AVTodrXSXgJecg9Pw= +github.com/ONSdigital/dp-api-clients-go v1.33.6 h1:RsVZ7a5Z8tpUpe6ARA2+XmdbG8BAYCJXBWkX6fDBnyE= +github.com/ONSdigital/dp-api-clients-go v1.33.6/go.mod h1:0pUK3MN1v7DTjq0JSAD+DqbsZ8AVTodrXSXgJecg9Pw= github.com/ONSdigital/dp-authorisation v0.1.0 h1:HzYwJdvk7ZAeB56KMAH6MP5+5uZuuJnEyGq6CViDoCg= github.com/ONSdigital/dp-authorisation v0.1.0/go.mod h1:rT81tcvWto5/cUWUFd0Q6gTqBoRfQmD6Qp0sq7FyiMg= +github.com/ONSdigital/dp-component-test v0.2.0 h1:lg1oc9k/xS7QgYhYi5cKBKmgqqDkHBae9xi05RzzaZQ= +github.com/ONSdigital/dp-component-test v0.2.0/go.mod h1:Y2TANFmqH9mk8ZERD22jWncSJyTxU0WmfnMp9EWPJb4= github.com/ONSdigital/dp-frontend-models v1.1.0/go.mod h1:TT96P7Mi69N3Tc/jFNdbjiwG4GAaMjP26HLotFQ6BPw= github.com/ONSdigital/dp-graph/v2 v2.7.1 h1:ImT0XxDiAcyXlfj+AnBQ4tFnhfZ/svmSiTEgvA0iNic= github.com/ONSdigital/dp-graph/v2 v2.7.1/go.mod h1:rblOFvTT+lMas8Y2DvoQdDL/6/EyfLLjnvL1rJnpPgg= @@ -48,30 +64,50 @@ github.com/ONSdigital/log.go v1.0.1-0.20200805084515-ee61165ea36a/go.mod h1:dDnQ github.com/ONSdigital/log.go v1.0.1-0.20200805145532-1f25087a0744/go.mod h1:y4E9MYC+cV9VfjRD0UBGj8PA7H3wABqQi87/ejrDhYc= github.com/ONSdigital/log.go v1.0.1 h1:SZ5wRZAwlt2jQUZ9AUzBB/PL+iG15KapfQpJUdA18/4= github.com/ONSdigital/log.go v1.0.1/go.mod h1:dIwSXuvFB5EsZG5x44JhsXZKMd80zlb0DZxmiAtpL4M= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Shopify/sarama v1.27.0 h1:tqo2zmyzPf1+gwTTwhI6W+EXDw4PVSczynpHKFtVAmo= github.com/Shopify/sarama v1.27.0/go.mod h1:aCdj6ymI8uyPEux1JJ9gcaDT6cinjGhNCAhs54taSUo= github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249 h1:fMi9ZZ/it4orHj3xWrM6cLkVFcCbkXQALFUiNtHtCPs= github.com/acobaugh/osrelease v0.0.0-20181218015638-a93a0a55a249/go.mod h1:iU1PxQMQwoHZZWmMKrMkrNlY+3+p9vxIjpZOVyxWa0g= -github.com/armakuni/dp-go-featuretest v0.0.0-20210225121618-47aebd5e92b6 h1:vbXjUtv6nagDev06BNtx5RWLRDPU1bUle3QQfJ2en08= -github.com/armakuni/dp-go-featuretest v0.0.0-20210225121618-47aebd5e92b6/go.mod h1:8PwfFu2YQNB3/if9L2VrrXyfEYf2NKOB6JInDYJKM/s= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/aslakhellesoy/gox v1.0.100/go.mod h1:AJl542QsKKG96COVsv0N74HHzVQgDIQPceVUh1aeU2M= github.com/aws/aws-sdk-go v1.34.28 h1:sscPpn/Ns3i0F4HPEWAVcwdIRaZZCuL7llJ2/60yPIk= github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/benweissmann/memongo v0.1.1 h1:L8pux/nWAmb6Zp+83vzqeW4+8GzzRIUBUhHMQizhX/M= github.com/benweissmann/memongo v0.1.1/go.mod h1:qMwr8bSVXCD9pUHgkcM3Nc8PZlzZy8UxNWteDDW/rw0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.24+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cucumber/gherkin-go/v11 v11.0.0 h1:cwVwN1Qn2VRSfHZNLEh5x00tPBmZcjATBWDpxsR5Xug= github.com/cucumber/gherkin-go/v11 v11.0.0/go.mod h1:CX33k2XU2qog4e+TFjOValoq6mIUq0DmVccZs238R9w= github.com/cucumber/godog v0.10.0 h1:W01u1+o8bRpgqJRLrclN3iAanU1jAao+TwOMoSV9g1Y= github.com/cucumber/godog v0.10.0/go.mod h1:0Q+MOUg8Z9AhzLV+nNMbThQ2x1b17yYwGyahApTLjJA= +github.com/cucumber/godog v0.11.0 h1:xgaWyJuAD6A+aW4TfVGNDBhuMyKW0jjl0cvY3KNxEak= +github.com/cucumber/godog v0.11.0/go.mod h1:GyxCIrsg1sgEgpL2GD/rMr3fIoNHpgkjm9nANw/89XY= github.com/cucumber/messages-go/v10 v10.0.1/go.mod h1:kA5T38CBlBbYLU12TIrJ4fk4wSkVVOgyh7Enyy8WnSg= github.com/cucumber/messages-go/v10 v10.0.3 h1:m/9SD/K/A15WP7i1aemIv7cwvUw+viS51Ui5HBw1cdE= github.com/cucumber/messages-go/v10 v10.0.3/go.mod h1:9jMZ2Y8ZxjLY6TG2+x344nt5rXstVVDYSdS5ySfI1WY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/eapache/go-resiliency v1.2.0 h1:v7g92e/KSN71Rq7vSThKaWIq68fL4YHvWyiUKorFR1Q= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= @@ -84,14 +120,22 @@ github.com/facebookgo/freeport v0.0.0-20150612182905-d4adf43b75b9/go.mod h1:uPmA github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0 h1:8xPHl4/q1VyqGIPif1F+1V3Y3lSmrq01EabUW3CoW5s= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.10.0 h1:s36xzo75JdqLaaWoiEHk767eHiwo0598uUxyfiPkDsg= +github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.10.0 h1:Gfh+GAJZOAoKZsIZeZbdn2JF10kN1XHNvjsvQK8gVkE= github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8 h1:DujepqpGd1hyOd7aW59XpK7Qymp8iy83xq74fLr21is= github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= github.com/go-avro/avro v0.0.0-20171219232920-444163702c11 h1:yswqe8UdKNWn4kjh1YTaAbvOSPeg95xhW7h4qeICL5E= github.com/go-avro/avro v0.0.0-20171219232920-444163702c11/go.mod h1:kxj6THYP0dmFPk4Z+bijIAhJoGgeBfyOKXMduhvdJPA= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= @@ -121,20 +165,40 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.2 h1:aeE13tS0IiQgFjYdoL8qN3K1N2bXXtI6Vi51/y7BpMw= -github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -146,19 +210,48 @@ github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= +github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.2.0 h1:l6UW37iCXwZkZoAbEYnptSHVE/cQ5bOTPYG5W3vf9+8= github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.0 h1:8exGP7ego3OmkfksihtSouGMZ+hQrhxx+FVELeXpVPE= +github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-memdb v1.2.1 h1:wI9btDjYUOJJHTCnRlAG/TkRyD/ij7meJMrLK9X31Cc= github.com/hashicorp/go-memdb v1.2.1/go.mod h1:OSvLJ662Jim8hMM+gWGyhktyWk2xPCnWMc7DWIqtkGA= +github.com/hashicorp/go-memdb v1.3.0 h1:xdXq34gBOMEloa9rlGStLxmfX/dyIK8htOv36dQUwHU= +github.com/hashicorp/go-memdb v1.3.0/go.mod h1:Mluclgwib3R93Hk5fxEfiRhB+6Dar64wWh71LpNSe3g= +github.com/hashicorp/go-memdb v1.3.1 h1:EIj6L28rTL41BDHBvwq1VJXzcmY2R2JBrxpWxF7Etyk= +github.com/hashicorp/go-memdb v1.3.1/go.mod h1:Mluclgwib3R93Hk5fxEfiRhB+6Dar64wWh71LpNSe3g= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= +github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e h1:0aewS5NTyxftZHSnFaJmWE5oCCrj4DyEXkAiMa1iZJM= github.com/hokaccha/go-prettyjson v0.0.0-20190818114111-108c894c2c0e/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= +github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519 h1:nqAlWFEdqI0ClbTDrhDvE/8LeQ4pftrqKUX9w5k0j3s= +github.com/hokaccha/go-prettyjson v0.0.0-20210113012101-fb4e108d2519/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= @@ -167,47 +260,75 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/justinas/alice v1.2.0 h1:+MHSA/vccVCF4Uq37S42jwlkvI2Xzl7zTPCN5BnZNVo= github.com/justinas/alice v1.2.0/go.mod h1:fN5HRH/reO/zrUflLfTN43t3vXvKzvZIENsNEe7i7qA= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.10.10 h1:a/y8CglcM7gLGYmlbP/stPE5sR3hbhFRUjCBfd/0B3I= github.com/klauspost/compress v1.10.10/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= -github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/maxcnunes/httpfake v1.2.1 h1:NRM5PWZxNcW/uKFSpIKeRMabQzqClNuosfdI6iO+R3I= github.com/maxcnunes/httpfake v1.2.1/go.mod h1:rWVxb0bLKtOUM/5hN3UO1VEdEitz1hfcTXs7UyiK6r0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg= github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mongo-go/testdb v0.0.0-20190724200850-a72a12eee610/go.mod h1:xyZcxcSxyRLfj4CDZzyycsGRcwfpY07ncmD2keFr548= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/gnatsd v1.4.1/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pierrec/lz4 v2.5.2+incompatible h1:WCjObylUIOlKy/+7Abdn34TLIkXiA4UWUMhxq9m9ZXI= github.com/pierrec/lz4 v2.5.2+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -217,14 +338,29 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM= -github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -233,12 +369,22 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M= github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/square/mongo-lock v0.0.0-20191001051310-282c90e422d0/go.mod h1:wR5++/O5fpa0UtI+9T8gKIi5jjl10va/EIEMRySqic4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -247,67 +393,191 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/unrolled/render v1.0.2/go.mod h1:gN9T0NhL4Bfbwu8ann7Ry/TGHYfosul+J0obPf6NBdM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM= go.mongodb.org/mongo-driver v1.4.6 h1:rh7GdYmDrb8AQSkF8yteAus8qYOgOASWDOv1BWqBXkU= go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200528225125-3c3fba18258b/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b h1:uwuIcX0g4Yl1NC5XAz37xsr2lTtcqevgzYNVt49waME= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d h1:1aflnvSoWWLI2k/dMUAl5lvU1YO4Mb4hz0gh+1rjcxU= +golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43 h1:SgQ6LNaYJU0JIuEHv9+s6EbhSCwYeAf5Yvj6lpYlqAE= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183 h1:PGIdqvwfpMUyUP+QAlAnKTSWQ671SmYjoou2/5j7HXk= gopkg.in/avro.v0 v0.0.0-20171217001914-a730b5802183/go.mod h1:FvqrFXt+jCsyQibeRv4xxEJBL5iG2DDW5aeJwzDiq4A= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -316,7 +586,10 @@ gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b h1:QRR6H1YWRnHb4Y/HeNFCTJLFVxaq6wH4YuVdsUOr75U= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= @@ -327,9 +600,20 @@ gopkg.in/jcmturner/gokrb5.v7 v7.5.0 h1:a9tsXlIDD9SKxotJMK3niV7rPZAJeX2aD/0yg3qlI gopkg.in/jcmturner/gokrb5.v7 v7.5.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2 h1:VEmvx0P+GVTgkNu2EdTN988YCZPcD3lo9AoczZpucwc= gopkg.in/yaml.v3 v3.0.0-20200601152816-913338de1bd2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/main_test.go b/main_test.go index b253cc0d..c5d9c7ea 100644 --- a/main_test.go +++ b/main_test.go @@ -5,8 +5,8 @@ import ( "os" "testing" + componenttest "github.com/ONSdigital/dp-component-test" steps_test "github.com/ONSdigital/dp-dataset-api/features/steps" - featuretest "github.com/armakuni/dp-go-featuretest" "github.com/cucumber/godog" "github.com/cucumber/godog/colors" ) @@ -18,18 +18,18 @@ const DatabaseName = "testing" var componentFlag = flag.Bool("component", false, "perform component tests") -type FeatureTest struct { - MongoFeature *featuretest.MongoFeature +type ComponentTest struct { + MongoFeature *componenttest.MongoFeature } -func (f *FeatureTest) InitializeScenario(ctx *godog.ScenarioContext) { - authorizationFeature := featuretest.NewAuthorizationFeature() +func (f *ComponentTest) InitializeScenario(ctx *godog.ScenarioContext) { + authorizationFeature := componenttest.NewAuthorizationFeature() datasetFeature, err := steps_test.NewDatasetFeature(f.MongoFeature, authorizationFeature.FakeAuthService.ResolveURL("")) if err != nil { panic(err) } - apiFeature := featuretest.NewAPIFeature(datasetFeature.InitialiseService) + apiFeature := componenttest.NewAPIFeature(datasetFeature.InitialiseService) ctx.BeforeScenario(func(*godog.Scenario) { apiFeature.Reset() @@ -48,9 +48,9 @@ func (f *FeatureTest) InitializeScenario(ctx *godog.ScenarioContext) { authorizationFeature.RegisterSteps(ctx) } -func (f *FeatureTest) InitializeTestSuite(ctx *godog.TestSuiteContext) { +func (f *ComponentTest) InitializeTestSuite(ctx *godog.TestSuiteContext) { ctx.BeforeSuite(func() { - f.MongoFeature = featuretest.NewMongoFeature(featuretest.MongoOptions{Port: MongoPort, MongoVersion: MongoVersion, DatabaseName: DatabaseName}) + f.MongoFeature = componenttest.NewMongoFeature(componenttest.MongoOptions{MongoVersion: MongoVersion, DatabaseName: DatabaseName}) }) ctx.AfterSuite(func() { f.MongoFeature.Close() @@ -67,7 +67,7 @@ func TestMain(t *testing.T) { Paths: flag.Args(), } - f := &FeatureTest{} + f := &ComponentTest{} status = godog.TestSuite{ Name: "feature_tests", diff --git a/models/dataset.go b/models/dataset.go index 818f2118..20c2efc4 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -452,26 +452,60 @@ func (ed *EditionUpdate) PublishLinks(ctx context.Context, host string, versionL return nil } +// CleanDataset trims URI and any hrefs contained in the database +func CleanDataset(dataset *Dataset) error { + if dataset == nil { + return errors.New("clean dataset called without a valid dataset") + } + dataset.URI = strings.TrimSpace(dataset.URI) + + for i := range dataset.Publications { + dataset.Publications[i].HRef = strings.TrimSpace(dataset.Publications[i].HRef) + } + + for i := range dataset.Methodologies { + dataset.Methodologies[i].HRef = strings.TrimSpace(dataset.Methodologies[i].HRef) + } + + for i := range dataset.RelatedDatasets { + dataset.RelatedDatasets[i].HRef = strings.TrimSpace(dataset.RelatedDatasets[i].HRef) + } + return nil +} + // ValidateDataset checks the dataset has invalid fields -func ValidateDataset(ctx context.Context, dataset *Dataset) error { +func ValidateDataset(dataset *Dataset) error { + var invalidFields []string if dataset.URI != "" { - dataset.URI = strings.TrimSpace(dataset.URI) - datasetURI, err := url.Parse(dataset.URI) + _, err := url.Parse(dataset.URI) if err != nil { invalidFields = append(invalidFields, "URI") - log.Event(ctx, "error parsing URI", log.ERROR, log.Error(err)) - } else { - if !strings.Contains(datasetURI.Path, "/datasets") || !strings.Contains(datasetURI.Path, dataset.ID) { - fmt.Println("hello") - invalidFields = append(invalidFields, "URI") - } } } + + invalidFields = append(invalidFields, validateGeneralDetails(dataset.Publications, "Publications")...) + + invalidFields = append(invalidFields, validateGeneralDetails(dataset.RelatedDatasets, "RelatedDatasets")...) + + invalidFields = append(invalidFields, validateGeneralDetails(dataset.Methodologies, "Methodologies")...) + if invalidFields != nil { return fmt.Errorf("invalid fields: %v", invalidFields) } + return nil + +} + +func validateGeneralDetails(generalDetails []GeneralDetails, identifier string) (invalidFields []string) { + for i, gd := range generalDetails { + _, err := url.Parse(gd.HRef) + if err != nil { + invalidFields = append(invalidFields, fmt.Sprintf("%s[%d].HRef", identifier, i)) + } + } + return } // ValidateDatasetType checks the dataset.type field has valid type diff --git a/models/dataset_test.go b/models/dataset_test.go index 5c253220..6eefce56 100644 --- a/models/dataset_test.go +++ b/models/dataset_test.go @@ -13,11 +13,33 @@ import ( . "github.com/smartystreets/goconvey/convey" ) +const ( + validURI = "http://localhost:22000/datasets/123" + validHref = "http://localhost:22000//datasets/href" + invalidHref = ":invalid" +) + func createDataset() Dataset { return Dataset{ ID: "123", - URI: "http://localhost:22000/datasets/123", + URI: validURI, + Publications: []GeneralDetails{{ + Description: "some publication description", + HRef: validHref, + Title: "some publication title", + }}, + Methodologies: []GeneralDetails{{ + Description: "some methodologies description", + HRef: validHref, + Title: "some publication title", + }}, + RelatedDatasets: []GeneralDetails{{ + Description: "some related datasets description", + HRef: validHref, + Title: "some publication title", + }}, } + } var testContext = context.Background() @@ -199,59 +221,166 @@ func TestCreateVersion(t *testing.T) { }) } +func TestCleanDataset(t *testing.T) { + t.Parallel() + + Convey("A clean dataset stays unmodified", t, func() { + Convey("When a clean dataset is cleaned, the URI and hrefs stay the same", func() { + dataset := createDataset() + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.URI, ShouldEqual, validURI) + So(dataset.Publications, ShouldHaveLength, 1) + So(dataset.Publications[0].HRef, ShouldEqual, validHref) + }) + }) + + Convey("A dirty dataset is cleaned", t, func() { + Convey("When a dataset URI has leading space it is trimmed", func() { + dataset := createDataset() + dataset.URI = " " + validURI + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.URI, ShouldEqual, validURI) + }) + + Convey("When a dataset URI has trailing space it is trimmed", func() { + dataset := createDataset() + dataset.URI = validURI + " " + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.URI, ShouldEqual, validURI) + }) + + Convey("When a Publications HRef has whitespace it is trimmed", func() { + dataset := createDataset() + dataset.Publications[0].HRef = " " + validHref + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.Publications, ShouldHaveLength, 1) + So(dataset.Publications[0].HRef, ShouldEqual, validHref) + }) + + Convey("When two Publications HRef's have whitespace they are trimmed", func() { + dataset := createDataset() + dataset.Publications[0].HRef = " " + validHref + dataset.Publications = append(dataset.Publications, GeneralDetails{HRef: validHref + " "}) + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.Publications, ShouldHaveLength, 2) + So(dataset.Publications[0].HRef, ShouldEqual, validHref) + So(dataset.Publications[1].HRef, ShouldEqual, validHref) + }) + + Convey("When a Methodologies HRef has whitespace it is trimmed", func() { + dataset := createDataset() + dataset.Methodologies[0].HRef = " " + validHref + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.Methodologies, ShouldHaveLength, 1) + So(dataset.Methodologies[0].HRef, ShouldEqual, validHref) + }) + + Convey("When two Methodologies HRef's have whitespace they are trimmed", func() { + dataset := createDataset() + dataset.Methodologies[0].HRef = " " + validHref + dataset.Methodologies = append(dataset.Methodologies, GeneralDetails{HRef: validHref + " "}) + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.Methodologies, ShouldHaveLength, 2) + So(dataset.Methodologies[0].HRef, ShouldEqual, validHref) + So(dataset.Methodologies[1].HRef, ShouldEqual, validHref) + }) + + Convey("When a RelatedDatasets HRef has whitespace it is trimmed", func() { + dataset := createDataset() + dataset.RelatedDatasets[0].HRef = " " + validHref + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.RelatedDatasets, ShouldHaveLength, 1) + So(dataset.RelatedDatasets[0].HRef, ShouldEqual, validHref) + }) + + Convey("When two RelatedDatasets HRef's have whitespace they are trimmed", func() { + dataset := createDataset() + dataset.RelatedDatasets[0].HRef = " " + validHref + dataset.RelatedDatasets = append(dataset.RelatedDatasets, GeneralDetails{HRef: validHref + " "}) + cleanErr := CleanDataset(&dataset) + So(cleanErr, ShouldBeNil) + So(dataset.RelatedDatasets, ShouldHaveLength, 2) + So(dataset.RelatedDatasets[0].HRef, ShouldEqual, validHref) + So(dataset.RelatedDatasets[1].HRef, ShouldEqual, validHref) + }) + + }) + + Convey("A nil dataset returns an error when cleaned", t, func() { + Convey("A nil dataset returns an error", func() { + cleanErr := CleanDataset(nil) + So(cleanErr, ShouldNotBeNil) + So(cleanErr.Error(), ShouldResemble, errors.New("clean dataset called without a valid dataset").Error()) + }) + }) +} + func TestValidateDataset(t *testing.T) { t.Parallel() + Convey("Successful validation (true) returned", t, func() { + + Convey("when dataset.URI contains its path in appropriate url format ", func() { + dataset := createDataset() + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldBeNil) + }) + }) + Convey("Unsuccessful validation (false) returned", t, func() { Convey("when dataset.URI is unable to be parsed into url format", func() { dataset := createDataset() dataset.URI = ":foo" fmt.Println(dataset.URI) - validationErr := ValidateDataset(testContext, &dataset) + validationErr := ValidateDataset(&dataset) So(validationErr, ShouldNotBeNil) So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) }) - Convey("when dataset.URI does not contain 'datasets' path", func() { + Convey("when Publications href is unable to be parsed into url format", func() { dataset := createDataset() - dataset.URI = "/123" - validationErr := ValidateDataset(testContext, &dataset) + dataset.Publications[0].HRef = invalidHref + validationErr := ValidateDataset(&dataset) So(validationErr, ShouldNotBeNil) - So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [Publications[0].HRef]").Error()) }) - Convey("when dataset.URI does not contain 'id' path", func() { + Convey("when Methodologies href is unable to be parsed into url format", func() { dataset := createDataset() - dataset.URI = "http://localhost:22000/datasets" - validationErr := ValidateDataset(testContext, &dataset) + dataset.Methodologies[0].HRef = invalidHref + validationErr := ValidateDataset(&dataset) So(validationErr, ShouldNotBeNil) - So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI]").Error()) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [Methodologies[0].HRef]").Error()) }) - }) - - Convey("Successful validation (true) returned", t, func() { - - Convey("when dataset.URI contains its path in appropriate url format ", func() { + Convey("when RelatedDatasets href is unable to be parsed into url format", func() { dataset := createDataset() - dataset.ID = "123" - dataset.URI = "http://localhost:22000/datasets/123/breadcrumbs" - validationErr := ValidateDataset(testContext, &dataset) - So(validationErr, ShouldBeNil) + dataset.RelatedDatasets[0].HRef = invalidHref + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldNotBeNil) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [RelatedDatasets[0].HRef]").Error()) }) - }) - - Convey("Successful validation (true) returned", t, func() { - Convey("when dataset.URI contains whitespace it should not return an error ", func() { + Convey("when all href and URI fields are unable to be parsed into url format", func() { dataset := createDataset() - dataset.ID = "123" - dataset.URI = " http://localhost:22000/datasets/123/breadcrumbs " - validationErr := ValidateDataset(testContext, &dataset) - So(validationErr, ShouldBeNil) - So(dataset.URI, ShouldEqual, "http://localhost:22000/datasets/123/breadcrumbs") + dataset.URI = invalidHref + dataset.Publications[0].HRef = invalidHref + dataset.Methodologies[0].HRef = invalidHref + dataset.RelatedDatasets[0].HRef = invalidHref + validationErr := ValidateDataset(&dataset) + So(validationErr, ShouldNotBeNil) + So(validationErr.Error(), ShouldResemble, errors.New("invalid fields: [URI Publications[0].HRef RelatedDatasets[0].HRef Methodologies[0].HRef]").Error()) }) + }) } diff --git a/mongo/dataset_store.go b/mongo/dataset_store.go index bb906606..cb184ffb 100644 --- a/mongo/dataset_store.go +++ b/mongo/dataset_store.go @@ -567,11 +567,6 @@ func createDatasetUpdateQuery(ctx context.Context, id string, dataset *models.Da updates["next.nomis_reference_url"] = dataset.NomisReferenceURL } - if err := models.ValidateDataset(ctx, dataset); err != nil { - log.Event(ctx, "failed validation check to create dataset", log.ERROR, log.Error(err)) - return nil - } - log.Event(ctx, "built update query for dataset resource", log.INFO, log.Data{"dataset_id": id, "dataset": dataset, "updates": updates}) return updates diff --git a/service/service.go b/service/service.go index f40ac50a..41f344b9 100644 --- a/service/service.go +++ b/service/service.go @@ -95,10 +95,9 @@ func (svc *Service) Run(ctx context.Context, buildTime, gitCommit, version strin } // Get graphDB connection for observation store - if !svc.config.EnableObservationEndpoint && !svc.config.EnablePrivateEndpoints { + if !svc.config.EnablePrivateEndpoints { log.Event(ctx, "skipping graph DB client creation, because it is not required by the enabled endpoints", log.INFO, log.Data{ - "EnableObservationEndpoint": svc.config.EnableObservationEndpoint, - "EnablePrivateEndpoints": svc.config.EnablePrivateEndpoints, + "EnablePrivateEndpoints": svc.config.EnablePrivateEndpoints, }) } else { svc.graphDB, svc.graphDBErrorConsumer, err = svc.serviceList.GetGraphDB(ctx) @@ -318,8 +317,8 @@ func (svc *Service) registerCheckers(ctx context.Context) (err error) { log.Event(ctx, "error adding check for mongo db", log.ERROR, log.Error(err)) } - if svc.config.EnablePrivateEndpoints || svc.config.EnableObservationEndpoint { - log.Event(ctx, "adding graph db health check as the private or observation endpoints are enabled", log.INFO) + if svc.config.EnablePrivateEndpoints { + log.Event(ctx, "adding graph db health check as the private endpoints are enabled", log.INFO) if err = svc.healthCheck.AddCheck("Graph DB", svc.graphDB.Checker); err != nil { hasErrors = true log.Event(ctx, "error adding check for graph db", log.ERROR, log.Error(err)) diff --git a/service/service_test.go b/service/service_test.go index 4c86445a..0593b59c 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -264,8 +264,7 @@ func TestRun(t *testing.T) { }) }) - Convey("Given that all dependencies are successfully initialised, observation and private endpoints are disabled", func() { - cfg.EnableObservationEndpoint = false + Convey("Given that all dependencies are successfully initialised, private endpoints are disabled", func() { cfg.EnablePrivateEndpoints = false initMock := &serviceMock.InitialiserMock{ DoGetMongoDBFunc: funcDoGetMongoDBOk, diff --git a/store/datastore.go b/store/datastore.go index cee903af..4e408188 100644 --- a/store/datastore.go +++ b/store/datastore.go @@ -4,7 +4,6 @@ import ( "context" "github.com/ONSdigital/dp-dataset-api/models" - "github.com/ONSdigital/dp-graph/v2/observation" "github.com/ONSdigital/dp-healthcheck/healthcheck" "github.com/globalsign/mgo/bson" ) @@ -67,7 +66,6 @@ type MongoDB interface { type dataGraphDB interface { AddVersionDetailsToInstance(ctx context.Context, instanceID string, datasetID string, edition string, version int) error SetInstanceIsPublished(ctx context.Context, instanceID string) error - StreamCSVRows(ctx context.Context, instanceID, filterID string, filters *observation.DimensionFilters, limit *int) (observation.StreamRowReader, error) } // GraphDB represents all the required methods from graph DB