Merge branch 'beta/1.1.0' into cmd-master

README.md

@@ -1,11 +1,38 @@
 dp-dataset-api
 ==================
-A ONS API used to navigate datasets which are published.
+A ONS API used to navigate datasets, editions and versions - which are published.
+
+### Installation
 
 #### Database
-* Run ```brew install mongo```
-* Run ```brew services start mongodb```
-* Run ```./scripts/InitDatabase.sh```
+* Run `brew install mongo`
+* Run `brew services start mongodb`
+* Run `./scripts/InitDatabase.sh`
+
+### State changes
+
+Normal sequential order of states:
+
+1. `created` (only on *instance*)
+2. `submitted` (only on *instance*)
+3. `completed` (only on *instance*)
+4. `edition-confirmed` (only on *instance* - this will create an *edition* and *version*,
+    in other words the *instance* will now be accessible by `version` endpoints).
+    Also the dataset `next` sub-document will also get updated here and so will the *edition*
+    (authorised users will see a different latest *version* link versus unauthorised users)
+5. `associated` (only on *version*) - dataset `next` sub-document will be updated again and so will the *edition*
+6. `published` (only on *version*) - both *edition* and *dataset* are updated - must not be changed
+
+There is the possibility to **rollback** from `associate`  to `edition-confirmed`
+where a PST user has attached the _version_ to the wrong collection and so not only does
+the `collection_id` need to be updated with the new one (or removed all together)
+but the state will need to revert back to `edition-confirmed`.
+
+Lastly, **skipping a state**: it is possibly to jump from `edition-confirmed` to `published`
+as long as all the mandatory fields are there. There also might be a scenario whereby
+the state can change from `created` to `completed`, missing out the step to `submitted`
+due to race conditions, this is not expected to happen,
+the path to get to `completed` is longer than the `submitted` one.
 
 ### Healthcheck
 
@@ -24,10 +51,12 @@ one of:
 | MONGODB_DATABASE           | datasets                             | The MongoDB dataset database
 | MONGODB_COLLECTION         | datasets                             | MongoDB collection
 | SECRET_KEY                 | FD0108EA-825D-411C-9B1D-41EF7727F465 | A secret key used authentication
-| CODE_LIST_API_URL          | http://localhost:22400               | The host name for the Dataset API
-| DATASET_API_URL            | http://localhost:22000               | The host name for the CodeList API
+| CODE_LIST_API_URL          | http://localhost:22400               | The host name for the CodeList API
+| DATASET_API_URL            | http://localhost:22000               | The host name for the Dataset API
+| GRACEFUL_SHUTDOWN_TIMEOUT  | 5s                                   | The graceful shutdown timeout in seconds
 | WEBSITE_URL                | http://localhost:20000               | The host name for the website
-| GRACEFUL_SHUTDOWN_TIMEOUT  | 5s                                   | The graceful shutdown timeout
+| KAFKA_ADDR                 | "localhost:9092"                     | The list of kafka hosts
+| GENERATE_DOWNLOADS_TOPIC   | "filter-job-submitted"               | The topic to send generate full dataset version downloads to
 | HEALTHCHECK_TIMEOUT        | 2s                                   | The timeout that the healthcheck allows for checked subsystems
 
 ### Contributing

api/api.go

@@ -14,10 +14,18 @@ import (
 	"github.com/gorilla/mux"
 )
 
-//go:generate moq -out test/api.go -pkg apitest . API
-
 var httpServer *server.Server
 
+//API provides an interface for the routes
+type API interface {
+	CreateDatasetAPI(string, *mux.Router, store.DataStore) *DatasetAPI
+}
+
+// DownloadsGenerator pre generates full file downloads for the specified dataset/edition/version
+type DownloadsGenerator interface {
+	Generate(datasetID, instanceID, edition, version string) error
+}
+
 // DatasetAPI manages importing filters against a dataset
 type DatasetAPI struct {
 	dataStore          store.DataStore
@@ -26,13 +34,14 @@ type DatasetAPI struct {
 	privateAuth        *auth.Authenticator
 	router             *mux.Router
 	urlBuilder         *url.Builder
+	downloadGenerator  DownloadsGenerator
 	healthCheckTimeout time.Duration
 }
 
 // CreateDatasetAPI manages all the routes configured to API
-func CreateDatasetAPI(host, bindAddr, secretKey string, dataStore store.DataStore, urlBuilder *url.Builder, errorChan chan error, healthCheckTimeout time.Duration) {
+func CreateDatasetAPI(host, bindAddr, secretKey string, dataStore store.DataStore, urlBuilder *url.Builder, errorChan chan error, downloadsGenerator DownloadsGenerator, healthCheckTimeout time.Duration) {
 	router := mux.NewRouter()
-	routes(host, secretKey, router, dataStore, urlBuilder, healthCheckTimeout)
+	routes(host, secretKey, router, dataStore, urlBuilder, downloadsGenerator, healthCheckTimeout)
 
 	httpServer = server.New(bindAddr, router)
 	// Disable this here to allow main to manage graceful shutdown of the entire app.
@@ -47,14 +56,15 @@ func CreateDatasetAPI(host, bindAddr, secretKey string, dataStore store.DataStor
 	}()
 }
 
-func routes(host, secretKey string, router *mux.Router, dataStore store.DataStore, urlBuilder *url.Builder, healthCheckTimeout time.Duration) *DatasetAPI {
+func routes(host, secretKey string, router *mux.Router, dataStore store.DataStore, urlBuilder *url.Builder, downloadGenerator DownloadsGenerator, healthCheckTimeout time.Duration) *DatasetAPI {
 	api := DatasetAPI{
 		privateAuth:        &auth.Authenticator{SecretKey: secretKey, HeaderName: "internal-token"},
 		dataStore:          dataStore,
 		host:               host,
 		internalToken:      secretKey,
 		router:             router,
 		urlBuilder:         urlBuilder,
+		downloadGenerator:  downloadGenerator,
 		healthCheckTimeout: healthCheckTimeout,
 	}
 
@@ -80,6 +90,7 @@ func routes(host, secretKey string, router *mux.Router, dataStore store.DataStor
 	api.router.HandleFunc("/instances/{id}", api.privateAuth.Check(instance.Update)).Methods("PUT")
 	api.router.HandleFunc("/instances/{id}/events", api.privateAuth.Check(instance.AddEvent)).Methods("POST")
 	api.router.HandleFunc("/instances/{id}/inserted_observations/{inserted_observations}", api.privateAuth.Check(instance.UpdateObservations)).Methods("PUT")
+	api.router.HandleFunc("/instances/{id}/import_tasks", api.privateAuth.Check(instance.UpdateImportTask)).Methods("PUT")
 
 	dimension := dimension.Store{Storer: api.dataStore.Backend}
 	api.router.HandleFunc("/instances/{id}/dimensions", dimension.GetNodes).Methods("GET")

api/dataset.go

@@ -6,13 +6,12 @@ import (
 	"net/http"
 	"time"
 
-	"gopkg.in/mgo.v2/bson"
-
-	"github.com/ONSdigital/dp-dataset-api/models"
-
 	errs "github.com/ONSdigital/dp-dataset-api/apierrors"
+	"github.com/ONSdigital/dp-dataset-api/models"
 	"github.com/ONSdigital/go-ns/log"
 	"github.com/gorilla/mux"
+	"github.com/pkg/errors"
+	"gopkg.in/mgo.v2/bson"
 )
 
 const (
@@ -462,12 +461,26 @@ func (api *DatasetAPI) putVersion(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	if versionDoc.State == models.AssociatedState {
+	if versionDoc.State == models.AssociatedState && currentVersion.State != models.AssociatedState {
 		if err := api.dataStore.Backend.UpdateDatasetWithAssociation(datasetID, versionDoc.State, versionDoc); err != nil {
 			log.ErrorC("failed to update dataset document after a version of a dataset has been associated with a collection", err, log.Data{"dataset_id": datasetID, "edition": edition, "version": version})
 			handleErrorType(versionDocType, err, w)
 			return
 		}
+
+		log.Info("generating full dataset version downloads", log.Data{"dataset_id": datasetID, "edition": edition, "version": version})
+
+		if err := api.downloadGenerator.Generate(datasetID, versionDoc.ID, edition, version); err != nil {
+			err = errors.Wrap(err, "error while attempting to generate full dataset version downloads")
+			log.Error(err, log.Data{
+				"dataset_id":  datasetID,
+				"instance_id": versionDoc.ID,
+				"edition":     edition,
+				"version":     version,
+			})
+			// TODO - TECH DEBT - need to add an error event for this.
+			handleErrorType(versionDocType, err, w)
+		}
 	}
 
 	setJSONContentType(w)
@@ -566,6 +579,22 @@ func createNewVersionDoc(currentVersion *models.Version, version *models.Version
 		}
 	}
 
+	if version.Downloads == nil {
+		version.Downloads = currentVersion.Downloads
+	} else {
+		if version.Downloads.XLS == nil {
+			if currentVersion.Downloads != nil && currentVersion.Downloads.XLS != nil {
+				version.Downloads.XLS = currentVersion.Downloads.XLS
+			}
+		}
+
+		if version.Downloads.CSV == nil {
+			if currentVersion.Downloads != nil && currentVersion.Downloads.CSV != nil {
+				version.Downloads.CSV = currentVersion.Downloads.CSV
+			}
+		}
+	}
+
 	return version
 }