Merge branch 'beta/1.3.0' into cmd-master

README.md

@@ -44,21 +44,22 @@ one of:
 
 ### Configuration
 
-| Environment variable       | Default                              | Description
-| -------------------------- | -------------------------------------| -----------
-| BIND_ADDR                  | :22000                               | The host and port to bind to
-| MONGODB_BIND_ADDR          | localhost:27017                      | The MongoDB bind address
-| MONGODB_DATABASE           | datasets                             | The MongoDB dataset database
-| MONGODB_COLLECTION         | datasets                             | MongoDB collection
-| SECRET_KEY                 | FD0108EA-825D-411C-9B1D-41EF7727F465 | A secret key used authentication
-| CODE_LIST_API_URL          | http://localhost:22400               | The host name for the CodeList API
-| DATASET_API_URL            | http://localhost:22000               | The host name for the Dataset API
-| GRACEFUL_SHUTDOWN_TIMEOUT  | 5s                                   | The graceful shutdown timeout in seconds
-| WEBSITE_URL                | http://localhost:20000               | The host name for the website
-| KAFKA_ADDR                 | "localhost:9092"                     | The list of kafka hosts
-| GENERATE_DOWNLOADS_TOPIC   | "filter-job-submitted"               | The topic to send generate full dataset version downloads to
-| HEALTHCHECK_TIMEOUT        | 2s                                   | The timeout that the healthcheck allows for checked subsystems
-| ENABLE_PRIVATE_ENDPOINTS   | false                                | Enable private endpoints for the API
+| Environment variable        | Default                                | Description
+| --------------------------- | ---------------------------------------| -----------
+| BIND_ADDR                   | :22000                                 | The host and port to bind to
+| MONGODB_BIND_ADDR           | localhost:27017                        | The MongoDB bind address
+| MONGODB_DATABASE            | datasets                               | The MongoDB dataset database
+| MONGODB_COLLECTION          | datasets                               | MongoDB collection
+| SECRET_KEY                  | FD0108EA-825D-411C-9B1D-41EF7727F465   | A secret key used authentication
+| CODE_LIST_API_URL           | http://localhost:22400                 | The host name for the CodeList API
+| DATASET_API_URL             | http://localhost:22000                 | The host name for the Dataset API
+| GRACEFUL_SHUTDOWN_TIMEOUT   | 5s                                     | The graceful shutdown timeout in seconds
+| WEBSITE_URL                 | http://localhost:20000                 | The host name for the website
+| KAFKA_ADDR                  | "localhost:9092"                       | The list of kafka hosts
+| GENERATE_DOWNLOADS_TOPIC    | "filter-job-submitted"                 | The topic to send generate full dataset version downloads to
+| HEALTHCHECK_TIMEOUT         | 2s                                     | The timeout that the healthcheck allows for checked subsystems
+| ENABLE_PRIVATE_ENDPOINTS    | false                                  | Enable private endpoints for the API
+| DOWNLOAD_SERVICE_SECRET_KEY | "QB0108EZ-825D-412C-9B1D-41EF7747F462" | A key specific for the download service to access public/private links
 
 ### Contributing
 

api/api.go

@@ -32,6 +32,7 @@ type DatasetAPI struct {
 	dataStore            store.DataStore
 	host                 string
 	internalToken        string
+	downloadServiceToken string
 	EnablePrePublishView bool
 	privateAuth          *auth.Authenticator
 	router               *mux.Router
@@ -71,6 +72,7 @@ func routes(cfg config.Configuration, router *mux.Router, dataStore store.DataSt
 		dataStore:            dataStore,
 		host:                 cfg.DatasetAPIURL,
 		internalToken:        cfg.SecretKey,
+		downloadServiceToken: cfg.DownloadServiceSecretKey,
 		EnablePrePublishView: cfg.EnablePrivateEnpoints,
 		router:               router,
 		urlBuilder:           urlBuilder,

api/dataset.go

@@ -20,6 +20,7 @@ const (
 	datasetDocType         = "dataset"
 	editionDocType         = "edition"
 	versionDocType         = "version"
+	downloadServiceToken   = "X-Download-Service-Token"
 	dimensionDocType       = "dimension"
 	dimensionOptionDocType = "dimension-option"
 )
@@ -246,6 +247,21 @@ func (api *DatasetAPI) getVersions(w http.ResponseWriter, r *http.Request) {
 			hasInvalidState = true
 			log.ErrorC("unpublished version has an invalid state", err, log.Data{"state": item.State})
 		}
+
+		// Only the download service should not have access to the public/private download
+		// fields
+		if r.Header.Get(downloadServiceToken) != api.downloadServiceToken {
+			if item.Downloads != nil {
+				if item.Downloads.CSV != nil {
+					item.Downloads.CSV.Private = ""
+					item.Downloads.CSV.Public = ""
+				}
+				if item.Downloads.XLS != nil {
+					item.Downloads.XLS.Private = ""
+					item.Downloads.XLS.Public = ""
+				}
+			}
+		}
 	}
 
 	if hasInvalidState {
@@ -313,6 +329,21 @@ func (api *DatasetAPI) getVersion(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Only the download service should not have access to the public/private download
+	// fields
+	if r.Header.Get(downloadServiceToken) != api.downloadServiceToken {
+		if results.Downloads != nil {
+			if results.Downloads.CSV != nil {
+				results.Downloads.CSV.Private = ""
+				results.Downloads.CSV.Public = ""
+			}
+			if results.Downloads.XLS != nil {
+				results.Downloads.XLS.Private = ""
+				results.Downloads.XLS.Public = ""
+			}
+		}
+	}
+
 	bytes, err := json.Marshal(results)
 	if err != nil {
 		log.ErrorC("failed to marshal version resource into bytes", err, logData)

ci/build.yml

@@ -6,7 +6,7 @@ image_resource:
   type: docker-image
   source:
     repository: golang
-    tag: 1.9.2
+    tag: 1.10.0
 
 inputs:
   - name: dp-dataset-api

ci/unit.yml

@@ -6,7 +6,7 @@ image_resource:
   type: docker-image
   source:
     repository: golang
-    tag: 1.9.2
+    tag: 1.10.0
 
 inputs:
   - name: dp-dataset-api

config/config.go

@@ -9,17 +9,18 @@ import (
 
 // Configuration structure which hold information for configuring the import API
 type Configuration struct {
-	BindAddr                string        `envconfig:"BIND_ADDR"`
-	KafkaAddr               []string      `envconfig:"KAFKA_ADDR"                 json:"-"`
-	GenerateDownloadsTopic  string        `envconfig:"GENERATE_DOWNLOADS_TOPIC"`
-	CodeListAPIURL          string        `envconfig:"CODE_LIST_API_URL"`
-	DatasetAPIURL           string        `envconfig:"DATASET_API_URL"`
-	WebsiteURL              string        `envconfig:"WEBSITE_URL"`
-	SecretKey               string        `envconfig:"SECRET_KEY"                 json:"-"`
-	GracefulShutdownTimeout time.Duration `envconfig:"GRACEFUL_SHUTDOWN_TIMEOUT"`
-	HealthCheckTimeout      time.Duration `envconfig:"HEALTHCHECK_TIMEOUT"`
-	EnablePrivateEnpoints   bool          `envconfig:"ENABLE_PRIVATE_ENDPOINTS"`
-	MongoConfig             MongoConfig
+	BindAddr                 string        `envconfig:"BIND_ADDR"`
+	KafkaAddr                []string      `envconfig:"KAFKA_ADDR"                       json:"-"`
+	GenerateDownloadsTopic   string        `envconfig:"GENERATE_DOWNLOADS_TOPIC"`
+	CodeListAPIURL           string        `envconfig:"CODE_LIST_API_URL"`
+	DatasetAPIURL            string        `envconfig:"DATASET_API_URL"`
+	DownloadServiceSecretKey string        `envconfig:"DOWNLOAD_SERVICE_SECRET_KEY"      json:"-"`
+	WebsiteURL               string        `envconfig:"WEBSITE_URL"`
+	SecretKey                string        `envconfig:"SECRET_KEY"                       json:"-"`
+	GracefulShutdownTimeout  time.Duration `envconfig:"GRACEFUL_SHUTDOWN_TIMEOUT"`
+	HealthCheckTimeout       time.Duration `envconfig:"HEALTHCHECK_TIMEOUT"`
+	EnablePrivateEnpoints    bool          `envconfig:"ENABLE_PRIVATE_ENDPOINTS"`
+	MongoConfig              MongoConfig
 }
 
 // MongoConfig contains the config required to connect to MongoDB.
@@ -38,16 +39,17 @@ func Get() (*Configuration, error) {
 	}
 
 	cfg = &Configuration{
-		BindAddr:                ":22000",
-		KafkaAddr:               []string{"localhost:9092"},
-		GenerateDownloadsTopic:  "filter-job-submitted",
-		CodeListAPIURL:          "http://localhost:22400",
-		DatasetAPIURL:           "http://localhost:22000",
-		WebsiteURL:              "http://localhost:20000",
-		SecretKey:               "FD0108EA-825D-411C-9B1D-41EF7727F465",
-		GracefulShutdownTimeout: 5 * time.Second,
-		HealthCheckTimeout:      2 * time.Second,
-		EnablePrivateEnpoints:   false,
+		BindAddr:                 ":22000",
+		KafkaAddr:                []string{"localhost:9092"},
+		GenerateDownloadsTopic:   "filter-job-submitted",
+		CodeListAPIURL:           "http://localhost:22400",
+		DatasetAPIURL:            "http://localhost:22000",
+		WebsiteURL:               "http://localhost:20000",
+		SecretKey:                "FD0108EA-825D-411C-9B1D-41EF7727F465",
+		DownloadServiceSecretKey: "QB0108EZ-825D-412C-9B1D-41EF7747F462",
+		GracefulShutdownTimeout:  5 * time.Second,
+		HealthCheckTimeout:       2 * time.Second,
+		EnablePrivateEnpoints:    false,
 		MongoConfig: MongoConfig{
 			BindAddr:   "localhost:27017",
 			Collection: "datasets",

config/config_test.go

@@ -23,6 +23,7 @@ func TestSpec(t *testing.T) {
 				So(cfg.GenerateDownloadsTopic, ShouldEqual, "filter-job-submitted")
 				So(cfg.DatasetAPIURL, ShouldEqual, "http://localhost:22000")
 				So(cfg.CodeListAPIURL, ShouldEqual, "http://localhost:22400")
+				So(cfg.DownloadServiceSecretKey, ShouldEqual, "QB0108EZ-825D-412C-9B1D-41EF7747F462")
 				So(cfg.WebsiteURL, ShouldEqual, "http://localhost:20000")
 				So(cfg.SecretKey, ShouldEqual, "FD0108EA-825D-411C-9B1D-41EF7727F465")
 				So(cfg.GracefulShutdownTimeout, ShouldEqual, 5*time.Second)

models/dataset.go

@@ -143,6 +143,7 @@ type Version struct {
 	Temporal      *[]TemporalFrequency `bson:"temporal,omitempty"       json:"temporal,omitempty"`
 	LastUpdated   time.Time            `bson:"last_updated,omitempty"   json:"-"`
 	Version       int                  `bson:"version,omitempty"        json:"version,omitempty"`
+	UsageNotes    *[]UsageNote         `bson:"usage_notes,omitempty"     json:"usage_notes,omitempty"`
 }
 
 // Alert represents an object containing information on an alert
@@ -163,7 +164,9 @@ type DownloadObject struct {
 	URL string `bson:"url,omitempty"  json:"url,omitempty"`
 	// TODO size is in bytes and probably should be an int64 instead of a string this
 	// will have to change for several services (filter API, exporter services and web)
-	Size string `bson:"size,omitempty" json:"size,omitempty"`
+	Size    string `bson:"size,omitempty" json:"size,omitempty"`
+	Public  string `bson:"public,omitempty" json:"public,omitempty"`
+	Private string `bson:"private,omitempty" json:"private,omitempty"`
 }
 
 // LatestChange represents an object contining
@@ -181,6 +184,11 @@ type TemporalFrequency struct {
 	StartDate string `bson:"start_date,omitempty"  json:"start_date,omitempty"`
 }
 
+type UsageNote struct {
+	Title string `bson:"title,omitempty"    json:"title,omitempty"`
+	Note  string `bson:"note,omitempty"     json:"note,omitempty"`
+}
+
 // VersionLinks represents a list of specific links related to the version resource for an edition of a dataset
 type VersionLinks struct {
 	Dataset    *LinkObject `bson:"dataset,omitempty"     json:"dataset,omitempty"`

models/metadata.go

@@ -32,6 +32,7 @@ type Metadata struct {
 	Title             string               `json:"title,omitempty"`
 	UnitOfMeasure     string               `json:"unit_of_measure,omitempty"`
 	URI               string               `json:"uri,omitempty"`
+	UsageNotes        *[]UsageNote         `json:"usage_notes,omitempty"`
 }
 
 // MetadataLinks represents a link object to list of metadata) relevant to a version
@@ -69,6 +70,7 @@ func CreateMetaDataDoc(datasetDoc *Dataset, versionDoc *Version, urlBuilder *url
 		Title:             datasetDoc.Title,
 		UnitOfMeasure:     datasetDoc.UnitOfMeasure,
 		URI:               datasetDoc.URI,
+		UsageNotes:        versionDoc.UsageNotes,
 	}
 
 	// Add relevant metdata links from dataset document

mongo/dataset_store.go

@@ -55,7 +55,12 @@ func (m *Mongo) GetDatasets() ([]models.DatasetUpdate, error) {
 	defer s.Close()
 
 	iter := s.DB(m.Database).C("datasets").Find(nil).Iter()
-	defer iter.Close()
+	defer func() {
+		err := iter.Close()
+		if err != nil {
+			log.ErrorC("error closing iterator", err, log.Data{})
+		}
+	}()
 
 	results := []models.DatasetUpdate{}
 	if err := iter.All(&results); err != nil {
@@ -92,7 +97,12 @@ func (m *Mongo) GetEditions(id, state string) (*models.EditionResults, error) {
 	selector := buildEditionsQuery(id, state)
 
 	iter := s.DB(m.Database).C(editionsCollection).Find(selector).Iter()
-	defer iter.Close()
+	defer func() {
+		err := iter.Close()
+		if err != nil {
+			log.ErrorC("error closing edition iterator", err, log.Data{"selector": selector})
+		}
+	}()
 
 	var results []models.Edition
 	if err := iter.All(&results); err != nil {
@@ -194,7 +204,12 @@ func (m *Mongo) GetVersions(id, editionID, state string) (*models.VersionResults
 	selector := buildVersionsQuery(id, editionID, state)
 
 	iter := s.DB(m.Database).C("instances").Find(selector).Iter()
-	defer iter.Close()
+	defer func() {
+		err := iter.Close()
+		if err != nil {
+			log.ErrorC("error closing instance iterator ", err, log.Data{"selector": selector})
+		}
+	}()
 
 	var results []models.Version
 	if err := iter.All(&results); err != nil {
@@ -299,7 +314,7 @@ func (m *Mongo) UpdateDataset(id string, dataset *models.Dataset, currentState s
 }
 
 func createDatasetUpdateQuery(id string, dataset *models.Dataset, currentState string) bson.M {
-	updates := make(bson.M, 0)
+	updates := make(bson.M)
 
 	log.Debug("building update query for dataset resource", log.Data{"dataset_id": id, "dataset": dataset, "updates": updates})
 
@@ -463,45 +478,49 @@ func (m *Mongo) UpdateVersion(id string, version *models.Version) (err error) {
 }
 
 func createVersionUpdateQuery(version *models.Version) bson.M {
-	updates := make(bson.M, 0)
+	setUpdates := make(bson.M)
 
 	if version.Alerts != nil {
-		updates["alerts"] = version.Alerts
+		setUpdates["alerts"] = version.Alerts
 	}
 
 	if version.CollectionID != "" {
-		updates["collection_id"] = version.CollectionID
+		setUpdates["collection_id"] = version.CollectionID
 	}
 
 	if version.LatestChanges != nil {
-		updates["latest_changes"] = version.LatestChanges
+		setUpdates["latest_changes"] = version.LatestChanges
 	}
 
 	if version.ReleaseDate != "" {
-		updates["release_date"] = version.ReleaseDate
+		setUpdates["release_date"] = version.ReleaseDate
 	}
 
 	if version.Links != nil {
 		if version.Links.Spatial != nil {
 			if version.Links.Spatial.HRef != "" {
-				updates["links.spatial.href"] = version.Links.Spatial.HRef
+				setUpdates["links.spatial.href"] = version.Links.Spatial.HRef
 			}
 		}
 	}
 
 	if version.State != "" {
-		updates["state"] = version.State
+		setUpdates["state"] = version.State
 	}
 
 	if version.Temporal != nil {
-		updates["temporal"] = version.Temporal
+		setUpdates["temporal"] = version.Temporal
 	}
 
 	if version.Downloads != nil {
-		updates["downloads"] = version.Downloads
+		setUpdates["downloads"] = version.Downloads
 	}
 
-	return updates
+	if version.UsageNotes != nil {
+		setUpdates["usage_notes"] = version.UsageNotes
+	}
+
+	return setUpdates
 }
 
 // UpsertDataset adds or overides an existing dataset document
@@ -625,6 +644,7 @@ func (m *Mongo) CheckEditionExists(id, editionID, state string) error {
 	return nil
 }
 
+// Ping the mongodb database
 func (m *Mongo) Ping(ctx context.Context) (time.Time, error) {
 	if time.Since(m.lastPingTime) < 1*time.Second {
 		return m.lastPingTime, m.lastPingResult

mongo/dimension_store.go

@@ -57,11 +57,8 @@ func (m *Mongo) AddDimensionToInstance(opt *models.CachedDimensionOption) error
 	option.LastUpdated = time.Now().UTC()
 	_, err := s.DB(m.Database).C(dimensionOptions).Upsert(bson.M{"instance_id": option.InstanceID, "name": option.Name,
 		"option": option.Option}, &option)
-	if err != nil {
-		return err
-	}
 
-	return nil
+	return err
 }
 
 // GetDimensions returns a list of all dimensions from a dataset

mongo/instance_store.go

@@ -5,6 +5,7 @@ import (
 
 	errs "github.com/ONSdigital/dp-dataset-api/apierrors"
 	"github.com/ONSdigital/dp-dataset-api/models"
+	"github.com/ONSdigital/go-ns/log"
 	"gopkg.in/mgo.v2"
 	"gopkg.in/mgo.v2/bson"
 )
@@ -22,7 +23,12 @@ func (m *Mongo) GetInstances(filters []string) (*models.InstanceResults, error)
 	}
 
 	iter := s.DB(m.Database).C(instanceCollection).Find(stateFilter).Iter()
-	defer iter.Close()
+	defer func() {
+		err := iter.Close()
+		if err != nil {
+			log.ErrorC("error closing iterator", err, log.Data{"data": filters})
+		}
+	}()
 
 	results := []models.Instance{}
 	if err := iter.All(&results); err != nil {