Merge branch 'feature/download-service' into cmd-develop

README.md

@@ -44,21 +44,22 @@ one of:
 
 ### Configuration
 
-| Environment variable       | Default                              | Description
-| -------------------------- | -------------------------------------| -----------
-| BIND_ADDR                  | :22000                               | The host and port to bind to
-| MONGODB_BIND_ADDR          | localhost:27017                      | The MongoDB bind address
-| MONGODB_DATABASE           | datasets                             | The MongoDB dataset database
-| MONGODB_COLLECTION         | datasets                             | MongoDB collection
-| SECRET_KEY                 | FD0108EA-825D-411C-9B1D-41EF7727F465 | A secret key used authentication
-| CODE_LIST_API_URL          | http://localhost:22400               | The host name for the CodeList API
-| DATASET_API_URL            | http://localhost:22000               | The host name for the Dataset API
-| GRACEFUL_SHUTDOWN_TIMEOUT  | 5s                                   | The graceful shutdown timeout in seconds
-| WEBSITE_URL                | http://localhost:20000               | The host name for the website
-| KAFKA_ADDR                 | "localhost:9092"                     | The list of kafka hosts
-| GENERATE_DOWNLOADS_TOPIC   | "filter-job-submitted"               | The topic to send generate full dataset version downloads to
-| HEALTHCHECK_TIMEOUT        | 2s                                   | The timeout that the healthcheck allows for checked subsystems
-| ENABLE_PRIVATE_ENDPOINTS   | false                                | Enable private endpoints for the API
+| Environment variable        | Default                                | Description
+| --------------------------- | ---------------------------------------| -----------
+| BIND_ADDR                   | :22000                                 | The host and port to bind to
+| MONGODB_BIND_ADDR           | localhost:27017                        | The MongoDB bind address
+| MONGODB_DATABASE            | datasets                               | The MongoDB dataset database
+| MONGODB_COLLECTION          | datasets                               | MongoDB collection
+| SECRET_KEY                  | FD0108EA-825D-411C-9B1D-41EF7727F465   | A secret key used authentication
+| CODE_LIST_API_URL           | http://localhost:22400                 | The host name for the CodeList API
+| DATASET_API_URL             | http://localhost:22000                 | The host name for the Dataset API
+| GRACEFUL_SHUTDOWN_TIMEOUT   | 5s                                     | The graceful shutdown timeout in seconds
+| WEBSITE_URL                 | http://localhost:20000                 | The host name for the website
+| KAFKA_ADDR                  | "localhost:9092"                       | The list of kafka hosts
+| GENERATE_DOWNLOADS_TOPIC    | "filter-job-submitted"                 | The topic to send generate full dataset version downloads to
+| HEALTHCHECK_TIMEOUT         | 2s                                     | The timeout that the healthcheck allows for checked subsystems
+| ENABLE_PRIVATE_ENDPOINTS    | false                                  | Enable private endpoints for the API
+| DOWNLOAD_SERVICE_SECRET_KEY | "QB0108EZ-825D-412C-9B1D-41EF7747F462" | A key specific for the download service to access public/private links
 
 ### Contributing
 

api/api.go

@@ -32,6 +32,7 @@ type DatasetAPI struct {
 	dataStore            store.DataStore
 	host                 string
 	internalToken        string
+	downloadServiceToken string
 	EnablePrePublishView bool
 	privateAuth          *auth.Authenticator
 	router               *mux.Router
@@ -71,6 +72,7 @@ func routes(cfg config.Configuration, router *mux.Router, dataStore store.DataSt
 		dataStore:            dataStore,
 		host:                 cfg.DatasetAPIURL,
 		internalToken:        cfg.SecretKey,
+		downloadServiceToken: cfg.DownloadServiceSecretKey,
 		EnablePrePublishView: cfg.EnablePrivateEnpoints,
 		router:               router,
 		urlBuilder:           urlBuilder,

api/dataset.go

@@ -20,6 +20,7 @@ const (
 	datasetDocType         = "dataset"
 	editionDocType         = "edition"
 	versionDocType         = "version"
+	downloadServiceToken   = "X-Download-Service-Token"
 	dimensionDocType       = "dimension"
 	dimensionOptionDocType = "dimension-option"
 )
@@ -246,6 +247,21 @@ func (api *DatasetAPI) getVersions(w http.ResponseWriter, r *http.Request) {
 			hasInvalidState = true
 			log.ErrorC("unpublished version has an invalid state", err, log.Data{"state": item.State})
 		}
+
+		// Only the download service should not have access to the public/private download
+		// fields
+		if r.Header.Get(downloadServiceToken) != api.downloadServiceToken {
+			if item.Downloads != nil {
+				if item.Downloads.CSV != nil {
+					item.Downloads.CSV.Private = ""
+					item.Downloads.CSV.Public = ""
+				}
+				if item.Downloads.XLS != nil {
+					item.Downloads.XLS.Private = ""
+					item.Downloads.XLS.Public = ""
+				}
+			}
+		}
 	}
 
 	if hasInvalidState {
@@ -313,6 +329,21 @@ func (api *DatasetAPI) getVersion(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Only the download service should not have access to the public/private download
+	// fields
+	if r.Header.Get(downloadServiceToken) != api.downloadServiceToken {
+		if results.Downloads != nil {
+			if results.Downloads.CSV != nil {
+				results.Downloads.CSV.Private = ""
+				results.Downloads.CSV.Public = ""
+			}
+			if results.Downloads.XLS != nil {
+				results.Downloads.XLS.Private = ""
+				results.Downloads.XLS.Public = ""
+			}
+		}
+	}
+
 	bytes, err := json.Marshal(results)
 	if err != nil {
 		log.ErrorC("failed to marshal version resource into bytes", err, logData)

config/config.go

@@ -9,17 +9,18 @@ import (
 
 // Configuration structure which hold information for configuring the import API
 type Configuration struct {
-	BindAddr                string        `envconfig:"BIND_ADDR"`
-	KafkaAddr               []string      `envconfig:"KAFKA_ADDR"                 json:"-"`
-	GenerateDownloadsTopic  string        `envconfig:"GENERATE_DOWNLOADS_TOPIC"`
-	CodeListAPIURL          string        `envconfig:"CODE_LIST_API_URL"`
-	DatasetAPIURL           string        `envconfig:"DATASET_API_URL"`
-	WebsiteURL              string        `envconfig:"WEBSITE_URL"`
-	SecretKey               string        `envconfig:"SECRET_KEY"                 json:"-"`
-	GracefulShutdownTimeout time.Duration `envconfig:"GRACEFUL_SHUTDOWN_TIMEOUT"`
-	HealthCheckTimeout      time.Duration `envconfig:"HEALTHCHECK_TIMEOUT"`
-	EnablePrivateEnpoints   bool          `envconfig:"ENABLE_PRIVATE_ENDPOINTS"`
-	MongoConfig             MongoConfig
+	BindAddr                 string        `envconfig:"BIND_ADDR"`
+	KafkaAddr                []string      `envconfig:"KAFKA_ADDR"                       json:"-"`
+	GenerateDownloadsTopic   string        `envconfig:"GENERATE_DOWNLOADS_TOPIC"`
+	CodeListAPIURL           string        `envconfig:"CODE_LIST_API_URL"`
+	DatasetAPIURL            string        `envconfig:"DATASET_API_URL"`
+	DownloadServiceSecretKey string        `envconfig:"DOWNLOAD_SERVICE_SECRET_KEY"      json:"-"`
+	WebsiteURL               string        `envconfig:"WEBSITE_URL"`
+	SecretKey                string        `envconfig:"SECRET_KEY"                       json:"-"`
+	GracefulShutdownTimeout  time.Duration `envconfig:"GRACEFUL_SHUTDOWN_TIMEOUT"`
+	HealthCheckTimeout       time.Duration `envconfig:"HEALTHCHECK_TIMEOUT"`
+	EnablePrivateEnpoints    bool          `envconfig:"ENABLE_PRIVATE_ENDPOINTS"`
+	MongoConfig              MongoConfig
 }
 
 // MongoConfig contains the config required to connect to MongoDB.
@@ -38,16 +39,17 @@ func Get() (*Configuration, error) {
 	}
 
 	cfg = &Configuration{
-		BindAddr:                ":22000",
-		KafkaAddr:               []string{"localhost:9092"},
-		GenerateDownloadsTopic:  "filter-job-submitted",
-		CodeListAPIURL:          "http://localhost:22400",
-		DatasetAPIURL:           "http://localhost:22000",
-		WebsiteURL:              "http://localhost:20000",
-		SecretKey:               "FD0108EA-825D-411C-9B1D-41EF7727F465",
-		GracefulShutdownTimeout: 5 * time.Second,
-		HealthCheckTimeout:      2 * time.Second,
-		EnablePrivateEnpoints:   false,
+		BindAddr:                 ":22000",
+		KafkaAddr:                []string{"localhost:9092"},
+		GenerateDownloadsTopic:   "filter-job-submitted",
+		CodeListAPIURL:           "http://localhost:22400",
+		DatasetAPIURL:            "http://localhost:22000",
+		WebsiteURL:               "http://localhost:20000",
+		SecretKey:                "FD0108EA-825D-411C-9B1D-41EF7727F465",
+		DownloadServiceSecretKey: "QB0108EZ-825D-412C-9B1D-41EF7747F462",
+		GracefulShutdownTimeout:  5 * time.Second,
+		HealthCheckTimeout:       2 * time.Second,
+		EnablePrivateEnpoints:    false,
 		MongoConfig: MongoConfig{
 			BindAddr:   "localhost:27017",
 			Collection: "datasets",

config/config_test.go

@@ -23,6 +23,7 @@ func TestSpec(t *testing.T) {
 				So(cfg.GenerateDownloadsTopic, ShouldEqual, "filter-job-submitted")
 				So(cfg.DatasetAPIURL, ShouldEqual, "http://localhost:22000")
 				So(cfg.CodeListAPIURL, ShouldEqual, "http://localhost:22400")
+				So(cfg.DownloadServiceSecretKey, ShouldEqual, "QB0108EZ-825D-412C-9B1D-41EF7747F462")
 				So(cfg.WebsiteURL, ShouldEqual, "http://localhost:20000")
 				So(cfg.SecretKey, ShouldEqual, "FD0108EA-825D-411C-9B1D-41EF7727F465")
 				So(cfg.GracefulShutdownTimeout, ShouldEqual, 5*time.Second)

models/dataset.go

@@ -164,7 +164,9 @@ type DownloadObject struct {
 	URL string `bson:"url,omitempty"  json:"url,omitempty"`
 	// TODO size is in bytes and probably should be an int64 instead of a string this
 	// will have to change for several services (filter API, exporter services and web)
-	Size string `bson:"size,omitempty" json:"size,omitempty"`
+	Size    string `bson:"size,omitempty" json:"size,omitempty"`
+	Public  string `bson:"public,omitempty" json:"public,omitempty"`
+	Private string `bson:"private,omitempty" json:"private,omitempty"`
 }
 
 // LatestChange represents an object contining

swagger.yaml

@@ -146,6 +146,11 @@ securityDefinitions:
     description: "API key used to allow only internal services to update the state of an import job"
     in: header
     type: apiKey
+  DownloadServiceAPIKey:
+    name: x-download-service-token
+    description: "API key used to allow the download service to access public and private links to a download"
+    in: header
+    type: apiKey
 paths:
   /search/datasets:
     get:
@@ -1507,6 +1512,12 @@ definitions:
       size:
         type: string
         description: "The size of the file in bytes"
+      public:
+        type: string
+        description: "The URL to a public-accessible download"
+      private:
+        type: string
+        description: "The URL to a non public-accessible download"
   Alert:
     # TODO Update description, so it is useful to an API customer
     description: "A single alert, ☃"