From fe8a231c2dcc3ba621787b510b8dbdc379fba566 Mon Sep 17 00:00:00 2001 From: mattrout92 Date: Mon, 5 Mar 2018 11:00:41 +0000 Subject: [PATCH 1/3] Allow public a private links to be visible for download poc --- models/dataset.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/models/dataset.go b/models/dataset.go index df05bd3b..d9cfa13f 100644 --- a/models/dataset.go +++ b/models/dataset.go @@ -161,7 +161,9 @@ type DownloadObject struct { URL string `bson:"url,omitempty" json:"url,omitempty"` // TODO size is in bytes and probably should be an int64 instead of a string this // will have to change for several services (filter API, exporter services and web) - Size string `bson:"size,omitempty" json:"size,omitempty"` + Size string `bson:"size,omitempty" json:"size,omitempty"` + Public string `bson:"public,omitempty" json:"public,omitempty"` + Private string `bson:"private,omitempty" json:"private,omitempty"` } // LatestChange represents an object contining From 7565e45f50408c61d10ddf56d0d2ec53b49ce207 Mon Sep 17 00:00:00 2001 From: mattrout92 Date: Fri, 9 Mar 2018 12:08:41 +0000 Subject: [PATCH 2/3] Update swagger spec for public/private download links --- api/api.go | 2 ++ api/dataset.go | 31 ++++++++++++++++++++++++++++++ config/config.go | 44 ++++++++++++++++++++++--------------------- config/config_test.go | 1 + swagger.yaml | 11 +++++++++++ 5 files changed, 68 insertions(+), 21 deletions(-) diff --git a/api/api.go b/api/api.go index 25611877..d1256888 100644 --- a/api/api.go +++ b/api/api.go @@ -32,6 +32,7 @@ type DatasetAPI struct { dataStore store.DataStore host string internalToken string + downloadServiceToken string EnablePrePublishView bool privateAuth *auth.Authenticator router *mux.Router @@ -71,6 +72,7 @@ func routes(cfg config.Configuration, router *mux.Router, dataStore store.DataSt dataStore: dataStore, host: cfg.DatasetAPIURL, internalToken: cfg.SecretKey, + downloadServiceToken: cfg.DownloadServiceSecretKey, EnablePrePublishView: cfg.EnablePrivateEnpoints, router: router, urlBuilder: urlBuilder, diff --git a/api/dataset.go b/api/dataset.go index c233a99b..cdaa6a23 100644 --- a/api/dataset.go +++ b/api/dataset.go @@ -20,6 +20,7 @@ const ( datasetDocType = "dataset" editionDocType = "edition" versionDocType = "version" + downloadServiceToken = "X-Download-Service-Token" dimensionDocType = "dimension" dimensionOptionDocType = "dimension-option" ) @@ -246,6 +247,21 @@ func (api *DatasetAPI) getVersions(w http.ResponseWriter, r *http.Request) { hasInvalidState = true log.ErrorC("unpublished version has an invalid state", err, log.Data{"state": item.State}) } + + // Only the download service should not have access to the public/private download + // fields + if r.Header.Get(downloadServiceToken) != api.downloadServiceToken { + if item.Downloads != nil { + if item.Downloads.CSV != nil { + item.Downloads.CSV.Private = "" + item.Downloads.CSV.Public = "" + } + if item.Downloads.XLS != nil { + item.Downloads.XLS.Private = "" + item.Downloads.XLS.Public = "" + } + } + } } if hasInvalidState { @@ -313,6 +329,21 @@ func (api *DatasetAPI) getVersion(w http.ResponseWriter, r *http.Request) { return } + // Only the download service should not have access to the public/private download + // fields + if r.Header.Get(downloadServiceToken) != api.downloadServiceToken { + if results.Downloads != nil { + if results.Downloads.CSV != nil { + results.Downloads.CSV.Private = "" + results.Downloads.CSV.Public = "" + } + if results.Downloads.XLS != nil { + results.Downloads.XLS.Private = "" + results.Downloads.XLS.Public = "" + } + } + } + bytes, err := json.Marshal(results) if err != nil { log.ErrorC("failed to marshal version resource into bytes", err, logData) diff --git a/config/config.go b/config/config.go index 0faa6471..9ff24287 100644 --- a/config/config.go +++ b/config/config.go @@ -9,17 +9,18 @@ import ( // Configuration structure which hold information for configuring the import API type Configuration struct { - BindAddr string `envconfig:"BIND_ADDR"` - KafkaAddr []string `envconfig:"KAFKA_ADDR" json:"-"` - GenerateDownloadsTopic string `envconfig:"GENERATE_DOWNLOADS_TOPIC"` - CodeListAPIURL string `envconfig:"CODE_LIST_API_URL"` - DatasetAPIURL string `envconfig:"DATASET_API_URL"` - WebsiteURL string `envconfig:"WEBSITE_URL"` - SecretKey string `envconfig:"SECRET_KEY" json:"-"` - GracefulShutdownTimeout time.Duration `envconfig:"GRACEFUL_SHUTDOWN_TIMEOUT"` - HealthCheckTimeout time.Duration `envconfig:"HEALTHCHECK_TIMEOUT"` - EnablePrivateEnpoints bool `envconfig:"ENABLE_PRIVATE_ENDPOINTS"` - MongoConfig MongoConfig + BindAddr string `envconfig:"BIND_ADDR"` + KafkaAddr []string `envconfig:"KAFKA_ADDR" json:"-"` + GenerateDownloadsTopic string `envconfig:"GENERATE_DOWNLOADS_TOPIC"` + CodeListAPIURL string `envconfig:"CODE_LIST_API_URL"` + DatasetAPIURL string `envconfig:"DATASET_API_URL"` + DownloadServiceSecretKey string `envconfig:"DOWNLOAD_SERVICE_SECRET_KEY" json:"-"` + WebsiteURL string `envconfig:"WEBSITE_URL"` + SecretKey string `envconfig:"SECRET_KEY" json:"-"` + GracefulShutdownTimeout time.Duration `envconfig:"GRACEFUL_SHUTDOWN_TIMEOUT"` + HealthCheckTimeout time.Duration `envconfig:"HEALTHCHECK_TIMEOUT"` + EnablePrivateEnpoints bool `envconfig:"ENABLE_PRIVATE_ENDPOINTS"` + MongoConfig MongoConfig } // MongoConfig contains the config required to connect to MongoDB. @@ -38,16 +39,17 @@ func Get() (*Configuration, error) { } cfg = &Configuration{ - BindAddr: ":22000", - KafkaAddr: []string{"localhost:9092"}, - GenerateDownloadsTopic: "filter-job-submitted", - CodeListAPIURL: "http://localhost:22400", - DatasetAPIURL: "http://localhost:22000", - WebsiteURL: "http://localhost:20000", - SecretKey: "FD0108EA-825D-411C-9B1D-41EF7727F465", - GracefulShutdownTimeout: 5 * time.Second, - HealthCheckTimeout: 2 * time.Second, - EnablePrivateEnpoints: false, + BindAddr: ":22000", + KafkaAddr: []string{"localhost:9092"}, + GenerateDownloadsTopic: "filter-job-submitted", + CodeListAPIURL: "http://localhost:22400", + DatasetAPIURL: "http://localhost:22000", + WebsiteURL: "http://localhost:20000", + SecretKey: "FD0108EA-825D-411C-9B1D-41EF7727F465", + DownloadServiceSecretKey: "QB0108EZ-825D-412C-9B1D-41EF7747F462", + GracefulShutdownTimeout: 5 * time.Second, + HealthCheckTimeout: 2 * time.Second, + EnablePrivateEnpoints: false, MongoConfig: MongoConfig{ BindAddr: "localhost:27017", Collection: "datasets", diff --git a/config/config_test.go b/config/config_test.go index f25f97ef..03ddee10 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -23,6 +23,7 @@ func TestSpec(t *testing.T) { So(cfg.GenerateDownloadsTopic, ShouldEqual, "filter-job-submitted") So(cfg.DatasetAPIURL, ShouldEqual, "http://localhost:22000") So(cfg.CodeListAPIURL, ShouldEqual, "http://localhost:22400") + So(cfg.DownloadServiceSecretKey, ShouldEqual, "QB0108EZ-825D-412C-9B1D-41EF7747F462") So(cfg.WebsiteURL, ShouldEqual, "http://localhost:20000") So(cfg.SecretKey, ShouldEqual, "FD0108EA-825D-411C-9B1D-41EF7727F465") So(cfg.GracefulShutdownTimeout, ShouldEqual, 5*time.Second) diff --git a/swagger.yaml b/swagger.yaml index 63f9e636..8b5defcd 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -146,6 +146,11 @@ securityDefinitions: description: "API key used to allow only internal services to update the state of an import job" in: header type: apiKey + DownloadServiceAPIKey: + name: x-download-service-token + description: "API key used to allow the download service to access public and private links to a download" + in: header + type: apiKey paths: /search/datasets: get: @@ -1502,6 +1507,12 @@ definitions: size: type: string description: "The size of the file in bytes" + public: + type: string + description: "The URL to a public-accessible download" + private: + type: string + description: "The URL to a non public-accessible download" Alert: # TODO Update description, so it is useful to an API customer description: "A single alert, ☃" From c0e406f3b26ea8f856e69c088ed28db54335da91 Mon Sep 17 00:00:00 2001 From: mattrout92 Date: Mon, 12 Mar 2018 13:47:31 +0000 Subject: [PATCH 3/3] Update README with new config --- README.md | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 497cf52b..ad86cf12 100644 --- a/README.md +++ b/README.md @@ -44,21 +44,22 @@ one of: ### Configuration -| Environment variable | Default | Description -| -------------------------- | -------------------------------------| ----------- -| BIND_ADDR | :22000 | The host and port to bind to -| MONGODB_BIND_ADDR | localhost:27017 | The MongoDB bind address -| MONGODB_DATABASE | datasets | The MongoDB dataset database -| MONGODB_COLLECTION | datasets | MongoDB collection -| SECRET_KEY | FD0108EA-825D-411C-9B1D-41EF7727F465 | A secret key used authentication -| CODE_LIST_API_URL | http://localhost:22400 | The host name for the CodeList API -| DATASET_API_URL | http://localhost:22000 | The host name for the Dataset API -| GRACEFUL_SHUTDOWN_TIMEOUT | 5s | The graceful shutdown timeout in seconds -| WEBSITE_URL | http://localhost:20000 | The host name for the website -| KAFKA_ADDR | "localhost:9092" | The list of kafka hosts -| GENERATE_DOWNLOADS_TOPIC | "filter-job-submitted" | The topic to send generate full dataset version downloads to -| HEALTHCHECK_TIMEOUT | 2s | The timeout that the healthcheck allows for checked subsystems -| ENABLE_PRIVATE_ENDPOINTS | false | Enable private endpoints for the API +| Environment variable | Default | Description +| --------------------------- | ---------------------------------------| ----------- +| BIND_ADDR | :22000 | The host and port to bind to +| MONGODB_BIND_ADDR | localhost:27017 | The MongoDB bind address +| MONGODB_DATABASE | datasets | The MongoDB dataset database +| MONGODB_COLLECTION | datasets | MongoDB collection +| SECRET_KEY | FD0108EA-825D-411C-9B1D-41EF7727F465 | A secret key used authentication +| CODE_LIST_API_URL | http://localhost:22400 | The host name for the CodeList API +| DATASET_API_URL | http://localhost:22000 | The host name for the Dataset API +| GRACEFUL_SHUTDOWN_TIMEOUT | 5s | The graceful shutdown timeout in seconds +| WEBSITE_URL | http://localhost:20000 | The host name for the website +| KAFKA_ADDR | "localhost:9092" | The list of kafka hosts +| GENERATE_DOWNLOADS_TOPIC | "filter-job-submitted" | The topic to send generate full dataset version downloads to +| HEALTHCHECK_TIMEOUT | 2s | The timeout that the healthcheck allows for checked subsystems +| ENABLE_PRIVATE_ENDPOINTS | false | Enable private endpoints for the API +| DOWNLOAD_SERVICE_SECRET_KEY | "QB0108EZ-825D-412C-9B1D-41EF7747F462" | A key specific for the download service to access public/private links ### Contributing