From dc11e82cbbdb8ab52d5a52364952fad59aaaf13b Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Fri, 11 Aug 2017 20:07:29 -0700 Subject: [PATCH 1/7] Add requester pays samples --- storage/requesterPays.js | 191 ++++++++++++++++++++++ storage/system-test/requesterPays.test.js | 89 ++++++++++ 2 files changed, 280 insertions(+) create mode 100644 storage/requesterPays.js create mode 100644 storage/system-test/requesterPays.test.js diff --git a/storage/requesterPays.js b/storage/requesterPays.js new file mode 100644 index 0000000000..ca5114f62c --- /dev/null +++ b/storage/requesterPays.js @@ -0,0 +1,191 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * This application demonstrates how to perform basic operations on buckets with + * the Google Cloud Storage API. + * + * For more information, see the README.md under /storage and the documentation + * at https://cloud.google.com/storage/docs. + */ + +'use strict'; + +function enableRequesterPays (bucketName) { + // [START enable_requester_pays] + // Imports the Google Cloud client library + const Storage = require('@google-cloud/storage'); + + // The name of the bucket to enable requester-paying for, e.g. "my-bucket" + // const bucketName = "my-bucket"; + + // Instantiates a client + const storage = Storage(); + + // Enables requester-pays requests + storage + .bucket(bucketName) + .enableRequesterPays() + .then(() => { + console.log(`Requester-pays requests have been enabled for bucket ${bucketName}.`); + }) + .catch((err) => { + console.error('ERROR:', err); + }); + // [END enable_requester_pays] +} + +function disableRequesterPays (bucketName) { + // [START disable_requester_pays] + // Imports the Google Cloud client library + const Storage = require('@google-cloud/storage'); + + // The name of the bucket to disable requester-paying for, e.g. "my-bucket" + // const bucketName = "my-bucket"; + + // Instantiates a client + const storage = Storage(); + + // Disables requester-pays requests + storage + .bucket(bucketName) + .disableRequesterPays() + .then(() => { + console.log(`Requester-pays requests have been disabled for bucket ${bucketName}.`); + }) + .catch((err) => { + console.error('ERROR:', err); + }); + // [END disable_requester_pays] +} + +function getRequesterPaysStatus (bucketName) { + // [START get_requester_pays_status] + // Imports the Google Cloud client library + const Storage = require('@google-cloud/storage'); + + // The name of the bucket to get the requester-payable status for, e.g. "my-bucket" + // const bucketName = "my-bucket"; + + // Instantiates a client + const storage = Storage(); + + // Gets the requester-pays status of a bucket + storage + .bucket(bucketName) + .getMetadata() + .then((data) => { + let status; + const metadata = data[0]; + if (metadata && metadata.billing && metadata.billing.requesterPays) { + status = `enabled`; + } else { + status = `disabled`; + } + console.log(`Requester-pays requests are ${status} for bucket ${bucketName}.`); + }) + .catch((err) => { + console.error('ERROR:', err); + }); + // [END get_requester_pays_status] +} + +function downloadFileUsingRequesterPays (projectId, bucketName, srcFilename, destFilename) { + // [START storage_download_file_requester_pays] + // Imports the Google Cloud client library + const Storage = require('@google-cloud/storage'); + + // The project ID to bill from + // const projectId = process.env.GCLOUD_PROJECT; + + // The name of the bucket to access, e.g. "my-bucket" + // const bucketName = "my-bucket"; + + // The name of the remote file to download, e.g. "file.txt" + // const srcFilename = "file.txt"; + + // The path to which the file should be downloaded, e.g. "./local/path/to/file.txt" + // const destFilename = "./local/path/to/file.txt"; + + // Instantiates a client + const storage = Storage(); + + const options = { + // The path to which the file should be downloaded, e.g. "./file.txt" + destination: destFilename, + + // Whether or not to bill the requesting project, if requester-pays requests are enabled + userProject: projectId + }; + + // Downloads the file + storage + .bucket(bucketName) + .file(srcFilename) + .download(options) + .then(() => { + console.log(`gs://${bucketName}/${srcFilename} downloaded to ${destFilename} using requester-pays requests.`); + }) + .catch((err) => { + console.error('ERROR:', err); + }); + // [END storage_download_file_requester_pays] +} + +const cli = require(`yargs`) + .demand(1) + .command( + `enable `, + `Enables requester-pays requests on a bucket.`, + {}, + (opts) => enableRequesterPays(opts.bucket) + ) + .command( + `disable `, + `Disables requester-pays requests on a bucket.`, + {}, + (opts) => disableRequesterPays(opts.bucket) + ) + .command( + `get-status `, + `Determines whether requester-pays requests are enabled on a bucket.`, + {}, + (opts) => getRequesterPaysStatus(opts.bucket) + ) + .command( + `download `, + `Downloads a file from a bucket using requester-pays requests.`, + { + projectId: { + type: 'string', + alias: 'p', + default: process.env.GCLOUD_PROJECT + } + }, + (opts) => downloadFileUsingRequesterPays(opts.projectId, opts.bucketName, opts.srcFileName, opts.destFileName) + ) + .example(`node $0 enable my-bucket`, `Enables requester-pays requests on a bucket named "my-bucket".`) + .example(`node $0 list`, `Disables requester-pays requests on a bucket named "my-bucket".`) + .example(`node $0 delete my-bucket`, `Determines whether requester-pays requests are enabled for a bucket named "my-bucket".`) + .example(`node $0 download my-bucket file.txt ./file.txt`, `Downloads "gs://my-bucket/file.txt" to "./file.txt" using requester-pays requests.`) + .wrap(120) + .recommendCommands() + .epilogue(`For more information, see https://cloud.google.com/storage/docs`) + .help() + .strict(); + +if (module === require.main) { + cli.parse(process.argv.slice(2)); +} diff --git a/storage/system-test/requesterPays.test.js b/storage/system-test/requesterPays.test.js new file mode 100644 index 0000000000..015ca9737d --- /dev/null +++ b/storage/system-test/requesterPays.test.js @@ -0,0 +1,89 @@ +/** + * Copyright 2017, Google, Inc. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const fs = require(`fs`); +const path = require(`path`); +const storage = require(`@google-cloud/storage`)(); +const test = require(`ava`); +const tools = require(`@google-cloud/nodejs-repo-tools`); +const uuid = require(`uuid`); + +const cwd = path.join(__dirname, `..`); +const cmd = `node requesterPays.js`; +const bucketName = `nodejs-docs-samples-test-${uuid.v4()}`; +const fileName = `test.txt`; +const bucket = storage.bucket(bucketName); + +const uploadFilePath = path.join(cwd, `resources`, fileName); +const downloadFilePath = path.join(__dirname, `test_${uuid.v4()}.txt`); + +test.before(async () => { + tools.checkCredentials(); + await bucket.create(); + + // Upload a test file (to download later) + await bucket.upload(uploadFilePath); +}); +test.after.always(async () => { + try { + fs.unlinkSync(downloadFilePath); + } catch (err) { + console.log(err); + } + // Try deleting all files twice, just to make sure + try { + await bucket.deleteFiles({ force: true }); + } catch (err) {} // ignore error + try { + await bucket.deleteFiles({ force: true }); + } catch (err) {} // ignore error + try { + await bucket.delete(); + } catch (err) {} // ignore error +}); + +test.serial(`should fetch requester-pays status on a default bucket`, async (t) => { + const output = await tools.runAsync(`${cmd} get-status ${bucketName}`, cwd); + t.is(output, `Requester-pays requests are disabled for bucket ${bucketName}.`); +}); + +test.serial(`should enable requester-pays requests`, async (t) => { + const output = await tools.runAsync(`${cmd} enable ${bucketName}`, cwd); + t.is(output, `Requester-pays requests have been enabled for bucket ${bucketName}.`); +}); + +test.serial(`should fetch requester-pays status on a modified bucket`, async (t) => { + const output = await tools.runAsync(`${cmd} get-status ${bucketName}`, cwd); + t.is(output, `Requester-pays requests are enabled for bucket ${bucketName}.`); +}); + +test.serial(`should download a file using requester-pays requests`, async (t) => { + const output = await tools.runAsync(`${cmd} download ${bucketName} ${fileName} ${downloadFilePath}`, cwd); + t.is(output, `gs://${bucketName}/${fileName} downloaded to ${downloadFilePath} using requester-pays requests.`); + await t.notThrows(() => fs.statSync(downloadFilePath)); +}); + +test.serial(`should disable requester-pays requests`, async (t) => { + const output = await tools.runAsync(`${cmd} disable ${bucketName}`, cwd); + t.is(output, `Requester-pays requests have been disabled for bucket ${bucketName}.`); +}); + +test.serial(`should error on requester-pays requests if they are disabled`, async (t) => { + const result = await tools.runAsyncWithIO(`${cmd} download ${bucketName} ${fileName} ${downloadFilePath}`, cwd); + t.truthy(result.stderr); + t.regex(result.stderr, /User project prohibited for non requester pays bucket/); +}); From b971c98422486f5d3333763bb2dc2c4319f1fc57 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Fri, 11 Aug 2017 20:10:47 -0700 Subject: [PATCH 2/7] Fix comment --- storage/requesterPays.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/requesterPays.js b/storage/requesterPays.js index ca5114f62c..2042ff61f6 100644 --- a/storage/requesterPays.js +++ b/storage/requesterPays.js @@ -126,7 +126,7 @@ function downloadFileUsingRequesterPays (projectId, bucketName, srcFilename, des // The path to which the file should be downloaded, e.g. "./file.txt" destination: destFilename, - // Whether or not to bill the requesting project, if requester-pays requests are enabled + // The project to bill from, if requester-pays requests are enabled userProject: projectId }; From 108b7556a9518cfa8e6063c1dbda0b5decbf6b57 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Tue, 15 Aug 2017 09:43:01 -0700 Subject: [PATCH 3/7] Update dependency to fix test --- storage/package.json | 2 +- storage/yarn.lock | 22 ++++++++++++++++++---- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/storage/package.json b/storage/package.json index 608276466b..822fb9d2b5 100644 --- a/storage/package.json +++ b/storage/package.json @@ -26,7 +26,7 @@ "yargs": "8.0.2" }, "devDependencies": { - "@google-cloud/nodejs-repo-tools": "1.4.16", + "@google-cloud/nodejs-repo-tools": "1.4.17", "ava": "0.21.0", "proxyquire": "1.8.0", "sinon": "3.0.0", diff --git a/storage/yarn.lock b/storage/yarn.lock index 813c7b8964..1d9f5cb80e 100644 --- a/storage/yarn.lock +++ b/storage/yarn.lock @@ -67,9 +67,9 @@ string-format-obj "^1.1.0" through2 "^2.0.3" -"@google-cloud/nodejs-repo-tools@1.4.16": - version "1.4.16" - resolved "https://registry.yarnpkg.com/@google-cloud/nodejs-repo-tools/-/nodejs-repo-tools-1.4.16.tgz#a87b1f9db8426494ee7ea21a3a0cf172c66fe350" +"@google-cloud/nodejs-repo-tools@1.4.17": + version "1.4.17" + resolved "https://registry.yarnpkg.com/@google-cloud/nodejs-repo-tools/-/nodejs-repo-tools-1.4.17.tgz#6458d12467cf93dc931d64640afabca1dfc19af2" dependencies: ava "0.21.0" colors "1.1.2" @@ -78,7 +78,7 @@ handlebars "4.0.10" lodash "4.17.4" proxyquire "1.8.0" - sinon "3.0.0" + sinon "3.2.0" string "3.3.3" supertest "3.0.0" yargs "8.0.2" @@ -2910,6 +2910,20 @@ sinon@3.0.0: text-encoding "0.6.4" type-detect "^4.0.0" +sinon@3.2.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/sinon/-/sinon-3.2.0.tgz#8848a66ab6e8b80b5532e3824f59f83ea2628c77" + dependencies: + diff "^3.1.0" + formatio "1.2.0" + lolex "^2.1.2" + native-promise-only "^0.8.1" + nise "^1.0.1" + path-to-regexp "^1.7.0" + samsam "^1.1.3" + text-encoding "0.6.4" + type-detect "^4.0.0" + slash@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55" From 407aca2a377b59421bc4086de8c28ca9518c27fe Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Tue, 15 Aug 2017 13:01:44 -0700 Subject: [PATCH 4/7] Add README --- storage/README.md | 34 ++++++++++++++++++++++++++++++++++ storage/package.json | 7 +++++++ 2 files changed, 41 insertions(+) diff --git a/storage/README.md b/storage/README.md index 06ff0a7749..53c37b795a 100644 --- a/storage/README.md +++ b/storage/README.md @@ -15,6 +15,7 @@ * [Encryption](#encryption) * [Files](#files) * [Storage Transfer API](#storage-transfer-api) + * [Requester Pays samples](#requester-pays-samples) * [Running the tests](#running-the-tests) ## Setup @@ -199,6 +200,39 @@ For more information, see https://cloud.google.com/storage/transfer [transfer_4_docs]: https://cloud.google.com/storage/transfer [transfer_4_code]: transfer.js +### Requester Pays samples + +View the [documentation][requesterPays_5_docs] or the [source code][requesterPays_5_code]. + +__Usage:__ `node requesterPays.js --help` + +``` +Commands: + enable Enables requester-pays requests on a bucket. + disable Disables requester-pays requests on a bucket. + get-status Determines whether requester-pays requests are enabled on a + bucket. + download Downloads a file from a bucket using requester-pays requests. + +Options: + --help Show help [boolean] + +Examples: + node requesterPays.js enable my-bucket Enables requester-pays requests on a bucket named + "my-bucket". + node requesterPays.js list Disables requester-pays requests on a bucket named + "my-bucket". + node requesterPays.js delete my-bucket Determines whether requester-pays requests are enabled + for a bucket named "my-bucket". + node requesterPays.js download my-bucket file.txt ./file.txt Downloads "gs://my-bucket/file.txt" to "./file.txt" + using requester-pays requests. + +For more information, see https://cloud.google.com/storage/docs +``` + +[requesterPays_5_docs]: https://cloud.google.com/storage/docs +[requesterPays_5_code]: requesterPays.js + ## Running the tests 1. Set the **GCLOUD_PROJECT** and **GOOGLE_APPLICATION_CREDENTIALS** environment variables. diff --git a/storage/package.json b/storage/package.json index 822fb9d2b5..660dab2b21 100644 --- a/storage/package.json +++ b/storage/package.json @@ -71,6 +71,13 @@ "file": "transfer.js", "docs_link": "https://cloud.google.com/storage/transfer", "usage": "node transfer.js --help" + }, + { + "id": "requesterPays", + "name": "Requester Pays samples", + "file": "requesterPays.js", + "docs_link": "https://cloud.google.com/storage/docs", + "usage": "node requesterPays.js --help" } ] } From 7ac02821ac9ac02c4bf812d7081d8f0662f62b77 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Tue, 15 Aug 2017 16:00:55 -0700 Subject: [PATCH 5/7] Fix typo --- storage/README.md | 4 ++-- storage/requesterPays.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/README.md b/storage/README.md index 53c37b795a..34c64e4f72 100644 --- a/storage/README.md +++ b/storage/README.md @@ -220,9 +220,9 @@ Options: Examples: node requesterPays.js enable my-bucket Enables requester-pays requests on a bucket named "my-bucket". - node requesterPays.js list Disables requester-pays requests on a bucket named + node requesterPays.js disable Disables requester-pays requests on a bucket named "my-bucket". - node requesterPays.js delete my-bucket Determines whether requester-pays requests are enabled + node requesterPays.js get-status my-bucket Determines whether requester-pays requests are enabled for a bucket named "my-bucket". node requesterPays.js download my-bucket file.txt ./file.txt Downloads "gs://my-bucket/file.txt" to "./file.txt" using requester-pays requests. diff --git a/storage/requesterPays.js b/storage/requesterPays.js index 2042ff61f6..2ab54024c1 100644 --- a/storage/requesterPays.js +++ b/storage/requesterPays.js @@ -177,8 +177,8 @@ const cli = require(`yargs`) (opts) => downloadFileUsingRequesterPays(opts.projectId, opts.bucketName, opts.srcFileName, opts.destFileName) ) .example(`node $0 enable my-bucket`, `Enables requester-pays requests on a bucket named "my-bucket".`) - .example(`node $0 list`, `Disables requester-pays requests on a bucket named "my-bucket".`) - .example(`node $0 delete my-bucket`, `Determines whether requester-pays requests are enabled for a bucket named "my-bucket".`) + .example(`node $0 disable`, `Disables requester-pays requests on a bucket named "my-bucket".`) + .example(`node $0 get-status my-bucket`, `Determines whether requester-pays requests are enabled for a bucket named "my-bucket".`) .example(`node $0 download my-bucket file.txt ./file.txt`, `Downloads "gs://my-bucket/file.txt" to "./file.txt" using requester-pays requests.`) .wrap(120) .recommendCommands() From 73dbd4faf052e89c72d008c874b3ac4dc5bcbac7 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Tue, 15 Aug 2017 16:06:07 -0700 Subject: [PATCH 6/7] Switch to backticks --- storage/requesterPays.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/storage/requesterPays.js b/storage/requesterPays.js index 2ab54024c1..b08698227e 100644 --- a/storage/requesterPays.js +++ b/storage/requesterPays.js @@ -26,7 +26,7 @@ function enableRequesterPays (bucketName) { // [START enable_requester_pays] // Imports the Google Cloud client library - const Storage = require('@google-cloud/storage'); + const Storage = require(`@google-cloud/storage`); // The name of the bucket to enable requester-paying for, e.g. "my-bucket" // const bucketName = "my-bucket"; @@ -42,7 +42,7 @@ function enableRequesterPays (bucketName) { console.log(`Requester-pays requests have been enabled for bucket ${bucketName}.`); }) .catch((err) => { - console.error('ERROR:', err); + console.error(`ERROR:`, err); }); // [END enable_requester_pays] } @@ -50,7 +50,7 @@ function enableRequesterPays (bucketName) { function disableRequesterPays (bucketName) { // [START disable_requester_pays] // Imports the Google Cloud client library - const Storage = require('@google-cloud/storage'); + const Storage = require(`@google-cloud/storage`); // The name of the bucket to disable requester-paying for, e.g. "my-bucket" // const bucketName = "my-bucket"; @@ -66,7 +66,7 @@ function disableRequesterPays (bucketName) { console.log(`Requester-pays requests have been disabled for bucket ${bucketName}.`); }) .catch((err) => { - console.error('ERROR:', err); + console.error(`ERROR:`, err); }); // [END disable_requester_pays] } @@ -74,7 +74,7 @@ function disableRequesterPays (bucketName) { function getRequesterPaysStatus (bucketName) { // [START get_requester_pays_status] // Imports the Google Cloud client library - const Storage = require('@google-cloud/storage'); + const Storage = require(`@google-cloud/storage`); // The name of the bucket to get the requester-payable status for, e.g. "my-bucket" // const bucketName = "my-bucket"; @@ -97,7 +97,7 @@ function getRequesterPaysStatus (bucketName) { console.log(`Requester-pays requests are ${status} for bucket ${bucketName}.`); }) .catch((err) => { - console.error('ERROR:', err); + console.error(`ERROR:`, err); }); // [END get_requester_pays_status] } @@ -105,7 +105,7 @@ function getRequesterPaysStatus (bucketName) { function downloadFileUsingRequesterPays (projectId, bucketName, srcFilename, destFilename) { // [START storage_download_file_requester_pays] // Imports the Google Cloud client library - const Storage = require('@google-cloud/storage'); + const Storage = require(`@google-cloud/storage`); // The project ID to bill from // const projectId = process.env.GCLOUD_PROJECT; @@ -139,7 +139,7 @@ function downloadFileUsingRequesterPays (projectId, bucketName, srcFilename, des console.log(`gs://${bucketName}/${srcFilename} downloaded to ${destFilename} using requester-pays requests.`); }) .catch((err) => { - console.error('ERROR:', err); + console.error(`ERROR:`, err); }); // [END storage_download_file_requester_pays] } @@ -169,8 +169,8 @@ const cli = require(`yargs`) `Downloads a file from a bucket using requester-pays requests.`, { projectId: { - type: 'string', - alias: 'p', + type: `string`, + alias: `p`, default: process.env.GCLOUD_PROJECT } }, From 37c1837d7181f81d4700ea671b7e0bf310ac070f Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Tue, 15 Aug 2017 16:32:26 -0700 Subject: [PATCH 7/7] Whoops, fix nit --- storage/README.md | 2 +- storage/requesterPays.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/README.md b/storage/README.md index 34c64e4f72..0d4e140ffb 100644 --- a/storage/README.md +++ b/storage/README.md @@ -220,7 +220,7 @@ Options: Examples: node requesterPays.js enable my-bucket Enables requester-pays requests on a bucket named "my-bucket". - node requesterPays.js disable Disables requester-pays requests on a bucket named + node requesterPays.js disable my-bucket Disables requester-pays requests on a bucket named "my-bucket". node requesterPays.js get-status my-bucket Determines whether requester-pays requests are enabled for a bucket named "my-bucket". diff --git a/storage/requesterPays.js b/storage/requesterPays.js index b08698227e..28a2916dbb 100644 --- a/storage/requesterPays.js +++ b/storage/requesterPays.js @@ -177,7 +177,7 @@ const cli = require(`yargs`) (opts) => downloadFileUsingRequesterPays(opts.projectId, opts.bucketName, opts.srcFileName, opts.destFileName) ) .example(`node $0 enable my-bucket`, `Enables requester-pays requests on a bucket named "my-bucket".`) - .example(`node $0 disable`, `Disables requester-pays requests on a bucket named "my-bucket".`) + .example(`node $0 disable my-bucket`, `Disables requester-pays requests on a bucket named "my-bucket".`) .example(`node $0 get-status my-bucket`, `Determines whether requester-pays requests are enabled for a bucket named "my-bucket".`) .example(`node $0 download my-bucket file.txt ./file.txt`, `Downloads "gs://my-bucket/file.txt" to "./file.txt" using requester-pays requests.`) .wrap(120)