From bfe3791a9490b49c1298e2e125aa01427579e417 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Fri, 2 Sep 2016 10:16:38 -0700 Subject: [PATCH 1/5] First draft of browseTable sample --- bigquery/system-test/tables.test.js | 72 +++++++++++------------------ bigquery/tables.js | 27 +++++++++-- bigquery/test/tables.test.js | 44 ++++++++++++++++-- 3 files changed, 91 insertions(+), 52 deletions(-) diff --git a/bigquery/system-test/tables.test.js b/bigquery/system-test/tables.test.js index 71e65ab052..075be852e5 100644 --- a/bigquery/system-test/tables.test.js +++ b/bigquery/system-test/tables.test.js @@ -36,47 +36,37 @@ var options = { schema: 'Name:string, Age:integer, Weight:float, IsMagic:boolean', rows: rows }; -var srcDataset = options.dataset; -var srcTable = options.table; -var destDataset = generateUuid(); -var destTable = generateUuid(); describe('bigquery:tables', function () { before(function (done) { // Create bucket storage.createBucket(options.bucket, function (err, bucket) { assert.ifError(err, 'bucket creation succeeded'); - // Upload data.csv + bucket.upload(options.localFilePath, function (err) { assert.ifError(err, 'file upload succeeded'); - // Create srcDataset - bigquery.createDataset(srcDataset, function (err) { - assert.ifError(err, 'srcDataset creation succeeded'); - // Create destDataset - bigquery.createDataset(destDataset, function (err) { - assert.ifError(err, 'destDataset creation succeeded'); - done(); - }); + + // Create dataset + bigquery.createDataset(options.dataset, function (err, dataset) { + assert.ifError(err, 'dataset creation succeeded'); + done(); }); }); }); }); after(function (done) { - // Delete srcDataset - bigquery.dataset(srcDataset).delete({ force: true }, function () { - // Delete destDataset - bigquery.dataset(destDataset).delete({ force: true }, function () { - // Delete files - storage.bucket(options.bucket).deleteFiles({ force: true }, function (err) { - if (err) { - return done(err); - } - // Delete bucket - setTimeout(function () { - storage.bucket(options.bucket).delete(done); - }, 2000); - }); + // Delete testing dataset/table + bigquery.dataset(options.dataset).delete({ force: true }, function () { + // Delete files + storage.bucket(options.bucket).deleteFiles({ force: true }, function (err) { + if (err) { + return done(err); + } + // Delete bucket + setTimeout(function () { + storage.bucket(options.bucket).delete(done); + }, 2000); }); }); }); @@ -167,26 +157,16 @@ describe('bigquery:tables', function () { }); }); - describe('copyTable', function () { - it('should copy a table between datasets', function (done) { - program.copyTable(srcDataset, srcTable, destDataset, destTable, function (err, metadata) { + describe('listRows', function () { + it('should list rows in a table', function (done) { + program.listRows(options.dataset, options.table, function (err, rows) { assert.equal(err, null); - assert.deepEqual(metadata.status, { state: 'DONE' }); - - bigquery.dataset(srcDataset).table(srcTable).exists( - function (err, exists) { - assert.equal(err, null); - assert.equal(exists, true, 'srcTable exists'); - - bigquery.dataset(destDataset).table(destTable).exists( - function (err, exists) { - assert.equal(err, null); - assert.equal(exists, true, 'destTable exists'); - done(); - } - ); - } - ); + assert.notEqual(rows, null); + assert.equal(Array.isArray(rows), true); + assert.equal(rows.length > 0, true); + assert.equal(console.log.calledOnce, true); + assert.deepEqual(console.log.firstCall.args, ['Found %d row(s)!', rows.length]); + done(); }); }); }); diff --git a/bigquery/tables.js b/bigquery/tables.js index 008497bc01..df5fde059b 100644 --- a/bigquery/tables.js +++ b/bigquery/tables.js @@ -80,6 +80,19 @@ function listTables (options, callback) { } // [END list_tables] +function listRows (dataset, table, callback) { + var bigquery = BigQuery(); + var tableObj = bigquery.dataset(dataset).table(table); + tableObj.getRows(function (err, rows) { + if (err) { + return callback(err); + } + + console.log('Found %d row(s)!', rows.length); + return callback(null, rows); + }); +} + // [START delete_table] /** * Deletes a table with the specified name from the specified dataset. @@ -236,6 +249,7 @@ var fs = require('fs'); var program = module.exports = { createTable: createTable, listTables: listTables, + listRows: listRows, deleteTable: deleteTable, importFile: importFile, exportTableToGCS: exportTableToGCS, @@ -252,7 +266,7 @@ cli .command('create ', 'Create a new table in the specified dataset.', {}, function (options) { program.createTable(utils.pick(options, ['dataset', 'table']), utils.makeHandler()); }) - .command('list ', 'List tables in the specified dataset.', {}, function (options) { + .command('tables ', 'List tables in the specified dataset.', {}, function (options) { program.listTables(utils.pick(options, ['dataset']), utils.makeHandler(true, 'id')); }) .command('delete
', 'Delete a table in the specified dataset.', {}, function (options) { @@ -270,6 +284,9 @@ cli ); } ) + .command('rows
', 'List the rows in a BigQuery table.', {}, function (options) { + program.listRows(options.dataset, options.table, utils.makeHandler()); + }) .command('import
', 'Import data from a local file or a Google Cloud Storage file into BigQuery.', { bucket: { alias: 'b', @@ -328,9 +345,13 @@ cli 'Create table "my_table" in "my_dataset".' ) .example( - 'node $0 list my_dataset', + 'node $0 tables my_dataset', 'List tables in "my_dataset".' ) + .example( + 'node $0 rows my_dataset my_table', + 'List rows from "my_table" in "my_dataset".' + ) .example( 'node $0 delete my_dataset my_table', 'Delete "my_table" from "my_dataset".' @@ -363,7 +384,7 @@ cli 'node $0 copy src_dataset src_table dest_dataset dest_table', 'Copy src_dataset:src_table to dest_dataset:dest_table.' ) - .wrap(100) + .wrap(120) .recommendCommands() .epilogue('For more information, see https://cloud.google.com/bigquery/docs'); diff --git a/bigquery/test/tables.test.js b/bigquery/test/tables.test.js index 267cc27886..16c9dc590a 100644 --- a/bigquery/test/tables.test.js +++ b/bigquery/test/tables.test.js @@ -58,9 +58,10 @@ function getSample () { var tableMock = { export: sinon.stub().yields(null, jobMock), copy: sinon.stub().yields(null, jobMock), - delete: sinon.stub().yields(null), import: sinon.stub().yields(null, jobMock), - insert: sinon.stub().yields(null, errorList) + insert: sinon.stub().yields(null, errorList), + getRows: sinon.stub().yields(null, jsonArray), + delete: sinon.stub().yields(null) }; var datasetMock = { table: sinon.stub().returns(tableMock), @@ -185,6 +186,34 @@ describe('bigquery:tables', function () { }); }); + describe('listRows', function () { + it('should list tables', function () { + var sample = getSample(); + var callback = sinon.stub(); + + sample.program.listRows(dataset, table, callback); + + assert.equal(sample.mocks.table.getRows.calledOnce, true); + assert.deepEqual(sample.mocks.table.getRows.firstCall.args.slice(0, -1), []); + assert.equal(callback.calledOnce, true); + assert.deepEqual(callback.firstCall.args, [null, jsonArray]); + assert.equal(console.log.calledOnce, true); + assert.deepEqual(console.log.firstCall.args, ['Found %d row(s)!', jsonArray.length]); + }); + + it('should handle error', function () { + var error = new Error('error'); + var sample = getSample(); + var callback = sinon.stub(); + sample.mocks.table.getRows.yields(error); + + sample.program.listRows(dataset, table, callback); + + assert.equal(callback.calledOnce, true); + assert.deepEqual(callback.firstCall.args, [error]); + }); + }); + describe('deleteTable', function () { it('should delete a table', function () { var sample = getSample(); @@ -395,11 +424,20 @@ describe('bigquery:tables', function () { var program = getSample().program; program.listTables = sinon.stub(); - program.main(['list', dataset]); + program.main(['tables', dataset]); assert.equal(program.listTables.calledOnce, true); assert.deepEqual(program.listTables.firstCall.args.slice(0, -1), [{ dataset: dataset }]); }); + it('should call listRows', function () { + var program = getSample().program; + program.listRows = sinon.stub(); + + program.main(['rows', dataset, table]); + assert.equal(program.listRows.calledOnce, true); + assert.deepEqual(program.listRows.firstCall.args.slice(0, -1), [dataset, table]); + }); + it('should call deleteTable', function () { var program = getSample().program; program.deleteTable = sinon.stub(); From 720ea9b3d2b94bec4b788241f94ce99050c67992 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Fri, 2 Sep 2016 10:30:08 -0700 Subject: [PATCH 2/5] A few minor tweaks --- bigquery/system-test/tables.test.js | 3 +-- bigquery/test/tables.test.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bigquery/system-test/tables.test.js b/bigquery/system-test/tables.test.js index 075be852e5..4bd90c27e7 100644 --- a/bigquery/system-test/tables.test.js +++ b/bigquery/system-test/tables.test.js @@ -42,10 +42,9 @@ describe('bigquery:tables', function () { // Create bucket storage.createBucket(options.bucket, function (err, bucket) { assert.ifError(err, 'bucket creation succeeded'); - + // Upload data.csv bucket.upload(options.localFilePath, function (err) { assert.ifError(err, 'file upload succeeded'); - // Create dataset bigquery.createDataset(options.dataset, function (err, dataset) { assert.ifError(err, 'dataset creation succeeded'); diff --git a/bigquery/test/tables.test.js b/bigquery/test/tables.test.js index 16c9dc590a..5487f8ccb2 100644 --- a/bigquery/test/tables.test.js +++ b/bigquery/test/tables.test.js @@ -187,7 +187,7 @@ describe('bigquery:tables', function () { }); describe('listRows', function () { - it('should list tables', function () { + it('should list rows', function () { var sample = getSample(); var callback = sinon.stub(); From b0aa11637abdcd31e348e25b08f526a81b12bf90 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Fri, 2 Sep 2016 12:14:40 -0700 Subject: [PATCH 3/5] Fix bad rebase --- bigquery/system-test/tables.test.js | 40 ++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/bigquery/system-test/tables.test.js b/bigquery/system-test/tables.test.js index 4bd90c27e7..5180c9865b 100644 --- a/bigquery/system-test/tables.test.js +++ b/bigquery/system-test/tables.test.js @@ -36,6 +36,10 @@ var options = { schema: 'Name:string, Age:integer, Weight:float, IsMagic:boolean', rows: rows }; +var srcDataset = options.dataset; +var srcTable = options.table; +var destDataset = generateUuid(); +var destTable = generateUuid(); describe('bigquery:tables', function () { before(function (done) { @@ -45,10 +49,14 @@ describe('bigquery:tables', function () { // Upload data.csv bucket.upload(options.localFilePath, function (err) { assert.ifError(err, 'file upload succeeded'); - // Create dataset - bigquery.createDataset(options.dataset, function (err, dataset) { - assert.ifError(err, 'dataset creation succeeded'); - done(); + // Create srcDataset + bigquery.createDataset(srcDataset, function (err) { + assert.ifError(err, 'srcDataset creation succeeded'); + // Create destDataset + bigquery.createDataset(destDataset, function (err) { + assert.ifError(err, 'destDataset creation succeeded'); + done(); + }); }); }); }); @@ -156,6 +164,30 @@ describe('bigquery:tables', function () { }); }); + describe('copyTable', function () { + it('should copy a table between datasets', function (done) { + program.copyTable(srcDataset, srcTable, destDataset, destTable, function (err, metadata) { + assert.equal(err, null); + assert.deepEqual(metadata.status, { state: 'DONE' }); + + bigquery.dataset(srcDataset).table(srcTable).exists( + function (err, exists) { + assert.equal(err, null); + assert.equal(exists, true, 'srcTable exists'); + + bigquery.dataset(destDataset).table(destTable).exists( + function (err, exists) { + assert.equal(err, null); + assert.equal(exists, true, 'destTable exists'); + done(); + } + ); + } + ); + }); + }); + }); + describe('listRows', function () { it('should list rows in a table', function (done) { program.listRows(options.dataset, options.table, function (err, rows) { From ec5e9ae91acd12cdf1879b0bb9f03146ef6cf8d1 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Fri, 2 Sep 2016 13:35:45 -0700 Subject: [PATCH 4/5] Address comments --- bigquery/system-test/tables.test.js | 6 +++--- bigquery/tables.js | 16 ++++++++-------- bigquery/test/tables.test.js | 20 ++++++++++---------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/bigquery/system-test/tables.test.js b/bigquery/system-test/tables.test.js index 5180c9865b..afe12e286c 100644 --- a/bigquery/system-test/tables.test.js +++ b/bigquery/system-test/tables.test.js @@ -188,9 +188,9 @@ describe('bigquery:tables', function () { }); }); - describe('listRows', function () { - it('should list rows in a table', function (done) { - program.listRows(options.dataset, options.table, function (err, rows) { + describe('browseRows', function () { + it('should display rows in a table', function (done) { + program.browseRows(options.dataset, options.table, function (err, rows) { assert.equal(err, null); assert.notEqual(rows, null); assert.equal(Array.isArray(rows), true); diff --git a/bigquery/tables.js b/bigquery/tables.js index df5fde059b..05f2a41287 100644 --- a/bigquery/tables.js +++ b/bigquery/tables.js @@ -80,7 +80,7 @@ function listTables (options, callback) { } // [END list_tables] -function listRows (dataset, table, callback) { +function browseRows (dataset, table, callback) { var bigquery = BigQuery(); var tableObj = bigquery.dataset(dataset).table(table); tableObj.getRows(function (err, rows) { @@ -249,7 +249,7 @@ var fs = require('fs'); var program = module.exports = { createTable: createTable, listTables: listTables, - listRows: listRows, + browseRows: browseRows, deleteTable: deleteTable, importFile: importFile, exportTableToGCS: exportTableToGCS, @@ -266,7 +266,7 @@ cli .command('create
', 'Create a new table in the specified dataset.', {}, function (options) { program.createTable(utils.pick(options, ['dataset', 'table']), utils.makeHandler()); }) - .command('tables ', 'List tables in the specified dataset.', {}, function (options) { + .command('list ', 'List tables in the specified dataset.', {}, function (options) { program.listTables(utils.pick(options, ['dataset']), utils.makeHandler(true, 'id')); }) .command('delete
', 'Delete a table in the specified dataset.', {}, function (options) { @@ -284,8 +284,8 @@ cli ); } ) - .command('rows
', 'List the rows in a BigQuery table.', {}, function (options) { - program.listRows(options.dataset, options.table, utils.makeHandler()); + .command('browse
', 'List the rows in a BigQuery table.', {}, function (options) { + program.browseRows(options.dataset, options.table, utils.makeHandler()); }) .command('import
', 'Import data from a local file or a Google Cloud Storage file into BigQuery.', { bucket: { @@ -345,12 +345,12 @@ cli 'Create table "my_table" in "my_dataset".' ) .example( - 'node $0 tables my_dataset', + 'node $0 list my_dataset', 'List tables in "my_dataset".' ) .example( - 'node $0 rows my_dataset my_table', - 'List rows from "my_table" in "my_dataset".' + 'node $0 browse my_dataset my_table', + 'Display rows from "my_table" in "my_dataset".' ) .example( 'node $0 delete my_dataset my_table', diff --git a/bigquery/test/tables.test.js b/bigquery/test/tables.test.js index 5487f8ccb2..51ec1abbed 100644 --- a/bigquery/test/tables.test.js +++ b/bigquery/test/tables.test.js @@ -186,12 +186,12 @@ describe('bigquery:tables', function () { }); }); - describe('listRows', function () { - it('should list rows', function () { + describe('browseRows', function () { + it('should display rows', function () { var sample = getSample(); var callback = sinon.stub(); - sample.program.listRows(dataset, table, callback); + sample.program.browseRows(dataset, table, callback); assert.equal(sample.mocks.table.getRows.calledOnce, true); assert.deepEqual(sample.mocks.table.getRows.firstCall.args.slice(0, -1), []); @@ -207,7 +207,7 @@ describe('bigquery:tables', function () { var callback = sinon.stub(); sample.mocks.table.getRows.yields(error); - sample.program.listRows(dataset, table, callback); + sample.program.browseRows(dataset, table, callback); assert.equal(callback.calledOnce, true); assert.deepEqual(callback.firstCall.args, [error]); @@ -424,18 +424,18 @@ describe('bigquery:tables', function () { var program = getSample().program; program.listTables = sinon.stub(); - program.main(['tables', dataset]); + program.main(['list', dataset]); assert.equal(program.listTables.calledOnce, true); assert.deepEqual(program.listTables.firstCall.args.slice(0, -1), [{ dataset: dataset }]); }); - it('should call listRows', function () { + it('should call browseRows', function () { var program = getSample().program; - program.listRows = sinon.stub(); + program.browseRows = sinon.stub(); - program.main(['rows', dataset, table]); - assert.equal(program.listRows.calledOnce, true); - assert.deepEqual(program.listRows.firstCall.args.slice(0, -1), [dataset, table]); + program.main(['browse', dataset, table]); + assert.equal(program.browseRows.calledOnce, true); + assert.deepEqual(program.browseRows.firstCall.args.slice(0, -1), [dataset, table]); }); it('should call deleteTable', function () { From 83ebd51f2c8bdc8095b8956d49f0818d436032e3 Mon Sep 17 00:00:00 2001 From: Ace Nassri Date: Fri, 2 Sep 2016 14:15:05 -0700 Subject: [PATCH 5/5] Address comments --- bigquery/system-test/tables.test.js | 26 ++++++++++++++------------ bigquery/tables.js | 3 +++ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/bigquery/system-test/tables.test.js b/bigquery/system-test/tables.test.js index afe12e286c..b42f2267f7 100644 --- a/bigquery/system-test/tables.test.js +++ b/bigquery/system-test/tables.test.js @@ -63,17 +63,20 @@ describe('bigquery:tables', function () { }); after(function (done) { - // Delete testing dataset/table - bigquery.dataset(options.dataset).delete({ force: true }, function () { - // Delete files - storage.bucket(options.bucket).deleteFiles({ force: true }, function (err) { - if (err) { - return done(err); - } - // Delete bucket - setTimeout(function () { - storage.bucket(options.bucket).delete(done); - }, 2000); + // Delete srcDataset + bigquery.dataset(srcDataset).delete({ force: true }, function () { + // Delete destDataset + bigquery.dataset(destDataset).delete({ force: true }, function () { + // Delete files + storage.bucket(options.bucket).deleteFiles({ force: true }, function (err) { + if (err) { + return done(err); + } + // Delete bucket + setTimeout(function () { + storage.bucket(options.bucket).delete(done); + }, 2000); + }); }); }); }); @@ -192,7 +195,6 @@ describe('bigquery:tables', function () { it('should display rows in a table', function (done) { program.browseRows(options.dataset, options.table, function (err, rows) { assert.equal(err, null); - assert.notEqual(rows, null); assert.equal(Array.isArray(rows), true); assert.equal(rows.length > 0, true); assert.equal(console.log.calledOnce, true); diff --git a/bigquery/tables.js b/bigquery/tables.js index 05f2a41287..694c0fc640 100644 --- a/bigquery/tables.js +++ b/bigquery/tables.js @@ -83,6 +83,8 @@ function listTables (options, callback) { function browseRows (dataset, table, callback) { var bigquery = BigQuery(); var tableObj = bigquery.dataset(dataset).table(table); + + // See https://googlecloudplatform.github.io/google-cloud-node/#/docs/bigquery/latest/bigquery/table?method=getRows tableObj.getRows(function (err, rows) { if (err) { return callback(err); @@ -123,6 +125,7 @@ function copyTable (srcDataset, srcTable, destDataset, destTable, callback) { var srcTableObj = bigquery.dataset(srcDataset).table(srcTable); var destTableObj = bigquery.dataset(destDataset).table(destTable); + // See https://googlecloudplatform.github.io/google-cloud-node/#/docs/bigquery/latest/bigquery/table?method=copy srcTableObj.copy(destTableObj, function (err, job) { if (err) { return callback(err);