Update tumugi to 0.6

examples/copy.rb

@@ -1,21 +1,21 @@
 task :task1, type: :bigquery_copy do
-  param_set :src_project_id, ->{ input.project_id }
-  param_set :src_dataset_id, ->{ input.dataset_id }
-  param_set :src_table_id, ->{ input.table_id }
-  param_set :dest_dataset_id, "test"
-  param_set :dest_table_id, ->{ "dest_table_#{Time.now.to_i}" }
+  src_project_id  { input.project_id }
+  src_dataset_id  { input.dataset_id }
+  src_table_id    { input.table_id }
+  dest_dataset_id "test"
+  dest_table_id   { "dest_table_#{Time.now.to_i}" }
 
   requires :task2
 end
 
 task :task2, type: :bigquery_query do
-  param_set :query, "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
-  param_set :dataset_id, "test" #->{ input.dataset_id }
-  param_set :table_id, "dest_#{Time.now.to_i}"
+  query      "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
+  dataset_id { input.dataset_id }
+  table_id   "dest_#{Time.now.to_i}"
 
   requires :task3
 end
 
 task :task3, type: :bigquery_dataset do
-  param_set :dataset_id, "test"
+  dataset_id "test"
 end

examples/dataset.rb

@@ -6,5 +6,5 @@
 end
 
 task :task2, type: :bigquery_dataset do
-  param_set :dataset_id, 'test'
+  dataset_id "test"
 end

examples/export.rb

@@ -0,0 +1,13 @@
+task :task1, type: :bigquery_export do
+  dataset_id  { input.dataset_id }
+  table_id    { input.table_id }
+
+  requires :task2
+  output target(:local_file, "tmp/export.csv")
+end
+
+task :task2, type: :bigquery_query do
+  query      "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
+  dataset_id "test"
+  table_id   "dest_#{Time.now.to_i}"
+end

examples/force_copy.rb

@@ -0,0 +1,22 @@
+task :task1, type: :bigquery_copy do
+  src_project_id  { input.project_id }
+  src_dataset_id  { input.dataset_id }
+  src_table_id    { input.table_id }
+  dest_dataset_id "test"
+  dest_table_id   "dest_table_1"
+  force_copy      true
+
+  requires :task2
+end
+
+task :task2, type: :bigquery_query do
+  query      "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
+  dataset_id { input.dataset_id }
+  table_id   "dest_#{Time.now.to_i}"
+
+  requires :task3
+end
+
+task :task3, type: :bigquery_dataset do
+  dataset_id "test"
+end

examples/load.rb

@@ -1,11 +1,11 @@
 task :task1, type: :bigquery_load do
   requires :task2
-  param_set :bucket, 'tumugi-plugin-bigquery'
-  param_set :key, 'test.csv'
-  param_set :dataset_id, -> { input.dataset_id }
-  param_set :table_id, 'load_test'
-  param_set :skip_leading_rows, 1
-  param_set :schema, [
+  bucket 'tumugi-plugin-bigquery'
+  key 'test.csv'
+  dataset_id { input.dataset_id }
+  table_id 'load_test'
+  skip_leading_rows 1
+  schema [
     {
       name: 'row_number',
       type: 'INTEGER',
@@ -20,5 +20,5 @@
 end
 
 task :task2, type: :bigquery_dataset do
-  param_set :dataset_id, 'test'
+  dataset_id "test"
 end

examples/query.rb

@@ -6,7 +6,7 @@
 end
 
 task :task2, type: :bigquery_query do
-  param_set :query, "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
-  param_set :dataset_id, 'test'
-  param_set :table_id, "dest_#{Time.now.to_i}"
+  query      "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
+  dataset_id "test"
+  table_id   "dest_#{Time.now.to_i}"
 end

examples/query_append.rb

@@ -6,8 +6,8 @@
 end
 
 task :task2, type: :bigquery_query do
-  param_set :query, "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
-  param_set :dataset_id, "test"
-  param_set :table_id, "dest_append"
-  param_set :mode, "append"
+  query      "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
+  dataset_id "test"
+  table_id   "dest_append"
+  mode       "append"
 end

lib/tumugi/plugin/task/bigquery_copy.rb

@@ -13,7 +13,7 @@ class BigqueryCopyTask < Tumugi::Task
       param :dest_dataset_id, type: :string, required: true
       param :dest_table_id, type: :string, required: true
       param :force_copy, type: :bool, default: false
-      param :wait, type: :int, default: 60
+      param :wait, type: :integer, default: 60
 
       def output
         return @output if @output
@@ -24,7 +24,7 @@ def output
       end
 
       def completed?
-        if force_copy && @state != :completed
+        if force_copy && !finished?
           false
         else
           super

lib/tumugi/plugin/task/bigquery_query.rb

@@ -13,15 +13,14 @@ class BigqueryQueryTask < Tumugi::Task
       param :mode, type: :string, default: 'truncate' # append, empty
       param :flatten_results, type: :bool, default: true
       param :use_legacy_sql, type: :bool, default: true
-
-      param :wait, type: :int, default: 60
+      param :wait, type: :integer, default: 60
 
       def output
         @output ||= Tumugi::Plugin::BigqueryTableTarget.new(project_id: project_id, dataset_id: dataset_id, table_id: table_id)
       end
 
       def completed?
-        if mode.to_sym == :append && @state != :completed
+        if mode.to_sym == :append && !finished?
           false
         else
           super

test/cli_test.rb

@@ -1,20 +1,30 @@
 require_relative './test_helper'
 require 'tumugi/cli'
 
-class Tumugi::Plugin::Bigquery::CLITest < Test::Unit::TestCase
+class Tumugi::Plugin::Bigquery::CLITest < Tumugi::Test::TumugiTestCase
   examples = {
     'copy' => ['copy.rb', 'task1'],
     'dataset' => ['dataset.rb', 'task1'],
+    'export' => ['export.rb', 'task1'],
+    'force_copy' => ['force_copy.rb', 'task1'],
     'query' => ['query.rb', 'task1'],
     'query_append' => ['query_append.rb', 'task1'],
   }
 
-  def invoke(file, task, options)
-    Tumugi::CLI.new.invoke(:run_, [task], options.merge(file: "./examples/#{file}", quiet: true))
+  setup do
+    system('rm -rf tmp/*')
   end
 
-  data(examples)
-  test 'success' do |(file, task)|
-    assert_true(invoke(file, task, worker: 4, config: "./examples/tumugi_config_example.rb"))
+  data do
+    data_set = {}
+    examples.each do |k, v|
+      [1, 2, 8].each do |n|
+        data_set["#{k}_workers_#{n}"] = (v.dup << n)
+      end
+    end
+    data_set
+  end
+  test 'success' do |(file, task, worker)|
+    assert_run_success("examples/#{file}", task, workers: worker, config: "./examples/tumugi_config_example.rb")
   end
 end

test/plugin/task/bigquery_copy_test.rb

@@ -14,11 +14,11 @@ def exist?; false; end
 
   setup do
     @klass = Class.new(Tumugi::Plugin::BigqueryCopyTask)
-    @klass.param_set :src_project_id, 'publicdata'
-    @klass.param_set :src_dataset_id, 'samples'
-    @klass.param_set :src_table_id, 'shakespeare'
-    @klass.param_set :dest_dataset_id, Tumugi::Plugin::BigqueryTestHelper::TEST_DATASETS[0]
-    @klass.param_set :dest_table_id, 'test'
+    @klass.set :src_project_id, 'publicdata'
+    @klass.set :src_dataset_id, 'samples'
+    @klass.set :src_table_id, 'shakespeare'
+    @klass.set :dest_dataset_id, Tumugi::Plugin::BigqueryTestHelper::TEST_DATASETS[0]
+    @klass.set :dest_table_id, 'test'
   end
 
   sub_test_case "parameters" do
@@ -42,7 +42,7 @@ def exist?; false; end
     })
     test "raise error when required parameter is not set" do |params|
       params.each do |param|
-        @klass.param_set(param, nil)
+        @klass.set(param, nil)
       end
       assert_raise(Tumugi::ParameterError) do
         @klass.new
@@ -68,12 +68,12 @@ def exist?; false; end
     "force_copy is true with completed state and not exist target" => [true, NotExistTarget, :completed, false],
   })
   test "#complted?" do |(force_copy, target_klass, state, expected)|
-    @klass.param_set :force_copy, force_copy
+    @klass.set :force_copy, force_copy
     @klass.send(:define_method, :output) do
       target_klass.new
     end
     task = @klass.new
-    task.state = state
+    task.instance_variable_set(:@state, state)
     assert_equal(expected, task.completed?)
   end
 

test/plugin/task/bigquery_dataset_test.rb

@@ -6,8 +6,8 @@ class Tumugi::Plugin::BigqueryDatasetTaskTest < Test::Unit::TestCase
 
   setup do
     @klass = Class.new(Tumugi::Plugin::BigqueryDatasetTask)
-    @klass.param_set :dataset_id, Tumugi::Plugin::BigqueryTestHelper::TEST_DATASETS[0]
-    @klass.param_set :project_id, ENV['PROJECT_ID']
+    @klass.set :dataset_id, Tumugi::Plugin::BigqueryTestHelper::TEST_DATASETS[0]
+    @klass.set :project_id, ENV['PROJECT_ID']
   end
 
   sub_test_case "parameters" do
@@ -22,7 +22,7 @@ class Tumugi::Plugin::BigqueryDatasetTaskTest < Test::Unit::TestCase
     })
     test "raise error when required parameter is not set" do |params|
       params.each do |param|
-        @klass.param_set(param, nil)
+        @klass.set(param, nil)
       end
       assert_raise(Tumugi::ParameterError) do
         @klass.new

test/plugin/task/bigquery_export_test.rb

@@ -8,11 +8,11 @@ class Tumugi::Plugin::BigqueryExportTaskTest < Test::Unit::TestCase
 
   setup do
     @klass = Class.new(Tumugi::Plugin::BigqueryExportTask)
-    @klass.param_set :project_id, 'bigquery-public-data'
-    @klass.param_set :job_project_id, 'tumugi-plugin-bigquery'
-    @klass.param_set :dataset_id, 'samples'
-    @klass.param_set :table_id, 'shakespeare'
-    @klass.param_set :compression, 'GZIP'
+    @klass.set :project_id, 'bigquery-public-data'
+    @klass.set :job_project_id, 'tumugi-plugin-bigquery'
+    @klass.set :dataset_id, 'samples'
+    @klass.set :table_id, 'shakespeare'
+    @klass.set :compression, 'GZIP'
   end
 
   sub_test_case "parameters" do
@@ -33,7 +33,7 @@ class Tumugi::Plugin::BigqueryExportTaskTest < Test::Unit::TestCase
     })
     test "raise error when required parameter is not set" do |params|
       params.each do |param|
-        @klass.param_set(param, nil)
+        @klass.set(param, nil)
       end
       assert_raise(Tumugi::ParameterError) do
         @klass.new
@@ -112,7 +112,7 @@ def output
   end
 
   test "raise error if format is AVRO and not to export Google Cloud Storage" do
-    @klass.param_set :destination_format, 'AVRO'
+    @klass.set :destination_format, 'AVRO'
     task = @klass.new
     task.instance_eval do
       def output

test/plugin/task/bigquery_load_test.rb

@@ -6,12 +6,12 @@ class Tumugi::Plugin::BigqueryLoadTaskTest < Test::Unit::TestCase
 
   setup do
     @klass = Class.new(Tumugi::Plugin::BigqueryLoadTask)
-    @klass.param_set :bucket, 'tumugi-plugin-bigquery'
-    @klass.param_set :key, 'test.csv'
-    @klass.param_set :dataset_id, Tumugi::Plugin::BigqueryTestHelper::TEST_DATASETS[0]
-    @klass.param_set :table_id, 'load_test'
-    @klass.param_set :skip_leading_rows, 1
-    @klass.param_set :schema, [
+    @klass.set :bucket, 'tumugi-plugin-bigquery'
+    @klass.set :key, 'test.csv'
+    @klass.set :dataset_id, Tumugi::Plugin::BigqueryTestHelper::TEST_DATASETS[0]
+    @klass.set :table_id, 'load_test'
+    @klass.set :skip_leading_rows, 1
+    @klass.set :schema, [
       {
         name: 'row_number',
         type: 'INTEGER',
@@ -44,7 +44,7 @@ class Tumugi::Plugin::BigqueryLoadTaskTest < Test::Unit::TestCase
     })
     test "raise error when required parameter is not set" do |params|
       params.each do |param|
-        @klass.param_set(param, nil)
+        @klass.set(param, nil)
       end
       assert_raise(Tumugi::ParameterError) do
         @klass.new

test/plugin/task/bigquery_query_test.rb

@@ -14,10 +14,10 @@ def exist?; false; end
 
   setup do
     @klass = Class.new(Tumugi::Plugin::BigqueryQueryTask)
-    @klass.param_set :query, "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
-    @klass.param_set :dataset_id, Tumugi::Plugin::BigqueryTestHelper::TEST_DATASETS[0]
-    @klass.param_set :table_id, 'test'
-    @klass.param_set :project_id, ENV['PROJECT_ID']
+    @klass.set :query, "SELECT COUNT(*) AS cnt FROM [bigquery-public-data:samples.wikipedia]"
+    @klass.set :dataset_id, Tumugi::Plugin::BigqueryTestHelper::TEST_DATASETS[0]
+    @klass.set :table_id, 'test'
+    @klass.set :project_id, ENV['PROJECT_ID']
   end
 
   sub_test_case "parameters" do
@@ -40,7 +40,7 @@ def exist?; false; end
     })
     test "raise error when required parameter is not set" do |params|
       params.each do |param|
-        @klass.param_set(param, nil)
+        @klass.set(param, nil)
       end
       assert_raise(Tumugi::ParameterError) do
         @klass.new
@@ -57,12 +57,12 @@ def exist?; false; end
     "append mode with completed state and not exist target" => ["append", NotExistTarget, :completed, false],
   })
   test "#completed?" do |(mode, target_klass, state, expected)|
-    @klass.param_set :mode, mode
+    @klass.set :mode, mode
     @klass.send(:define_method, :output) do
       target_klass.new
     end
     task = @klass.new
-    task.state = state
+    task.instance_variable_set(:@state, state)
     assert_equal(expected, task.completed?)
   end
 
@@ -84,8 +84,8 @@ def exist?; false; end
   end
 
   test "#run with append mode" do
-    @klass.param_set :table_id, 'test_append'
-    @klass.param_set :mode, 'append'
+    @klass.set :table_id, 'test_append'
+    @klass.set :mode, 'append'
     task = @klass.new
     output = task.output
     task.run

test/test_helper.rb

@@ -11,6 +11,9 @@
 require 'test/unit/rr'
 
 require 'tumugi'
+require 'tumugi/test/helper'
+include Tumugi::Test::Helpers
+
 require 'kura'
 
 Dir.mkdir('tmp') unless Dir.exist?('tmp')

tumugi-plugin-bigquery.gemspec

@@ -20,14 +20,15 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_runtime_dependency "tumugi", ">= 0.5.1"
+  spec.add_runtime_dependency "tumugi", ">= 0.6.1"
   spec.add_runtime_dependency "kura", "~> 0.2.17"
+  spec.add_runtime_dependency "google-api-client", "0.9.9"
+  spec.add_runtime_dependency "json", "~> 1.8.3" # json 2.0 does not work with JRuby + MultiJson
 
   spec.add_development_dependency 'bundler', '~> 1.11'
   spec.add_development_dependency 'rake', '~> 10.0'
   spec.add_development_dependency 'test-unit', '~> 3.1'
   spec.add_development_dependency 'test-unit-rr'
   spec.add_development_dependency 'coveralls'
-  spec.add_development_dependency 'github_changelog_generator'
   spec.add_development_dependency 'tumugi-plugin-google_cloud_storage'
 end