First implementation

.travis.yml

@@ -1,13 +1,8 @@
 language: ruby
 cache: bundler
 rvm:
-  - 2.1.10
-  - 2.2.5
+  - 2.1
   - 2.3.1
-  - ruby-head
-  - jruby-9.0.5.0
+  - jruby-9.1.2.0
 before_install:
   - gem install bundler
-matrix:
-  allow_failures:
-    - rvm: ruby-head

README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/tumugi/tumugi-plugin-google_drive.svg?branch=master)](https://travis-ci.org/tumugi/tumugi-plugin-google_drive) [![Code Climate](https://codeclimate.com/github/tumugi/tumugi-plugin-google_drive/badges/gpa.svg)](https://codeclimate.com/github/tumugi/tumugi-plugin-google_drive) [![Coverage Status](https://coveralls.io/repos/github/tumugi/tumugi-plugin-google_drive/badge.svg?branch=master)](https://coveralls.io/github/tumugi/tumugi-plugin-google_drive?branch=master) [![Gem Version](https://badge.fury.io/rb/tumugi-plugin-google_drive.svg)](https://badge.fury.io/rb/tumugi-plugin-google_drive)
+
 # tumugi-plugin-google_drive
 
 [tumugi](https://github.com/tumugi/tumugi) plugin for Google Drive.
@@ -27,6 +29,55 @@ $ gem install tumugi-plugin-google_drive
 ### Tumugi::Plugin::GoogleDriveFileTarget
 
 This target represent file on Googl Drive.
+This target has following parameters.
+
+- name
+  - Filename **string**
+- parents
+  - Parent folder ID **string** or **array of string**
+
+Tumugi workflow file using this target is like this:
+
+```rb
+task :task1 do
+  param :day, type: :time, auto_bind: true, required: true
+  output do
+    target(:google_drive_file,
+          name: "test_#{day.strftime('%Y%m%d')}.txt",
+          parents: "xyz")
+  end
+  run do
+    log 'task1#run'
+    output.open('w') {|f| f.puts('done') }
+  end
+end
+```
+
+### Config Section
+
+tumugi-plugin-google_drive provide config section named "google_drive" which can specified Google Drive autenticaion info.
+
+#### Authenticate by client_email and private_key
+
+```rb
+Tumugi.config do |config|
+  config.section("google_drive") do |section|
+    section.project_id = "xxx"
+    section.client_email = "[email protected]"
+    section.private_key = "zzz"
+  end
+end
+```
+
+#### Authenticate by JSON key file
+
+```rb
+Tumugi.configure do |config|
+  config.section("google_drive") do |section|
+    section.private_key_file = "/path/to/key.json"
+  end
+end
+```
 
 ## Development
 

examples/example.rb

@@ -0,0 +1,12 @@
+task :task1 do
+  param :day, type: :time, auto_bind: true, required: true
+  output do
+    target(:google_drive_file,
+          name: "test_#{day.strftime('%Y%m%d')}.txt",
+          parents: "xyz")
+  end
+  run do
+    log 'task1#run'
+    output.open('w') {|f| f.puts('done') }
+  end
+end

examples/tumgui_config_example.rb

@@ -0,0 +1,7 @@
+Tumugi.configure do |config|
+  config.section('google_drive') do |section|
+    section.project_id = ENV["PROJECT_ID"]
+    section.client_email = ENV["CLIENT_EMAIL"]
+    section.private_key = ENV["PRIVATE_KEY"].gsub(/\\n/, "\n")
+  end
+end

lib/tumugi/plugin/google_drive/atomic_file.rb

@@ -0,0 +1,22 @@
+require 'tumugi/atomic_file'
+
+module Tumugi
+  module Plugin
+    module GoogleDrive
+      class AtomicFile < Tumugi::AtomicFile
+        attr_reader :id
+
+        def initialize(path, fs, file_id: nil, parents: nil)
+          super(path)
+          @fs = fs
+          @parents = parents
+          @id = (file_id.nil? ? @fs.generate_file_id : file_id)
+        end
+
+        def move_to_final_destination(temp_file)
+          @fs.upload(temp_file, path, file_id: @id, parents: @parents)
+        end
+      end
+    end
+  end
+end

lib/tumugi/plugin/google_drive/file_system.rb

@@ -0,0 +1,190 @@
+require 'google/apis/drive_v3'
+require 'tumugi/error'
+
+module Tumugi
+  module Plugin
+    module GoogleDrive
+      class FileSystem
+
+        MIME_TYPE_FOLDER = 'application/vnd.google-apps.folder'
+
+        def initialize(config)
+          save_config(config)
+        end
+
+        def exist?(file_id)
+          client.get_file(file_id, options: request_options)
+          true
+        rescue => e
+          return false if e.respond_to?(:status_code) && e.status_code == 404
+          process_error(e)
+        end
+
+        def remove(file_id)
+          return unless exist?(file_id)
+
+          client.delete_file(file_id, options: request_options)
+          wait_until { !exist?(file_id) }
+          file_id
+        rescue
+          process_error($!)
+        end
+
+        def mkdir(name)
+          file_metadata = Google::Apis::DriveV3::File.new({
+            name: name,
+            mime_type: MIME_TYPE_FOLDER
+          })
+          file = client.create_file(file_metadata, options: request_options)
+          wait_until { exist?(file.id) }
+          file
+        rescue
+          process_error($!)
+        end
+
+        def directory?(file_id)
+          file = client.get_file(file_id, options: request_options)
+          file.mime_type == MIME_TYPE_FOLDER
+        rescue
+          process_error($!)
+        end
+
+        def move(src_file_id, dest_name, dest_parents: nil)
+          file = copy(src_file_id, dest_name, dest_parents: dest_parents)
+          remove(src_file_id)
+          file
+        end
+
+        def copy(src_file_id, dest_name, dest_parents: nil)
+          dest_parents = [dest_parents] if dest_parents.is_a?(String)
+          dest_file_metadata = Google::Apis::DriveV3::File.new({
+            name: dest_name,
+            parents: dest_parents
+          })
+          file = client.copy_file(src_file_id, dest_file_metadata, options: request_options)
+          wait_until { exist?(file.id) }
+          file
+        rescue
+          process_error($!)
+        end
+
+        def upload(media, name, content_type: nil, file_id: nil, parents: nil)
+          parents = [parents] if parents.is_a?(String)
+          file_metadata = Google::Apis::DriveV3::File.new({
+            id: file_id,
+            name: name,
+            parents: parents
+          })
+          file = client.create_file(file_metadata, upload_source: media, content_type: content_type, options: request_options)
+          wait_until { exist?(file.id) }
+          file
+        rescue
+          process_error($!)
+        end
+
+        def put_string(contents, name, content_type: 'text/plain', file_id: nil, parents: nil)
+          media = StringIO.new(contents)
+          upload(media, name, content_type: content_type, file_id: file_id, parents: parents)
+        end
+
+        def download(file_id, download_path: nil, mode: 'r', &block)
+          if download_path.nil?
+            download_path = Tempfile.new('tumugi_google_drive_file_system').path
+          end
+          client.get_file(file_id, download_dest: download_path, options: request_options)
+          wait_until { File.exist?(download_path) }
+
+          if block_given?
+            File.open(download_path, mode, &block)
+          else
+            File.open(download_path, mode)
+          end
+        rescue
+          process_error($!)
+        end
+
+        def generate_file_id
+          client.generate_file_ids(count: 1, options: request_options).ids.first
+        rescue
+          process_error($!)
+        end
+
+        private
+
+        def save_config(config)
+          if config.private_key_file.nil?
+            @project_id = config.project_id
+            client_email = config.client_email
+            private_key = config.private_key
+          elsif config.private_key_file
+            json = JSON.parse(File.read(config.private_key_file))
+            @project_id = json['project_id']
+            client_email = json['client_email']
+            private_key = json['private_key']
+          end
+          @key = {
+            client_email: client_email,
+            private_key: private_key
+          }
+        end
+
+        def client
+          return @cached_client if @cached_client && @cached_client_expiration > Time.now
+
+          client = Google::Apis::DriveV3::DriveService.new
+          scope = Google::Apis::DriveV3::AUTH_DRIVE
+
+          if @key[:client_email] && @key[:private_key]
+            options = {
+              json_key_io: StringIO.new(JSON.generate(@key)),
+              scope: scope
+            }
+            auth = Google::Auth::ServiceAccountCredentials.make_creds(options)
+          else
+            auth = Google::Auth.get_application_default([scope])
+          end
+          auth.fetch_access_token!
+          client.authorization = auth
+
+          @cached_client_expiration = Time.now + (auth.expires_in / 2)
+          @cached_client = client
+        end
+
+        def wait_until(&block)
+          while not block.call
+            sleep 3
+          end
+        end
+
+        def process_error(err)
+          if err.respond_to?(:body)
+            begin
+              if err.body.nil?
+                reason = err.status_code.to_s
+                errors = "HTTP Status: #{err.status_code}\nHeaders: #{err.header.inspect}"
+              else
+                jobj = JSON.parse(err.body)
+                error = jobj["error"]
+                reason = error["errors"].map{|e| e["reason"]}.join(",")
+                errors = error["errors"].map{|e| e["message"] }.join("\n")
+              end
+            rescue JSON::ParserError
+              reason = err.status_code.to_s
+              errors = "HTTP Status: #{err.status_code}\nHeaders: #{err.header.inspect}\nBody:\n#{err.body}"
+            end
+            raise Tumugi::FileSystemError.new(errors, reason)
+          else
+            raise err
+          end
+        end
+
+        def request_options
+          {
+            retries: 5,
+            timeout_sec: 60
+          }
+        end
+      end
+    end
+  end
+end

lib/tumugi/plugin/target/google_drive_file.rb

@@ -1,32 +1,46 @@
 require 'tumugi/config'
 require 'tumugi/plugin'
-#require 'tumugi/plugin/gcs/atomic_gcs_file'
-#require 'tumugi/plugin/gcs/gcs_file_system'
+require 'tumugi/plugin/google_drive/atomic_file'
+require 'tumugi/plugin/google_drive/file_system'
 
 module Tumugi
   module Plugin
     class GoogleDriveFileTarget < Tumugi::Target
       Tumugi::Plugin.register_target('google_drive_file', self)
-      Tumugi::Config.register_section('google_drive', :client_id, :client_secret, :refresh_token)
+      Tumugi::Config.register_section('google_drive', :project_id, :client_email, :private_key, :private_key_file)
 
-      attr_reader :key, :folder
+      attr_reader :file_id, :name, :parents
 
-      def initialize(key:, folder: nil)
-        @key = key
-        @folder = folder
-        log "key='#{key}, folder='#{folder}'"
+      def initialize(file_id: nil, name: nil, parents: nil, fs: nil)
+        @fs = fs unless fs.nil?
+        @file_id = file_id || self.fs.generate_file_id
+        @name = name
+        @parents = parents
+        log "file_id='#{file_id}', name='#{name}', parents='#{parents}'"
       end
 
-      def exist?
-        true #TODO
+      def fs
+        @fs ||= Tumugi::Plugin::GoogleDrive::FileSystem.new(Tumugi.config.section('google_drive'))
       end
 
-      def uri
-        "https://drive.google.com/open?id=#{key}"
+      def open(mode="r", &block)
+        if mode.include? 'r'
+          fs.download(file_id, mode: mode, &block)
+        elsif mode.include? 'w'
+          file = Tumugi::Plugin::GoogleDrive::AtomicFile.new(name, fs)
+          file.open(&block)
+          @file_id = file.id
+        else
+          raise Tumugi::TumugiError.new('Invalid mode: #{mode}')
+        end
       end
 
-      def to_s
-        uri
+      def exist?
+        if file_id.nil?
+          false
+        else
+          fs.exist?(file_id)
+        end
       end
     end
   end

test/plugin/google_drive/atomic_file_test.rb

@@ -0,0 +1,21 @@
+require_relative '../../test_helper'
+require 'tumugi/plugin/google_drive/atomic_file'
+require 'tumugi/plugin/google_drive/file_system'
+
+class Tumugi::Plugin::GoogleDrive::AtomicFileTest < Test::Unit::TestCase
+  setup do
+    @fs = Tumugi::Plugin::GoogleDrive::FileSystem.new(credential)
+  end
+
+  test "after open and close file, file upload to Google Drive" do
+    path = 'test.txt'
+    file = Tumugi::Plugin::GoogleDrive::AtomicFile.new(path, @fs)
+    file.open do |f|
+      f.puts 'test'
+    end
+    @fs.exist?(file.id)
+    @fs.download(file.id) do |f|
+      assert_equal("test\n", f.read)
+    end
+  end
+end