From bc154d8c96bf1a9428395a75d422fc0dccaa5ca5 Mon Sep 17 00:00:00 2001 From: Osamu Takiya Date: Mon, 12 Jun 2023 21:26:12 +0900 Subject: [PATCH] =?UTF-8?q?Star=20=E3=83=A2=E3=83=87=E3=83=AB=E3=82=92?= =?UTF-8?q?=E4=BD=9C=E6=88=90=E3=81=97=E3=80=81=E3=82=A4=E3=83=B3=E3=83=9D?= =?UTF-8?q?=E3=83=BC=E3=82=BF=E3=82=82=E4=BD=9C=E6=88=90=E3=81=97=E3=81=9F?= =?UTF-8?q?=20(#29)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add Star model and its migration & test * Add spreadsheet configuration file * Add SpreadsheetService API with managers * Add spreadsheet credentials file path * Add data import service for Star model * Add star table with name, name_en, seating_order * Refactor ImportService::Base class and IndexesThe commit refactors the ImportService::Base class and removes name_en index from the schema * Update database.yml for development and test environments * Add default value for SPREADSHEET_CREDENTIALS_FILEPATH --- .env.tpl | 5 ++ .gitignore | 2 + app/models/star.rb | 2 + app/service/fetch_data_table_service/base.rb | 19 +++++++ .../from_spreadsheet.rb | 49 +++++++++++++++++++ app/service/import_service/base.rb | 21 ++++++++ app/service/import_service/star.rb | 28 +++++++++++ app/service/import_service/title.rb | 20 -------- app/service/spreadsheet_service.rb | 37 -------------- app/service/spreadsheet_service/sheet_id.rb | 16 ++++++ app/service/spreadsheet_service/sheets_api.rb | 28 +++++++++++ .../spreadsheet_manager.rb | 10 ++++ .../spreadsheet_service/worksheet_manager.rb | 36 ++++++++++++++ config/database.yml | 30 ++++++------ config/spreadsheet/.keep | 0 db/migrate/20230612121238_create_stars.rb | 14 ++++++ db/schema.rb | 12 ++++- lib/tasks/db/reconstruction.rake | 6 ++- spec/factories/stars.rb | 5 ++ spec/models/star_spec.rb | 5 ++ 20 files changed, 271 insertions(+), 74 deletions(-) create mode 100644 app/models/star.rb create mode 100644 app/service/fetch_data_table_service/base.rb create mode 100644 app/service/fetch_data_table_service/from_spreadsheet.rb create mode 100644 app/service/import_service/base.rb create mode 100644 app/service/import_service/star.rb delete mode 100644 app/service/import_service/title.rb delete mode 100644 app/service/spreadsheet_service.rb create mode 100644 app/service/spreadsheet_service/sheet_id.rb create mode 100644 app/service/spreadsheet_service/sheets_api.rb create mode 100644 app/service/spreadsheet_service/spreadsheet_manager.rb create mode 100644 app/service/spreadsheet_service/worksheet_manager.rb create mode 100644 config/spreadsheet/.keep create mode 100644 db/migrate/20230612121238_create_stars.rb create mode 100644 spec/factories/stars.rb create mode 100644 spec/models/star_spec.rb diff --git a/.env.tpl b/.env.tpl index a47c18d..10bfb3e 100644 --- a/.env.tpl +++ b/.env.tpl @@ -1,5 +1,6 @@ # $ op inject -i .env.tpl -o .env RAILS_MASTER_KEY=op://Personal/b7nzrxcuyvy37cafgu3octmoyq/add more/suikoden_vault +SPREADSHEET_CREDENTIALS_FILEPATH=config/spreadsheet/creds.json PG_DEVELOPMENT_HOST=op://Personal/7osscqrsds2jmobocq2twd5pjq/yl3pacrx662v4yd7vxmhypw52i PG_DEVELOPMENT_PORT=op://Personal/7osscqrsds2jmobocq2twd5pjq/port @@ -18,3 +19,7 @@ PG_PRODUCTION_PASS=op://Personal/irwrv7xiofnmimbcdz7d3wclqi/password PG_SSLCERT_PATH= PG_SSLKEY_PATH= PG_SSLROOT_PATH= + +# スプレッドシートID +SPREADSHEET_ID_BASIC_ATTRIBUTES=op://Personal/nouerwwfqtd67aeagpyrug3kiq/fudicawf773buqbjztwh3xzmeu/basic_attributes +SPREADSHEET_ID_PRODUCTS=op://Personal/nouerwwfqtd67aeagpyrug3kiq/fudicawf773buqbjztwh3xzmeu/products diff --git a/.gitignore b/.gitignore index c5ce5c8..0ef5689 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,8 @@ config/postgresql/client.crt config/postgresql/client.key config/postgresql/root.crt +config/spreadsheet/creds.json + # デフォルト項目 /.bundle vendor/ diff --git a/app/models/star.rb b/app/models/star.rb new file mode 100644 index 0000000..63ea001 --- /dev/null +++ b/app/models/star.rb @@ -0,0 +1,2 @@ +class Star < ApplicationRecord +end diff --git a/app/service/fetch_data_table_service/base.rb b/app/service/fetch_data_table_service/base.rb new file mode 100644 index 0000000..cb3a831 --- /dev/null +++ b/app/service/fetch_data_table_service/base.rb @@ -0,0 +1,19 @@ +module FetchDataTableService + class Base + def initialize(*args) + # TODO: 書く + end + + def headers + raise NotImplementedError + end + + def rows + raise NotImplementedError + end + + def header_to_rows + raise NotImplementedError + end + end +end diff --git a/app/service/fetch_data_table_service/from_spreadsheet.rb b/app/service/fetch_data_table_service/from_spreadsheet.rb new file mode 100644 index 0000000..b59a302 --- /dev/null +++ b/app/service/fetch_data_table_service/from_spreadsheet.rb @@ -0,0 +1,49 @@ +module FetchDataTableService + class FromSpreadsheet < Base + def initialize(spreadsheet_id, worksheet_name) + super + + sheets_api = SpreadsheetService::SheetsApi.create + spreadsheet_manager = spreadsheet_manager(sheets_api, spreadsheet_id) + spreadsheet = spreadsheet_manager.spreadsheet + + @worksheet_manager = worksheet_manager(sheets_api, spreadsheet, worksheet_name) + end + + def headers + @worksheet_manager.headers + end + + def rows + @worksheet_manager.rows + end + + def header_to_rows + {}.tap do |hash| + headers.each do |header| + rows.each do |row| + hash[header.to_sym] ||= [] + hash[header.to_sym] << row[headers.index(header)] + end + end + end + end + + private + + def spreadsheet_manager(sheets_api, spreadsheet_id) + SpreadsheetService::SpreadsheetManager.new( + sheets_api, + spreadsheet_id + ) + end + + def worksheet_manager(sheets_api, spreadsheet, worksheet_name) + SpreadsheetService::WorksheetManager.new( + sheets_api, + spreadsheet, + worksheet_name + ) + end + end +end diff --git a/app/service/import_service/base.rb b/app/service/import_service/base.rb new file mode 100644 index 0000000..46ca250 --- /dev/null +++ b/app/service/import_service/base.rb @@ -0,0 +1,21 @@ +module ImportService + class Base + class << self + def execute + raise NotImplementedError + end + + def columns + raise NotImplementedError + end + + def values + raise NotImplementedError + end + + def data_table + raise NotImplementedError + end + end + end +end diff --git a/app/service/import_service/star.rb b/app/service/import_service/star.rb new file mode 100644 index 0000000..1467848 --- /dev/null +++ b/app/service/import_service/star.rb @@ -0,0 +1,28 @@ +module ImportService + class Star < Base + class << self + def execute + ActiveRecord::Base.transaction do + ::Star.import(columns, values, validate: true) + end + end + + def columns + data_table.headers.map(&:to_sym) + end + + def values + data_table.rows + end + + def data_table + spreadsheet_title = 'basic_attributes' + + spreadsheet_id = SpreadsheetService::SheetId.retrieve(spreadsheet_title) + worksheet_name = 'stars' + + FetchDataTableService::FromSpreadsheet.new(spreadsheet_id, worksheet_name) + end + end + end +end diff --git a/app/service/import_service/title.rb b/app/service/import_service/title.rb deleted file mode 100644 index d8dc43a..0000000 --- a/app/service/import_service/title.rb +++ /dev/null @@ -1,20 +0,0 @@ -module ImportService - class Title - def execute - ActiveRecord::Base.transaction do - ::Title.import(columns, values, validate: true) - end - end - - def columns - [:name] - end - - def values - [ - ['abc'], - ['def'], - ] - end - end -end diff --git a/app/service/spreadsheet_service.rb b/app/service/spreadsheet_service.rb deleted file mode 100644 index 90f2ab4..0000000 --- a/app/service/spreadsheet_service.rb +++ /dev/null @@ -1,37 +0,0 @@ -require 'google/apis/sheets_v4' - -class SpreadsheetService - SPREADSHEET_ID = '12345'.freeze - RAW_PROPERTY_WORKSHEET_RANGE = 'raw_properties!A1:Z1000'.freeze - CREDS_JSON_FILEPATH = '/tmp/creds.json'.freeze - - def initialize - authorizer = Google::Auth::ServiceAccountCredentials.make_creds( - json_key_io: File.open(CREDS_JSON_FILEPATH), - - # drive と drive.file も必要になることには注意する - scope: %w[ - https://www.googleapis.com/auth/drive - https://www.googleapis.com/auth/drive.file - https://www.googleapis.com/auth/spreadsheets - ] - ) - authorizer.fetch_access_token! - - @api = Google::Apis::SheetsV4::SheetsService.new - @api.authorization = authorizer - end - - def raw_properties_rows - @api.get_spreadsheet_values( - SPREADSHEET_ID, - RAW_PROPERTY_WORKSHEET_RANGE - ) - end - - class << self - def raw_properties_rows - new.raw_properties_rows - end - end -end diff --git a/app/service/spreadsheet_service/sheet_id.rb b/app/service/spreadsheet_service/sheet_id.rb new file mode 100644 index 0000000..4760af3 --- /dev/null +++ b/app/service/spreadsheet_service/sheet_id.rb @@ -0,0 +1,16 @@ +module SpreadsheetService + class SheetId + class << self + def retrieve(spreadsheet_title) + title_to_id[spreadsheet_title] + end + + def title_to_id + { + 'basic_attributes' => ENV.fetch('SPREADSHEET_ID_BASIC_ATTRIBUTES'), + 'products' => ENV.fetch('SPREADSHEET_ID_PRODUCTS') + } + end + end + end +end diff --git a/app/service/spreadsheet_service/sheets_api.rb b/app/service/spreadsheet_service/sheets_api.rb new file mode 100644 index 0000000..3b9f2eb --- /dev/null +++ b/app/service/spreadsheet_service/sheets_api.rb @@ -0,0 +1,28 @@ +require 'google/apis/sheets_v4' + +module SpreadsheetService + class SheetsApi + CREDS_JSON_FILEPATH = ENV.fetch('SPREADSHEET_CREDENTIALS_FILEPATH', nil) + + class << self + def create + authorizer = Google::Auth::ServiceAccountCredentials.make_creds( + json_key_io: File.open(CREDS_JSON_FILEPATH), + + # drive と drive.file も必要になることには注意する + scope: %w[ + https://www.googleapis.com/auth/drive + https://www.googleapis.com/auth/drive.file + https://www.googleapis.com/auth/spreadsheets + ] + ) + authorizer.fetch_access_token! + + api = Google::Apis::SheetsV4::SheetsService.new + api.authorization = authorizer + + api + end + end + end +end diff --git a/app/service/spreadsheet_service/spreadsheet_manager.rb b/app/service/spreadsheet_service/spreadsheet_manager.rb new file mode 100644 index 0000000..3dac423 --- /dev/null +++ b/app/service/spreadsheet_service/spreadsheet_manager.rb @@ -0,0 +1,10 @@ +module SpreadsheetService + class SpreadsheetManager + attr_reader :spreadsheet, :worksheet_names + + def initialize(sheets_api, spreadsheet_id) + @spreadsheet = sheets_api.get_spreadsheet(spreadsheet_id) + @worksheet_names = @spreadsheet.sheets.map(&:properties).map(&:title) + end + end +end diff --git a/app/service/spreadsheet_service/worksheet_manager.rb b/app/service/spreadsheet_service/worksheet_manager.rb new file mode 100644 index 0000000..68ad503 --- /dev/null +++ b/app/service/spreadsheet_service/worksheet_manager.rb @@ -0,0 +1,36 @@ +module SpreadsheetService + class WorksheetManager + attr_reader :worksheet, :headers, :rows + + def initialize(sheets_api, spreadsheet, worksheet_name, range: 'A1:Z') + @sheets_api = sheets_api + @spreadsheet = spreadsheet + + @worksheet = @spreadsheet.sheets.find do |sheet| + sheet.properties.title == worksheet_name + end + + @all_cells = all_cells(@worksheet, range:) + @headers = @all_cells.first + @rows = @all_cells.drop(1) + + raise 'Some cells are invalid.' unless valid_all_cells? + end + + def all_cells(worksheet, range: 'A1:Z') + # 戻り値からは nil が取り除かれる + @sheets_api.get_spreadsheet_values( + @spreadsheet.spreadsheet_id, + "#{worksheet.properties.title}!#{range}" + ).values + end + + def valid_all_cells? + return false if @headers.blank? || @rows.blank? + + @rows.each { |row| return false if row.size != @headers.size } + + true + end + end +end diff --git a/config/database.yml b/config/database.yml index 3d231ad..99a80ac 100644 --- a/config/database.yml +++ b/config/database.yml @@ -6,27 +6,27 @@ default: &default development: <<: *default database: suikoden_vault_development - host: <%= ENV.fetch('PG_DEVELOPMENT_HOST') || 'localhost' %> - port: <%= ENV.fetch('PG_DEVELOPMENT_PORT') || 5432 %> - username: <%= ENV.fetch('PG_DEVELOPMENT_USER') || 'root' %> - password: <%= ENV.fetch('PG_DEVELOPMENT_PASS') || 'postgres' %> + host: <%= ENV['PG_DEVELOPMENT_HOST'] || 'localhost' %> + port: <%= ENV['PG_DEVELOPMENT_PORT'] || 5432 %> + username: <%= ENV['PG_DEVELOPMENT_USER'] || 'root' %> + password: <%= ENV['PG_DEVELOPMENT_PASS'] || 'postgres' %> test: <<: *default database: suikoden_vault_test - host: <%= ENV.fetch('PG_TEST_HOST') || 'localhost' %> - port: <%= ENV.fetch('PG_TEST_PORT') || 5432 %> - username: <%= ENV.fetch('PG_TEST_USER') || 'root' %> - password: <%= ENV.fetch('PG_TEST_PASS') || 'postgres' %> + host: <%= ENV['PG_TEST_HOST'] || 'localhost' %> + port: <%= ENV['PG_TEST_PORT'] || 5432 %> + username: <%= ENV['PG_TEST_USER'] || 'root' %> + password: <%= ENV['PG_TEST_PASS'] || 'postgres' %> production: <<: *default database: suikoden_vault_production - host: <%= ENV.fetch('PG_PRODUCTION_HOST') || 'localhost' %> - port: <%= ENV.fetch('PG_PRODUCTION_PORT') || 5432 %> - username: <%= ENV.fetch('PG_PRODUCTION_USER') %> - password: <%= ENV.fetch('PG_PRODUCTION_PASS') %> + host: <%= ENV['PG_PRODUCTION_HOST'] || 'localhost' %> + port: <%= ENV['PG_PRODUCTION_PORT'] || 5432 %> + username: <%= ENV['PG_PRODUCTION_USER'] %> + password: <%= ENV['PG_PRODUCTION_PASS'] %> sslmode: verify-ca - sslcert: <%= ENV.fetch('PG_SSLCERT_PATH') || 'config/postgresql/client.crt' %> - sslkey: <%= ENV.fetch('PG_SSLKEY_PATH') || 'config/postgresql/client.key' %> - sslrootcert: <%= ENV.fetch('PG_SSLROOT_PATH') || 'config/postgresql/root.crt' %> + sslcert: <%= ENV['PG_SSLCERT_PATH'] || 'config/postgresql/client.crt' %> + sslkey: <%= ENV['PG_SSLKEY_PATH'] || 'config/postgresql/client.key' %> + sslrootcert: <%= ENV['PG_SSLROOT_PATH'] || 'config/postgresql/root.crt' %> diff --git a/config/spreadsheet/.keep b/config/spreadsheet/.keep new file mode 100644 index 0000000..e69de29 diff --git a/db/migrate/20230612121238_create_stars.rb b/db/migrate/20230612121238_create_stars.rb new file mode 100644 index 0000000..449a8c6 --- /dev/null +++ b/db/migrate/20230612121238_create_stars.rb @@ -0,0 +1,14 @@ +class CreateStars < ActiveRecord::Migration[7.0] + def change + create_table :stars do |t| + t.string :name, null: false, comment: '宿星の名前（日本語）' + t.string :name_en, null: false, comment: '宿星の名前（英語）' + t.integer :seating_order, null: false, comment: '席次' + + t.timestamps + end + + add_index :stars, :name, unique: true + add_index :stars, :seating_order, unique: true + end +end diff --git a/db/schema.rb b/db/schema.rb index b783f98..7a6a490 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,8 +10,18 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema[7.0].define(version: 0) do +ActiveRecord::Schema[7.0].define(version: 2023_06_12_121238) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" + create_table "stars", force: :cascade do |t| + t.string "name", null: false, comment: "宿星の名前（日本語）" + t.string "name_en", null: false, comment: "宿星の名前（英語）" + t.integer "seating_order", null: false, comment: "席次" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["name"], name: "index_stars_on_name", unique: true + t.index ["seating_order"], name: "index_stars_on_seating_order", unique: true + end + end diff --git a/lib/tasks/db/reconstruction.rake b/lib/tasks/db/reconstruction.rake index 0068f5e..b64c55e 100644 --- a/lib/tasks/db/reconstruction.rake +++ b/lib/tasks/db/reconstruction.rake @@ -4,7 +4,11 @@ namespace :db do task execute: :environment do Rake::Task['db:migrate:reset'].invoke - # 時間がかかるようになったらこのあたりにスキップするロジックを組み込む + puts "[#{Time.zone.now}] ImportService::Star.execute の実行を開始します。" + ImportService::Star.execute + puts "[#{Time.zone.now}] ImportService::Star.execute の実行が終了しました。" + + # 時間がかかるようになったらスキップするロジックを組み込む # Rake::Task['importer:foo:bar'].invoke # Rake::Task['importer:hoge:fuga'].invoke # Rake::Task['importer:piyo:puyo'].invoke diff --git a/spec/factories/stars.rb b/spec/factories/stars.rb new file mode 100644 index 0000000..f84bd25 --- /dev/null +++ b/spec/factories/stars.rb @@ -0,0 +1,5 @@ +FactoryBot.define do + factory :star do + + end +end diff --git a/spec/models/star_spec.rb b/spec/models/star_spec.rb new file mode 100644 index 0000000..b4ec517 --- /dev/null +++ b/spec/models/star_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe Star, type: :model do + pending "add some examples to (or delete) #{__FILE__}" +end