MODS/PURL importer

Gemfile

@@ -83,6 +83,7 @@ gem 'lograge'
 gem 'zk'
 
 gem 'riiif', '~> 0.2.0'
+gem 'mods', '~> 2.0.3'
 
 gem 'sidekiq'
 gem 'iiif_manifest', '~> 0.1.2'

Gemfile.lock

@@ -344,6 +344,7 @@ GEM
       activesupport (>= 4)
       iiif-presentation (~> 0.1.0)
     is_it_working (1.1.0)
+    iso-639 (0.2.6)
     jbuilder (2.4.1)
       activesupport (>= 3.0.0, < 5.1)
       multi_json (~> 1.2)
@@ -425,6 +426,10 @@ GEM
     mini_magick (4.5.1)
     mini_portile2 (2.0.0)
     minitest (5.9.0)
+    mods (2.0.3)
+      iso-639
+      nokogiri
+      nom-xml (~> 0.5.2)
     multi_json (1.12.0)
     multi_xml (0.5.5)
     multipart-post (2.0.0)
@@ -761,6 +766,7 @@ DEPENDENCIES
   jbuilder (~> 2.0)
   jquery-rails
   lograge
+  mods (~> 2.0.3)
   peek
   peek-faraday
   peek-git

README.md

@@ -28,3 +28,8 @@ Start sidekiq from the root of your Rails application so the jobs will be proces
 bundle exec sidekiq
 ```
 
+## Importing from purl:
+
+```
+$ ./bin/import_from_purl ../hybox-objects bc390xk2647 bc402fk6835 bc483gc9313
+```

bin/import_from_purl

@@ -0,0 +1,47 @@
+#!/usr/bin/env ruby
+
+def validate_hostname!(hostname)
+  return if hostname
+  usage
+  $stderr.puts 'Please provide the hostname to import to.'
+  exit(1)
+end
+
+def validate_druids!(druids)
+  return if druids
+  usage
+  $stderr.puts 'Please provide a list of druids you want to import.'
+  exit(1)
+end
+
+def validate_imagepath!(imagepath)
+  return if imagepath
+  usage
+  $stderr.puts 'Image directory was left blank. No images will be ingested'
+  exit(1)
+end
+
+def load_rails
+  puts 'Loading environment...'
+  require File.expand_path('../../config/environment', __FILE__)
+  require 'stanford'
+  puts 'Starting import...'
+end
+
+def main(hostname, imagepath, druids)
+  validate_hostname!(hostname)
+  validate_druids!(druids)
+  validate_imagepath!(imagepath)
+  load_rails
+
+  Account.use_account!(hostname)
+  size = Stanford::Importer::PurlImporter.new(imagepath, druids).import_all
+
+  puts "Imported #{size} records."
+end
+
+def usage
+  $stderr.puts "Usage: #{$PROGRAM_NAME} <hostname> <image directory> <druids to import>"
+end
+
+main(ARGV[0], ARGV[1], ARGV.drop(2))

bin/import_mods_files

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+
+def validate_hostname!(hostname)
+  return if hostname
+  usage
+  $stderr.puts 'Please provide the hostname to import to.'
+  exit(1)
+end
+
+def validate_mods_directory!(mods_directory)
+  if mods_directory
+    unless File.exist?(mods_directory)
+      $stderr.puts "Directory doesn't exist #{mods_directory}"
+      exit(1)
+    end
+
+    unless File.directory?(mods_directory)
+      $stderr.puts "#{mods_directory} is not a directory"
+      exit(1)
+    end
+  else
+    usage
+    $stderr.puts 'Please provide the a file path you want to import.'
+    exit(1)
+  end
+end
+
+def validate_imagepath!(imagepath)
+  return if imagepath
+  usage
+  $stderr.puts 'Image directory was left blank. No images will be ingested'
+  exit(1)
+end
+
+def load_rails
+  puts 'Loading environment...'
+  require File.expand_path('../../config/environment', __FILE__)
+  require 'importer'
+  puts 'Starting import...'
+end
+
+
+def main(hostname, mods_directory, imagepath)
+  validate_hostname!(hostname)
+  validate_mods_directory!(mods_directory)
+  validate_imagepath!(imagepath)
+  load_rails
+
+  Account.use_account!(hostname)
+  size = Importer::ModsImporter.new(imagepath, mods_directory).import_all
+
+  puts "Imported #{size} records."
+end
+
+def usage
+  $stderr.puts "Usage: #{$PROGRAM_NAME} <hostname> <mods_directory> <image directory>"
+end
+
+
+main(ARGV[0], ARGV[1], ARGV[2])

config/initializers/assets.rb

@@ -8,4 +8,3 @@
 
 # Precompile additional assets.
 # application.js, application.css, and all non-JS/CSS in app/assets folder are already added.
-# Rails.application.config.assets.precompile += %w( search.js )

lib/importer.rb

@@ -0,0 +1,7 @@
+module Importer
+  extend ActiveSupport::Autoload
+  autoload :AttachFiles
+  autoload :ModsImporter
+  autoload :ModsParser
+  autoload :Factory
+end

lib/importer/attach_files.rb

@@ -0,0 +1,41 @@
+module Importer
+  class AttachFiles
+    def self.run(work, files_directory, file_name)
+      AttachFiles.new(work, files_directory, file_name).run
+    end
+
+    attr_reader :work, :files_directory, :file_names
+
+    def initialize(work, files_directory, file_names)
+      @work = work
+      @files_directory = files_directory
+      @file_names = file_names
+    end
+
+    def run
+      # only attach files once
+      return if work.file_sets.count > 0
+      file_names.each do |file_path|
+        create_file(file_path)
+      end
+    end
+
+    private
+
+      def create_file(file_name)
+        path = file_path(file_name)
+        unless File.exist?(path)
+          Rails.logger.error "  * File doesn't exist at #{path}"
+          return
+        end
+
+        actor = CurationConcerns::Actors::FileSetActor.new(FileSet.new, User.batchuser)
+        actor.create_metadata(work)
+        actor.create_content(File.new(path))
+      end
+
+      def file_path(file_name)
+        File.join(files_directory, file_name)
+      end
+  end
+end

lib/importer/factory.rb

@@ -0,0 +1,18 @@
+module Importer
+  module Factory
+    extend ActiveSupport::Autoload
+
+    eager_autoload do
+      autoload :CollectionFactory
+      autoload :ETDFactory
+      autoload :ImageFactory
+      autoload :ObjectFactory
+      autoload :StringLiteralProcessor
+      autoload :WithAssociatedCollection
+    end
+
+    def self.for(model_name)
+      const_get "#{model_name}Factory"
+    end
+  end
+end

lib/importer/factory/collection_factory.rb

@@ -0,0 +1,18 @@
+module Importer
+  module Factory
+    class CollectionFactory < ObjectFactory
+      self.klass = Collection
+      self.system_identifier_field = :identifier
+
+      def find_or_create
+        collection = find
+        return collection if collection
+        run(&:save!)
+      end
+
+      def attach_files
+        # nop
+      end
+    end
+  end
+end

lib/importer/factory/etd_factory.rb

@@ -0,0 +1,17 @@
+module Importer
+  module Factory
+    class ETDFactory < ObjectFactory
+      include WithAssociatedCollection
+
+      self.klass = GenericWork
+      self.attach_files_service = Importer::AttachFiles
+      # A way to identify objects that are not Hydra minted identifiers
+      self.system_identifier_field = :identifier
+
+      # TODO: add resource type?
+      # def create_attributes
+      #   #super.merge(resource_type: 'ETD')
+      # end
+    end
+  end
+end

lib/importer/factory/image_factory.rb

@@ -0,0 +1,17 @@
+module Importer
+  module Factory
+    class ImageFactory < ObjectFactory
+      include WithAssociatedCollection
+
+      self.klass = GenericWork
+      self.attach_files_service = Importer::AttachFiles
+      # A way to identify objects that are not Hydra minted identifiers
+      self.system_identifier_field = :identifier
+
+      # TODO: add resource type?
+      # def create_attributes
+      #   #super.merge(resource_type: 'Image')
+      # end
+    end
+  end
+end