Change from MongoDB to ActiveRecord

.travis.yml

@@ -3,7 +3,6 @@ cache: bundler
 sudo: false
 rvm:
 - 2.4
-services: mongodb
 env:
   global:
     - PUSHER_APP_ID=abc123

Gemfile

@@ -54,7 +54,7 @@ group :development, :test do
   gem 'vcr', '~> 6.0'
   gem 'cucumber-api-steps', require: false, git: 'https://github.com/Data-Liberation-Front/cucumber-api-steps.git', branch: 'feature-test-content-type'
   # Use sqlite3 as the database for Active Record
-  gem 'sqlite3', '~> 1.4'
+  gem 'sqlite3', '~> 1.3.13'
 end
 
 group :development do
@@ -74,10 +74,6 @@ end
 
 gem 'bootstrap-sass', '~> 3.4'
 gem 'rack-google-analytics', '~> 1.2'
-gem 'mongo', '~> 2.1'
-gem 'mongoid', '~> 5.1'
-gem 'kaminari-mongoid', '~> 1.0'
-gem 'mongoid-grid_fs', '~> 2.4'
 gem 'kaminari', '~> 1.2'
 gem 'bootstrap-kaminari-views', '~> 0.0'
 gem 'data_kitten', git: 'https://github.com/Data-Liberation-Front/data_kitten.git', ref: "e343510bd15e3329c1f2fab35035e248195348be", require: false

Gemfile.lock

@@ -97,7 +97,6 @@ GEM
     bootstrap-sass (3.4.1)
       autoprefixer-rails (>= 5.2.1)
       sassc (>= 2.0.0)
-    bson (3.2.7)
     builder (3.2.4)
     capybara (2.4.4)
       mime-types (>= 1.16)
@@ -438,9 +437,6 @@ GEM
       activerecord
       kaminari-core (= 1.2.1)
     kaminari-core (1.2.1)
-    kaminari-mongoid (1.0.1)
-      kaminari-core (~> 1.0)
-      mongoid
     launchy (2.4.3)
       addressable (~> 2.3)
     ld-patch (0.3.3)
@@ -489,17 +485,7 @@ GEM
     mime-types (2.99.3)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
-    minitest (5.14.2)
-    mongo (2.1.2)
-      bson (~> 3.0)
-    mongoid (5.1.6)
-      activemodel (~> 4.0)
-      mongo (~> 2.1)
-      origin (~> 2.2)
-      tzinfo (>= 0.3.37)
-    mongoid-grid_fs (2.4.0)
-      mime-types (>= 1.0, < 4.0)
-      mongoid (>= 3.0, < 8.0)
+    minitest (5.14.3)
     multi_json (1.15.0)
     multi_test (0.1.2)
     multipart-post (2.1.1)
@@ -517,7 +503,6 @@ GEM
       shellany (~> 0.0)
     open_uri_redirections (0.2.1)
     optimist (3.0.1)
-    origin (2.3.1)
     poltergeist (1.6.0)
       capybara (~> 2.1)
       cliver (~> 0.3.1)
@@ -717,11 +702,11 @@ GEM
     sprockets (3.7.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
-    sprockets-rails (3.2.1)
+    sprockets-rails (3.2.2)
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
-    sqlite3 (1.4.2)
+    sqlite3 (1.3.13)
     sxp (1.1.0)
       rdf (~> 3.1)
     sync (0.5.0)
@@ -812,10 +797,6 @@ DEPENDENCIES
   jquery-dotdotdot-rails (~> 1.6)
   jquery-rails (~> 4.4)
   kaminari (~> 1.2)
-  kaminari-mongoid (~> 1.0)
-  mongo (~> 2.1)
-  mongoid (~> 5.1)
-  mongoid-grid_fs (~> 2.4)
   nokogiri (~> 1.10)
   poltergeist (~> 1.6)
   pry (~> 0.13)
@@ -835,7 +816,7 @@ DEPENDENCIES
   sidekiq (~> 4.2)
   simplecov (~> 0.16)
   spring (~> 2.1)
-  sqlite3 (~> 1.4)
+  sqlite3 (~> 1.3.13)
   therubyracer (~> 0.12)
   thin (~> 1.8)
   timecop (~> 0.9)

README.md

@@ -23,7 +23,7 @@ Follow the [public feature roadmap for CSVLint](https://trello.com/b/2xc7Q0kd/la
 
 Ruby version 2.3.1
 
-The application uses mongod and redis databases as background jobs for data persistence
+The application uses SQL and redis databases as background jobs for data persistence
 
 `.env` file (see below)
 
@@ -103,14 +103,8 @@ S3_BUCKET=<YOURNEWS3BUCKETNAME>
 
 ## System dependencies and Configuration
 
-Install mongo:
-    `brew install mongo redis` (if using brew)
-
-make a data directory for mongo databases
-  `sudo mkdir -p /data/db`
-
-change directory ownership so that mongodb can operate
-  `sudo chown -R $USERNAME /data/`
+Install redis:
+    `brew install redis` (if using brew)
 
 ### Development: Running the full application locally
 
@@ -120,8 +114,6 @@ Checkout the repository and run ```bundle``` in the checked out directory.
 
 #### Database initialization
 
-run mongo  : `mongod`
-
 run redis  : `redis-server`
 
 #### Services (job queues, cache servers, search engines, etc.)

app.json

@@ -38,7 +38,6 @@
     }
   },
   "addons": [
-    "mongolab:sandbox",
     "heroku-postgresql",
     "redistogo:nano",
     "pusher:sandbox"

app/controllers/package_controller.rb

@@ -26,7 +26,7 @@ def show
       redirect_to validation_path(@package.validations.first)
     end
 
-    @dataset = Marshal.load(@package.dataset) rescue nil
+    @dataset = nil#Marshal.load(@package.dataset) rescue nil
     @validations = @package.validations
   end
 

app/controllers/schemas_controller.rb

@@ -1,7 +1,7 @@
 class SchemasController < ApplicationController
 
   # schemas are only persisted to database in the case of a URL existing on processing of data
-  # all instance variables in this controller are retrieved from the persisted data, in this instance a MongoDB
+  # all instance variables in this controller are retrieved from the persisted data
   # This explains the sparseness of the Schema model
 
   def index

app/models/package.rb

@@ -36,24 +36,21 @@ def detect_publishing_format
   end
 end
 
-class Package
-  include Mongoid::Document
-  include Mongoid::Timestamps
+class Package < ActiveRecord::Base
 
-  field :url, type: String
-  field :dataset, type: String
-  field :type, type: String
+  validates :url, presence: true
+  validates :dataset, presence: true
+  validates :type, presence: true
 
   has_many :validations
 
   def parse_package(dataset, validations)
     attributes = {
       :url => dataset.origin == :local ? nil : dataset.access_url,
-      :dataset => Marshal.dump(dataset),
+      :dataset => '',#Marshal.dump(dataset),
       :validations => validations,
       :type => dataset.publishing_format
     }
-
     return attributes
   end
 

app/models/schema.rb

@@ -1,14 +1,10 @@
-class Schema
-  include Mongoid::Document
-  include Mongoid::Timestamps
-
-  field :url, type: String
-
+class Schema < ActiveRecord::Base
   has_many :validations
 
+  validates :url, presence: true
+
   def to_param
     id.to_s
   end
 
 end
-

app/models/summary.rb

@@ -1,35 +1,11 @@
-class LevelSummary
-  include Mongoid::Document
-  embedded_in :summary
-  field :errors_breakdown, type: Hash
-  field :warnings_breakdown, type: Hash
-  field :info_messages_breakdown, type: Hash
-end
-
-class CategorySummary
-  include Mongoid::Document
-  embedded_in :summary
-  field :structure_breakdown, type: Hash
-  field :schema_breakdown, type: Hash
-  field :context_breakdown, type: Hash
-end
-
-class Summary
-  include Mongoid::Document
-  include Mongoid::Timestamps
-
-  field :sources, type: Integer
-  field :states, type: Hash
-  field :hosts, type: Hash
 
-  embeds_one :level_summary
-  embeds_one :category_summary
+class Summary < ActiveRecord::Base
 
   def self.generate
     summary = Summary.create
 
     validations = Validation.where(:url.ne => nil).order_by(:created_at.desc)
-    # retrieve validations from Mongo Datastore, ordered in reverse by date created
+    # retrieve validations ordered in reverse by date created
 
     summary.sources = validations.length
     summary.states = {"warnings"=>0, "valid"=>0, "not_found"=>0, "invalid"=>0}

app/models/validation.rb

@@ -1,19 +1,11 @@
-class Validation
-  include Mongoid::Document
-  include Mongoid::Timestamps
-
-  field :filename, type: String
-  field :url, type: String
-  field :state, type: String
-  field :result, type: String
-  field :csv_id, type: String
-  field :parse_options, type: Hash
-  field :expirable_created_at, type: Time
-
-  index :created_at => 1
-  index({expirable_created_at: 1}, {expire_after_seconds: 24.hours})
-  # invoke the mongo time-to-live feature which will automatically expire entries
-  # - this index is only enabled for a subset of validations, which are validations uploaded as file
+class Validation < ActiveRecord::Base
+
+  validates :filename, presence: true
+  validates :url, presence: true
+  validates :state, presence: true
+  validates :result, presence: true
+  validates :csv_id, presence: true
+  validates :parse_options, presence: true
 
   belongs_to :schema
   accepts_nested_attributes_for :schema
@@ -70,7 +62,7 @@ def self.validate(io, schema_url = nil, schema = nil, dialect = nil, expiry)
       :url => url,
       :filename => filename,
       :state => state,
-      :result => Marshal.dump(validator).force_encoding("UTF-8"),
+      :result => '',#Marshal.dump(validator).force_encoding("UTF-8"),
       :parse_options => Validation.generate_options(validator.dialect)
     }
 
@@ -92,7 +84,6 @@ def self.validate(io, schema_url = nil, schema = nil, dialect = nil, expiry)
 
 
   def self.fetch_validation(id, format, revalidate = nil)
-    # returns a mongo database record
     v = self.find(id)
 
     unless revalidate.to_s == "false"
@@ -170,20 +161,18 @@ def self.create_validation(io, schema_url = nil, schema = nil)
   def validate(io, schema_url = nil, schema = nil, expiry)
     validation = Validation.validate(io, schema_url, schema, nil, expiry)
     self.update_attributes(validation)
-    # update_attributes is a method from Mongoid
   end
 
   def update_validation(dialect = nil, expiry=nil)
     loaded_schema = schema ? Csvlint::Schema.load_from_uri(schema.url) : nil
     io = self.url.nil? ? StoredCSV.fetch(self.filename) : self.url
     validation = Validation.validate(io, schema.try(:url), loaded_schema, dialect, expiry)
     self.update_attributes(validation)
-    # update mongoDB record
     self
   end
 
   def csv
-    # method that retrieves stored entire CSV file from mongoDB
+    # method that retrieves stored entire CSV file
     if self.url
       csv = open(self.url).read
     else
@@ -227,7 +216,7 @@ def check_validation
   end
 
   def validator
-    Marshal.load(self.result)
+    nil #Marshal.load(self.result)
   end
 
   # Empty method? Intended functionality?

config/application.rb

@@ -1,7 +1,7 @@
 require File.expand_path('../boot', __FILE__)
 
 # Pick the frameworks you want:
-#require "active_record/railtie"
+require "active_record/railtie"
 require "action_controller/railtie"
 require "action_mailer/railtie"
 require "sprockets/railtie"

db/migrate/20210108233739_create_schemas.rb

@@ -0,0 +1,9 @@
+class CreateSchemas < ActiveRecord::Migration
+  def change
+    create_table :schemas do |t|
+      t.string :url
+
+      t.timestamps null: false
+    end
+  end
+end

db/migrate/20210108234550_create_packages.rb

@@ -0,0 +1,11 @@
+class CreatePackages < ActiveRecord::Migration
+  def change
+    create_table :packages do |t|
+      t.string :url
+      t.string :dataset
+      t.string :type
+
+      t.timestamps null: false
+    end
+  end
+end

db/migrate/20210108235223_create_validations.rb

@@ -0,0 +1,14 @@
+class CreateValidations < ActiveRecord::Migration
+  def change
+    create_table :validations do |t|
+      t.string :filename
+      t.string :url
+      t.string :state
+      t.binary :result
+      t.string :csv_id
+      t.string :parse_options
+
+      t.timestamps null: false
+    end
+  end
+end