WIP: Added data sources.

TODO: Add tests for data sources.
jamesiarmes · Dec 8, 2022 · a21772d · a21772d
1 parent 41f18ea
commit a21772d
Show file tree

Hide file tree

Showing 54 changed files with 524 additions and 193 deletions.
diff --git a/.rubocop.yml b/.rubocop.yml
@@ -16,6 +16,15 @@ AllCops:
     - "config/environments/*"
     - "node_modules/**/*"
 
+Metrics/AbcSize:
+  CountRepeatedAttributes: false
+
+Metrics/MethodLength:
+  CountAsOne:
+    - array
+    - hash
+    - heredoc
+
 Naming/VariableNumber:
   EnforcedStyle: non_integer
   AllowedIdentifiers:

diff --git a/Dockerfile b/Dockerfile
@@ -4,10 +4,13 @@ FROM ruby:3.1.2
 RUN apt-get update -q && \
     apt-get upgrade -q -y && \
     apt-get install -q -y nodejs postgresql-client && \
+#    apt-get install -q -y libcurl4-openssl-dev libgcrypt11-dev libgnutls-dev libidn11-dev libldap2-dev librtmp-dev libtasn1-3-dev m4 && \
+    apt-get install -q -y libcurl4-openssl-dev libgcrypt20-dev libgnutls28-dev libidn11-dev libldap2-dev librtmp-dev libtasn1-6-dev m4 && \
     apt-get clean
 
 WORKDIR /opt/classifyr
 COPY . /opt/classifyr
+RUN gem update && gem cleanup
 RUN bundle install
 RUN rails assets:precompile
 

diff --git a/Gemfile b/Gemfile
@@ -3,6 +3,7 @@
 source 'https://rubygems.org'
 git_source(:github) { |repo| "https://github.com/#{repo}.git" }
 
+# ruby '3.1.2'
 ruby '3.1.2'
 
 gem 'active_storage_validations', '~> 0.9'
@@ -25,6 +26,9 @@ gem 'puma', '~> 5.6'
 gem 'rails', '~> 7.0', '>= 7.0.2.3'
 gem 'seed_dump', '~> 3.3'
 gem 'sprockets-rails', '~> 3.4'
+
+gem 'soda-ruby', '~> 1.0'
+
 gem 'stimulus-rails', '~> 1.1'
 gem 'tailwindcss-rails', '2.0.12'
 gem 'turbo-rails', '~> 1.1'
@@ -39,6 +43,13 @@ group :development, :test do
   gem 'pry', '~> 0.14'
   gem 'rspec-github', '~> 2.3'
   gem 'rspec-rails', '~> 5.1'
+
+  # gem 'debase', '~> 0.2.5.beta2'
+  # source 'file:///opt/classifyr/repo'
+  # gem 'debase', '3.0.0.beta.5'
+  # gem 'debase', source: 'file:///opt/classifyr/repo'
+  # gem 'debase', source: 'file:///Users/jamesarmes/IdeaProjects/classifyr/repo'
+  # gem 'ruby-debug-ide'
 end
 
 group :development do

diff --git a/Gemfile.lock b/Gemfile.lock
@@ -87,19 +87,19 @@ GEM
       public_suffix (>= 2.0.2, < 6.0)
     ast (2.4.2)
     aws-eventstream (1.2.0)
-    aws-partitions (1.648.0)
-    aws-record (2.8.0)
+    aws-partitions (1.669.0)
+    aws-record (2.9.0)
       aws-sdk-dynamodb (~> 1.18)
-    aws-sdk-core (3.162.0)
+    aws-sdk-core (3.168.2)
       aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.525.0)
-      aws-sigv4 (~> 1.1)
+      aws-partitions (~> 1, >= 1.651.0)
+      aws-sigv4 (~> 1.5)
       jmespath (~> 1, >= 1.6.1)
-    aws-sdk-dynamodb (1.77.0)
-      aws-sdk-core (~> 3, >= 3.127.0)
+    aws-sdk-dynamodb (1.79.0)
+      aws-sdk-core (~> 3, >= 3.165.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-kms (1.58.0)
-      aws-sdk-core (~> 3, >= 3.127.0)
+    aws-sdk-kms (1.60.0)
+      aws-sdk-core (~> 3, >= 3.165.0)
       aws-sigv4 (~> 1.1)
     aws-sdk-rails (3.6.4)
       aws-record (~> 2)
@@ -108,15 +108,15 @@ GEM
       aws-sessionstore-dynamodb (~> 2)
       concurrent-ruby (~> 1)
       railties (>= 5.2.0)
-    aws-sdk-s3 (1.115.0)
-      aws-sdk-core (~> 3, >= 3.127.0)
+    aws-sdk-s3 (1.117.2)
+      aws-sdk-core (~> 3, >= 3.165.0)
       aws-sdk-kms (~> 1)
       aws-sigv4 (~> 1.4)
-    aws-sdk-ses (1.47.0)
-      aws-sdk-core (~> 3, >= 3.127.0)
+    aws-sdk-ses (1.48.0)
+      aws-sdk-core (~> 3, >= 3.165.0)
       aws-sigv4 (~> 1.1)
-    aws-sdk-sqs (1.51.1)
-      aws-sdk-core (~> 3, >= 3.127.0)
+    aws-sdk-sqs (1.52.0)
+      aws-sdk-core (~> 3, >= 3.165.0)
       aws-sigv4 (~> 1.1)
     aws-sessionstore-dynamodb (2.0.1)
       aws-sdk-dynamodb (~> 1)
@@ -125,14 +125,14 @@ GEM
       aws-eventstream (~> 1, >= 1.0.2)
     bcrypt (3.1.18)
     bindex (0.8.1)
-    bootsnap (1.13.0)
+    bootsnap (1.15.0)
       msgpack (~> 1.2)
-    brakeman (5.3.1)
+    brakeman (5.4.0)
     builder (3.2.4)
-    bullet (7.0.3)
+    bullet (7.0.4)
       activesupport (>= 3.0.0)
       uniform_notifier (~> 1.11)
-    capybara (3.37.1)
+    capybara (3.38.0)
       addressable
       matrix
       mini_mime (>= 0.1.3)
@@ -157,7 +157,7 @@ GEM
       activerecord (>= 5.a)
       database_cleaner-core (~> 2.0.0)
     database_cleaner-core (2.0.1)
-    debug (1.6.2)
+    debug (1.6.3)
       irb (>= 1.3.6)
       reline (>= 0.3.1)
     diff-lcs (1.5.0)
@@ -169,11 +169,13 @@ GEM
       railties (>= 5.0.0)
     faker (2.23.0)
       i18n (>= 1.8.11, < 2)
+    ffi (1.15.5)
     friendly_id (5.4.2)
       activerecord (>= 4.0.0)
     globalid (1.0.0)
       activesupport (>= 5.0)
     hashdiff (1.0.1)
+    hashie (3.6.0)
     health-monitor-rails (9.3.0)
       railties (>= 4.0)
     i18n (1.12.0)
@@ -182,12 +184,12 @@ GEM
       actionpack (>= 6.0.0)
       railties (>= 6.0.0)
     io-console (0.5.11)
-    irb (1.4.2)
+    irb (1.5.1)
       reline (>= 0.3.0)
     jbuilder (2.11.5)
       actionview (>= 5.0.0)
       activesupport (>= 5.0.0)
-    jmespath (1.6.1)
+    jmespath (1.6.2)
     json (2.6.2)
     kaminari (1.2.2)
       activesupport (>= 4.1.0)
@@ -216,13 +218,14 @@ GEM
     mini_mime (1.1.2)
     minitest (5.16.3)
     msgpack (1.6.0)
+    multipart-post (2.0.0)
     net-imap (0.3.1)
       net-protocol
     net-pop (0.1.2)
       net-protocol
     net-protocol (0.1.3)
       timeout
-    net-smtp (0.3.2)
+    net-smtp (0.3.3)
       net-protocol
     nio4r (2.5.8)
     nokogiri (1.13.9-x86_64-darwin)
@@ -234,9 +237,9 @@ GEM
       activerecord (>= 5.2)
       request_store (~> 1.1)
     parallel (1.22.1)
-    parser (3.1.2.1)
+    parser (3.1.3.0)
       ast (~> 2.4.1)
-    pg (1.4.4)
+    pg (1.4.5)
     pg_search (2.3.6)
       activerecord (>= 5.2)
       activesupport (>= 5.2)
@@ -278,7 +281,7 @@ GEM
       zeitwerk (~> 2.5)
     rainbow (3.1.1)
     rake (13.0.6)
-    regexp_parser (2.6.0)
+    regexp_parser (2.6.1)
     reline (0.3.1)
       io-console (~> 0.5)
     request_store (1.5.1)
@@ -287,16 +290,16 @@ GEM
       actionpack (>= 5.0)
       railties (>= 5.0)
     rexml (3.2.5)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.1)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-github (2.3.1)
+      rspec-support (~> 3.12.0)
+    rspec-github (2.4.0)
       rspec-core (~> 3.0)
-    rspec-mocks (3.11.1)
+    rspec-mocks (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
+      rspec-support (~> 3.12.0)
     rspec-rails (5.1.2)
       actionpack (>= 5.2)
       activesupport (>= 5.2)
@@ -305,54 +308,60 @@ GEM
       rspec-expectations (~> 3.10)
       rspec-mocks (~> 3.10)
       rspec-support (~> 3.10)
-    rspec-support (3.11.1)
-    rubocop (1.36.0)
+    rspec-support (3.12.0)
+    rubocop (1.39.0)
       json (~> 2.3)
       parallel (~> 1.10)
       parser (>= 3.1.2.1)
       rainbow (>= 2.2.2, < 4.0)
       regexp_parser (>= 1.8, < 3.0)
       rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.20.1, < 2.0)
+      rubocop-ast (>= 1.23.0, < 2.0)
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 3.0)
-    rubocop-ast (1.22.0)
+    rubocop-ast (1.24.0)
       parser (>= 3.1.1.0)
-    rubocop-performance (1.15.0)
+    rubocop-performance (1.15.1)
       rubocop (>= 1.7.0, < 2.0)
       rubocop-ast (>= 0.4.0)
-    rubocop-rails (2.16.1)
+    rubocop-rails (2.17.3)
       activesupport (>= 4.2.0)
       rack (>= 1.1)
       rubocop (>= 1.33.0, < 2.0)
-    rubocop-rspec (2.13.2)
+    rubocop-rspec (2.15.0)
       rubocop (~> 1.33)
     ruby-progressbar (1.11.0)
     rubyzip (2.3.2)
     seed_dump (3.3.1)
       activerecord (>= 4)
       activesupport (>= 4)
-    selenium-webdriver (4.5.0)
+    selenium-webdriver (4.6.1)
       childprocess (>= 0.5, < 5.0)
       rexml (~> 3.2, >= 3.2.5)
       rubyzip (>= 1.2.2, < 3.0)
       websocket (~> 1.0)
+    soda-ruby (1.0.1)
+      hashie (~> 3.5)
+      multipart-post (~> 2.0.0)
+      sys-uname (~> 1.0.2)
     sprockets (4.1.1)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.4.2)
       actionpack (>= 5.2)
       activesupport (>= 5.2)
       sprockets (>= 3.0.0)
-    stimulus-rails (1.1.0)
+    stimulus-rails (1.2.1)
       railties (>= 6.0.0)
+    sys-uname (1.0.4)
+      ffi (>= 1.0.0)
     tailwindcss-rails (2.0.12-x86_64-darwin)
       railties (>= 6.0.0)
     tailwindcss-rails (2.0.12-x86_64-linux)
       railties (>= 6.0.0)
     thor (1.2.1)
     timeout (0.3.0)
-    turbo-rails (1.3.1)
+    turbo-rails (1.3.2)
       actionpack (>= 6.0.0)
       activejob (>= 6.0.0)
       railties (>= 6.0.0)
@@ -382,10 +391,11 @@ GEM
     websocket-extensions (0.1.5)
     xpath (3.2.0)
       nokogiri (~> 1.8)
-    zeitwerk (2.6.1)
+    zeitwerk (2.6.6)
 
 PLATFORMS
   x86_64-darwin-20
+  x86_64-darwin-21
   x86_64-linux
 
 DEPENDENCIES
@@ -426,6 +436,7 @@ DEPENDENCIES
   rubocop-rspec (~> 2.12)
   seed_dump (~> 3.3)
   selenium-webdriver (~> 4.3)
+  soda-ruby (~> 1.0)
   sprockets-rails (~> 3.4)
   stimulus-rails (~> 1.1)
   tailwindcss-rails (= 2.0.12)
@@ -440,4 +451,4 @@ RUBY VERSION
    ruby 3.1.2p20
 
 BUNDLED WITH
-   2.3.7
+   2.3.26
diff --git a/app/controllers/data_sets_controller.rb b/app/controllers/data_sets_controller.rb
@@ -7,7 +7,7 @@ class DataSetsController < ApplicationController
 
   def index
     authorize! :index, :data_sets
-    @data_sets = DataSet.includes(files_attachments: :blob).ordered
+    @data_sets = DataSet.includes(:data_source).ordered
   end
 
   def show
@@ -18,6 +18,7 @@ def show
   def new
     authorize! :create, :data_sets
     @data_set = DataSet.new
+    @data_set.build_data_source
     add_breadcrumb('Create a new dataset', nil)
   end
 
@@ -37,7 +38,7 @@ def create
     end
   end
 
-  def update # rubocop:disable Metrics/AbcSize
+  def update
     authorize! :update, :data_sets
     if params['data_set']['step'] == 'map_fields'
       @data_set.map_fields(params['data_set']['common_types'])
@@ -95,7 +96,8 @@ def data_set_params
     params.require(:data_set).permit(
       :title, :data_link, :api_links, :source, :exclusions, :license, :format,
       :documentation_link, :city, :state, :description, :has_911, :has_ems,
-      :has_fire, files: []
+      :has_fire,
+      data_source_attributes: [:type, :api_domain, :api_resource, :api_key, { files: [] }]
     )
   end
 

diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb
@@ -2,7 +2,7 @@
 
 # Application level helper methods.
 module ApplicationHelper
-  def us_states # rubocop:disable Metrics/MethodLength
+  def us_states
     [
       ['', nil],
       %w[Alabama AL],

diff --git a/app/helpers/data_sets_helper.rb b/app/helpers/data_sets_helper.rb
@@ -8,7 +8,7 @@ def completed_card_class(data_set)
     'bg-white'
   end
 
-  def location(data_set) # rubocop:disable Metrics/AbcSize
+  def location(data_set)
     return '' unless data_set.city.present? || data_set.state.present?
     return data_set.city if data_set.city.present? && data_set.state.blank?
     return data_set.state if data_set.city.blank? && data_set.state.present?

diff --git a/app/javascript/controllers/application.js b/app/javascript/controllers/application.js
@@ -1,5 +1,6 @@
 import { Application } from "@hotwired/stimulus"
 import { Autocomplete } from 'stimulus-autocomplete'
+import RevealController from 'stimulus-reveal'
 
 const application = Application.start()
 
@@ -18,5 +19,6 @@ application.register('popover', Popover)
 application.register('toggle', Toggle)
 application.register('slideover', Slideover)
 application.register('autocomplete', Autocomplete)
+application.register('reveal', RevealController)
 
 export { application }