Work with new Lexicon tables

This commit was supposed to address a small change in Lexicon,
but resulted in tearing out a lot of defunct or deprecated code,
namely:
- Removed (almost) all references to Impala, refs #262
- Removed a lot of OMOP-related tests, refs #261
- More work on defaulting to GDM, refs #265

Gemfile

@@ -3,7 +3,6 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in conceptql.gemspec
 gemspec
 gem "pg"
-gem "sequel_impala", path: "../sequel_impala"
 
 group :test, :development do
   gem "pry"

README.md

@@ -96,26 +96,19 @@ If you're feeling bold, feel free to try your hand at authoring a ConceptQL stat
 
 ### Setup
 
-In order to run the tests for ConceptQL, you first have to create a database and load the OMOP vocabularies into it.  You can use [loadmop](https://github.com/outcomesinsights/loadmop/tree/develop) do to so.
+You must have the latest version of ConceptQL's test database.
 
-Follow the instructions in [loadmop](https://github.com/outcomesinsights/loadmop/#preparation) to setup the vocabulary files correctly.
+To set up this test database in PostgreSQL, execute the following command:
 
-After loading the vocabulary data file into the database,
-create an .env file in the root of the conceptql directory,
-similar or identical to the one used for loadmop.  Then
-run `rake test_db_setup`.  This will load the ConceptQL test
-data into the database.  This only needs to be done once, not
-every time you run the tests.
-
-### Running
+```
+curl -sSL http://chisel.test_data.jsaw.io | pigz -dc | psql <url to a test database>
+```
 
-If any of the CSV files in `test/data` has been updated since you
-last updated the database, you should run `rake test_db_teardown test_db_setup`
-to reset the data in the database.  Updates to the CSV files in
-`test/data` should be infrequent.
+Then run the tests using:
 
-After the test database has been setup, you can run the tests
-using `rake`, as the default task is set to run the tests.
+```
+SEQUELIZER_URL=<same url as above> SEQUELIZER_SEARCH_PATH=gdm_250 bundle exec ruby test/all.rb
+```
 
 ## Contributing
 

Rakefile

@@ -5,16 +5,6 @@ end
 
 ENV['CONCEPTQL_DATA_MODEL'] ||= ConceptQL::DEFAULT_DATA_MODEL.to_s
 
-desc "Setup test database"
-task :test_db_setup do
-  require_relative 'test/db_setup'
-end
-
-desc "Setup test database"
-task :test_db_teardown do
-  require_relative 'test/db_teardown'
-end
-
 run_spec = lambda do |data_model|
   sh "CONCEPTQL_DATA_MODEL=#{data_model} #{FileUtils::RUBY} test/all.rb"
 end

lib/conceptql/database.rb

@@ -9,7 +9,7 @@ class Database
 
     def initialize(db, opts={})
       @db = db
-      db_type = :impala
+      db_type = :postgres
       if db
         self.class.db_extensions(db)
         db_type = db.database_type.to_sym

lib/conceptql/lexicon.rb

@@ -54,9 +54,7 @@ def concepts(vocabulary_id, codes)
     end
 
     def translate_vocab_id(vocabulary_id)
-      Array(vocabulary_id).map do |vocab_id|
-        vocab_translator[vocab_id.to_s]
-      end
+      return vocabulary_id
     end
 
     def vocab_translator
@@ -68,9 +66,8 @@ def vocab_translator
 
     def vocabularies
       lexicon_db[:vocabularies]
-        .select(Sequel[:omopv5_id].as(:id),
-                Sequel[:omopv5_id].as(:omopv5_vocabulary_id),
-                Sequel[:omopv4_id].as(:omopv4_vocabulary_id),
+        .select(Sequel[:id].as(:id),
+                Sequel[:id].as(:omopv5_vocabulary_id),
                 Sequel[:vocabulary_name].as(:vocabulary_short_name),
                 Sequel[:vocabulary_name].as(:vocabulary_full_name),
                 Sequel[:domain],

lib/conceptql/nodifier.rb

@@ -7,7 +7,7 @@ class Nodifier
     def initialize(opts={})
       @scope = opts[:scope] || Scope.new(opts.delete(:scope_opts) || {})
       @data_model = get_data_model(opts)
-      @database_type = opts[:database_type] || :impala
+      @database_type = opts[:database_type] || ConceptQL::DEFAULT_DATA_MODEL
       @algorithm_fetcher = opts[:algorithm_fetcher] || (proc do |alg|
         nil
       end)

lib/conceptql/operators/except.rb

@@ -23,9 +23,6 @@ def query(db)
           lquery.send("columns=", query_cols)
           rquery.send("columns=", query_cols)
 
-          if impala?
-            lquery = lquery.except_strategy(:not_exists, *(matching_columns + [:criterion_id, :criterion_domain]))
-          end
           lquery.select(*query_cols).except(rquery.select(*query_cols))
         end
       end

lib/conceptql/operators/operator.rb

@@ -392,10 +392,6 @@ def gdm?
         data_model == :gdm
       end
 
-      def impala?
-        database_type.to_sym == :impala
-      end
-
       def dm
         @dm ||= DataModel.for(self, nodifier)
       end
@@ -630,30 +626,6 @@ def add_warning(*args)
         warnings << args
       end
 
-      def needs_arguments_cte?(args)
-        impala? && arguments.length > 5000
-      end
-
-      def arguments_fix(db, args = nil)
-        args ||= arguments
-        return args unless needs_arguments_cte?(args)
-        args = args.dup
-        first_arg = Sequel.expr(args.shift).as(:arg)
-        args.unshift(first_arg)
-        args = args.map { |v| [v] }
-        args_cte = db.values(args)
-        args_cte_name = cte_name(:args)
-
-        if args_cte_name.is_a?(Sequel::SQL::QualifiedIdentifier)
-          args_cte_name = Sequel.identifier(args_cte_name.column)
-        end
-
-        # CTE here only used on impala due to needs_arguments_cte? above
-        db[args_cte_name]
-          .with(args_cte_name, args_cte, :no_temp_table=>true)
-          .select(:arg)
-      end
-
       def include_counts?(db, opts)
         !(no_db?(db, opts) || opts[:skip_counts])
       end

lib/conceptql/operators/vocabulary.rb

@@ -120,7 +120,7 @@ def where_clause(db)
         return { clinical_code_vocabulary_id: vocabulary_id } if select_all?
 
         where_conds = {vocabulary_id: vocabulary_id}
-        where_conds[:concept_code] = arguments_fix(db)
+        where_conds[:concept_code] = arguments
         concept_ids = db[:concepts].where(where_conds).select(:id)
 
         { clinical_code_concept_id: concept_ids }

lib/conceptql/rdbms.rb

@@ -1,12 +1,9 @@
 require_relative "rdbms/postgres"
-require_relative "rdbms/impala"
 
 module ConceptQL
   module Rdbms
     def self.generate(database_type)
       case database_type.to_sym
-      when :impala
-        ConceptQL::Rdbms::Impala.new
       when :postgres
         ConceptQL::Rdbms::Postgres.new
       else

lib/conceptql/utils.rb

@@ -70,7 +70,7 @@ def timed_capture(*commands)
         opts = extract_opts!(commands)
         timeout = opts.fetch(:timeout)
 
-        stdin, stdout, stderr, wait_thread = Open3.popen3(*commands)
+        stdin, stdout, _, wait_thread = Open3.popen3(*commands)
         wait_thread[:timed_out] = false
         stdin.puts opts[:stdin_data] if opts[:stdin_data]
         stdin.close
@@ -124,9 +124,6 @@ def assemble_date(*symbols)
         concatted_strings = Sequel.join(strings_with_dashes)
 
         date = concatted_strings
-        if opts[:database_type] == :impala
-          date = Sequel.cast(Sequel.function(:concat_ws, '-', *strings), DateTime)
-        end
 
         date
       end

lib/conceptql/vocabularies/behaviors/omopish.rb

@@ -9,7 +9,7 @@ def tables
 
         def where_clause(db)
           conds = { dm.source_vocabulary_id(domain) => vocabulary_id.to_i }
-          conds[dm.source_value_column(domain)] = arguments_fix(db) unless select_all?
+          conds[dm.source_value_column(domain)] = arguments unless select_all?
           conds
         end
 

test/annotations_test.rb

@@ -14,7 +14,7 @@
       seq_db = Sequel.connect(DB.opts.merge(search_path: 'bad_path'))
       db = ConceptQL::Database.new(seq_db)
       query = db.query(["union",["cpt", "00000", "99213"],["icd9", "000.00", "412"]])
-      query.scope_annotate(skip_counts: true).must_equal(
+      _(query.scope_annotate(skip_counts: true)).must_equal(
         {:errors=>{},
         :warnings=>{"cpt"=>[["unknown code(s)", "00000"]], "icd9"=>[["unknown code(s)", "000.00"]]},
         :counts=>{"cpt"=>{:procedure_occurrence=>{:rows=>0, :n=>0}},
@@ -27,7 +27,7 @@
       seq_db = Sequel.connect(DB.opts.merge(search_path: 'bad_path'))
       db = ConceptQL::Database.new(seq_db)
       query = db.query(["union",["cpt","0000"],["icd9", "00.00"]])
-      query.scope_annotate(skip_counts: true).must_equal(
+      _(query.scope_annotate(skip_counts: true)).must_equal(
         {:errors=>{},
          :warnings=>{"cpt"=>[["improperly formatted code", "0000"]], "icd9"=>[["improperly formatted code", "00.00"]]},
          :counts=>{"cpt"=>{:procedure_occurrence=>{:rows=>0, :n=>0}},

test/db.rb

@@ -1,29 +1,6 @@
 require 'sequelizer'
 
 DB = Object.new.extend(Sequelizer).db unless defined?(DB)
-if DB.database_type == :impala
-  # Make sure to issue USE statement for every new connection
-  ac = DB.pool.after_connect
-  DB.pool.after_connect = proc do |conn, server, db|
-    DB.send(:log_connection_execute, conn, "USE #{DB.opts[:database]}")
-    ac.call(conn, server, db) if ac
-  end
-
-  # Remove existing connnections, so that the next query will use a new connection
-  # that the USE statement has been executed on
-  DB.disconnect
-end
-
-if %w(omopv4 omopv4_plus).include?(ENV['CONCEPTQL_DATA_MODEL']) && !DB.table_exists?(:source_to_concept_map)
-  $stderr.puts <<END
-The source_to_concept_map table doesn't exist in this database,
-so it appears this doesn't include the necessary OMOP vocabulary
-data. Please review the README for how to setup the test database
-with the vocabulary, which needs to be done before running tests.
-END
-  exit 1
-end
-
 unless ENV["LEXICON_URL"]
   $stderr.puts <<END
 !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

test/db_helper.rb

@@ -5,7 +5,7 @@
 require "pp"
 require "fileutils"
 
-CDB = ConceptQL::Database.new(DB, :data_model=>ENV["CONCEPTQL_DATA_MODEL"].to_sym)
+CDB = ConceptQL::Database.new(DB, :data_model=>(ENV["CONCEPTQL_DATA_MODEL"] || ConceptQL::DEFAULT_DATA_MODEL).to_sym)
 DB.extension :error_sql
 
 PRINT_CONCEPTQL = ENV["CONCEPTQL_PRINT_SQL"]
@@ -91,7 +91,7 @@ def check_output(test_name, results, statement, has_windows = false)
     message += " (with windows)" if has_windows
     message += PP.pp(statement, "".dup, 10) if PRINT_CONCEPTQL
 
-    JSON.parse(results.to_json).must_equal(JSON.parse(expected), message)
+    _(JSON.parse(results.to_json)).must_equal(JSON.parse(expected), message)
     results
   end