From f47ec070d4d8501a1fce09f938a87da7d6a62379 Mon Sep 17 00:00:00 2001 From: Ryan Duryea Date: Mon, 9 Dec 2019 19:26:07 +0000 Subject: [PATCH] Work with new Lexicon tables This commit was supposed to address a small change in Lexicon, but resulted in tearing out a lot of defunct or deprecated code, namely: - Removed (almost) all references to Impala, refs #262 - Removed a lot of OMOP-related tests, refs #261 - More work on defaulting to GDM, refs #265 --- Gemfile | 1 - README.md | 25 +- Rakefile | 10 - lib/conceptql/database.rb | 2 +- lib/conceptql/lexicon.rb | 9 +- lib/conceptql/nodifier.rb | 2 +- lib/conceptql/operators/except.rb | 3 - lib/conceptql/operators/operator.rb | 28 -- lib/conceptql/operators/vocabulary.rb | 2 +- lib/conceptql/rdbms.rb | 3 - lib/conceptql/rdbms/impala.rb | 75 ---- lib/conceptql/utils.rb | 5 +- .../vocabularies/behaviors/omopish.rb | 2 +- test/annotations_test.rb | 4 +- test/db.rb | 23 - test/db_helper.rb | 4 +- test/db_setup.rb | 408 ------------------ test/db_teardown.rb | 33 -- .../scope/impala_date_range_under_gdm.txt | 1 - ...pala_date_range_under_gdm_source_vocab.txt | 1 - ...la_date_range_under_gdm_standard_vocab.txt | 1 - ...e_range_under_omopv4_plus_source_vocab.txt | 1 - ...range_under_omopv4_plus_standard_vocab.txt | 1 - .../scope/impala_window_table_under_gdm.txt | 1 - ...la_window_table_under_gdm_source_vocab.txt | 1 - ..._window_table_under_gdm_standard_vocab.txt | 1 - ...w_table_under_omopv4_plus_source_vocab.txt | 1 - ...table_under_omopv4_plus_standard_vocab.txt | 1 - ...indow_table_with_adjustments_under_gdm.txt | 1 - ...ith_adjustments_under_gdm_source_vocab.txt | 1 - ...h_adjustments_under_gdm_standard_vocab.txt | 1 - ...stments_under_omopv4_plus_source_vocab.txt | 1 - ...ments_under_omopv4_plus_standard_vocab.txt | 1 - test/helper.rb | 1 + test/lib/conceptql/data_model/gdm/gdm_test.rb | 4 +- test/lib/conceptql/date_adjuster_test.rb | 60 +-- test/lib/conceptql/lexicon_test.rb | 18 +- test/lib/conceptql/operators/except_test.rb | 30 -- test/lib/conceptql/operators/from_test.rb | 4 +- test/lib/conceptql/operators/gender_test.rb | 10 +- test/lib/conceptql/operators/icd9_test.rb | 22 - .../operators/information_periods_test.rb | 8 +- .../operators/multiple_vocabularies_test.rb | 10 +- .../operators/provider_filter_test.rb | 8 +- test/lib/conceptql/operators/read_test.rb | 8 +- .../conceptql/operators/vocabulary_test.rb | 32 +- test/lib/conceptql/query_test.rb | 32 +- test/lib/conceptql/rdbms/impala_test.rb | 44 -- test/lib/conceptql/rdbms/postgres_test.rb | 6 +- test/lib/conceptql/scope_test.rb | 173 +------- test/lib/conceptql/utils_test.rb | 11 +- 51 files changed, 123 insertions(+), 1012 deletions(-) delete mode 100644 lib/conceptql/rdbms/impala.rb delete mode 100644 test/db_setup.rb delete mode 100644 test/db_teardown.rb delete mode 100644 test/fixtures/scope/impala_date_range_under_gdm.txt delete mode 100644 test/fixtures/scope/impala_date_range_under_gdm_source_vocab.txt delete mode 100644 test/fixtures/scope/impala_date_range_under_gdm_standard_vocab.txt delete mode 100644 test/fixtures/scope/impala_date_range_under_omopv4_plus_source_vocab.txt delete mode 100644 test/fixtures/scope/impala_date_range_under_omopv4_plus_standard_vocab.txt delete mode 100644 test/fixtures/scope/impala_window_table_under_gdm.txt delete mode 100644 test/fixtures/scope/impala_window_table_under_gdm_source_vocab.txt delete mode 100644 test/fixtures/scope/impala_window_table_under_gdm_standard_vocab.txt delete mode 100644 test/fixtures/scope/impala_window_table_under_omopv4_plus_source_vocab.txt delete mode 100644 test/fixtures/scope/impala_window_table_under_omopv4_plus_standard_vocab.txt delete mode 100644 test/fixtures/scope/impala_window_table_with_adjustments_under_gdm.txt delete mode 100644 test/fixtures/scope/impala_window_table_with_adjustments_under_gdm_source_vocab.txt delete mode 100644 test/fixtures/scope/impala_window_table_with_adjustments_under_gdm_standard_vocab.txt delete mode 100644 test/fixtures/scope/impala_window_table_with_adjustments_under_omopv4_plus_source_vocab.txt delete mode 100644 test/fixtures/scope/impala_window_table_with_adjustments_under_omopv4_plus_standard_vocab.txt delete mode 100644 test/lib/conceptql/operators/except_test.rb delete mode 100644 test/lib/conceptql/operators/icd9_test.rb delete mode 100644 test/lib/conceptql/rdbms/impala_test.rb diff --git a/Gemfile b/Gemfile index 5324af20..cb5b64e7 100644 --- a/Gemfile +++ b/Gemfile @@ -3,7 +3,6 @@ source 'https://rubygems.org' # Specify your gem's dependencies in conceptql.gemspec gemspec gem "pg" -gem "sequel_impala", path: "../sequel_impala" group :test, :development do gem "pry" diff --git a/README.md b/README.md index 5aef2fa7..95a074f6 100644 --- a/README.md +++ b/README.md @@ -96,26 +96,19 @@ If you're feeling bold, feel free to try your hand at authoring a ConceptQL stat ### Setup -In order to run the tests for ConceptQL, you first have to create a database and load the OMOP vocabularies into it. You can use [loadmop](https://github.com/outcomesinsights/loadmop/tree/develop) do to so. +You must have the latest version of ConceptQL's test database. -Follow the instructions in [loadmop](https://github.com/outcomesinsights/loadmop/#preparation) to setup the vocabulary files correctly. +To set up this test database in PostgreSQL, execute the following command: -After loading the vocabulary data file into the database, -create an .env file in the root of the conceptql directory, -similar or identical to the one used for loadmop. Then -run `rake test_db_setup`. This will load the ConceptQL test -data into the database. This only needs to be done once, not -every time you run the tests. - -### Running +``` +curl -sSL http://chisel.test_data.jsaw.io | pigz -dc | psql +``` -If any of the CSV files in `test/data` has been updated since you -last updated the database, you should run `rake test_db_teardown test_db_setup` -to reset the data in the database. Updates to the CSV files in -`test/data` should be infrequent. +Then run the tests using: -After the test database has been setup, you can run the tests -using `rake`, as the default task is set to run the tests. +``` +SEQUELIZER_URL= SEQUELIZER_SEARCH_PATH=gdm_250 bundle exec ruby test/all.rb +``` ## Contributing diff --git a/Rakefile b/Rakefile index 5b5863e3..e5211c43 100644 --- a/Rakefile +++ b/Rakefile @@ -5,16 +5,6 @@ end ENV['CONCEPTQL_DATA_MODEL'] ||= ConceptQL::DEFAULT_DATA_MODEL.to_s -desc "Setup test database" -task :test_db_setup do - require_relative 'test/db_setup' -end - -desc "Setup test database" -task :test_db_teardown do - require_relative 'test/db_teardown' -end - run_spec = lambda do |data_model| sh "CONCEPTQL_DATA_MODEL=#{data_model} #{FileUtils::RUBY} test/all.rb" end diff --git a/lib/conceptql/database.rb b/lib/conceptql/database.rb index 61e59a7a..6a47c5b6 100644 --- a/lib/conceptql/database.rb +++ b/lib/conceptql/database.rb @@ -9,7 +9,7 @@ class Database def initialize(db, opts={}) @db = db - db_type = :impala + db_type = :postgres if db self.class.db_extensions(db) db_type = db.database_type.to_sym diff --git a/lib/conceptql/lexicon.rb b/lib/conceptql/lexicon.rb index 39d8ac58..a5d9a5e1 100644 --- a/lib/conceptql/lexicon.rb +++ b/lib/conceptql/lexicon.rb @@ -54,9 +54,7 @@ def concepts(vocabulary_id, codes) end def translate_vocab_id(vocabulary_id) - Array(vocabulary_id).map do |vocab_id| - vocab_translator[vocab_id.to_s] - end + return vocabulary_id end def vocab_translator @@ -68,9 +66,8 @@ def vocab_translator def vocabularies lexicon_db[:vocabularies] - .select(Sequel[:omopv5_id].as(:id), - Sequel[:omopv5_id].as(:omopv5_vocabulary_id), - Sequel[:omopv4_id].as(:omopv4_vocabulary_id), + .select(Sequel[:id].as(:id), + Sequel[:id].as(:omopv5_vocabulary_id), Sequel[:vocabulary_name].as(:vocabulary_short_name), Sequel[:vocabulary_name].as(:vocabulary_full_name), Sequel[:domain], diff --git a/lib/conceptql/nodifier.rb b/lib/conceptql/nodifier.rb index 8cb56014..ef32a513 100644 --- a/lib/conceptql/nodifier.rb +++ b/lib/conceptql/nodifier.rb @@ -7,7 +7,7 @@ class Nodifier def initialize(opts={}) @scope = opts[:scope] || Scope.new(opts.delete(:scope_opts) || {}) @data_model = get_data_model(opts) - @database_type = opts[:database_type] || :impala + @database_type = opts[:database_type] || ConceptQL::DEFAULT_DATA_MODEL @algorithm_fetcher = opts[:algorithm_fetcher] || (proc do |alg| nil end) diff --git a/lib/conceptql/operators/except.rb b/lib/conceptql/operators/except.rb index b9ad9709..a379d532 100644 --- a/lib/conceptql/operators/except.rb +++ b/lib/conceptql/operators/except.rb @@ -23,9 +23,6 @@ def query(db) lquery.send("columns=", query_cols) rquery.send("columns=", query_cols) - if impala? - lquery = lquery.except_strategy(:not_exists, *(matching_columns + [:criterion_id, :criterion_domain])) - end lquery.select(*query_cols).except(rquery.select(*query_cols)) end end diff --git a/lib/conceptql/operators/operator.rb b/lib/conceptql/operators/operator.rb index 423b1b66..f05846b5 100644 --- a/lib/conceptql/operators/operator.rb +++ b/lib/conceptql/operators/operator.rb @@ -392,10 +392,6 @@ def gdm? data_model == :gdm end - def impala? - database_type.to_sym == :impala - end - def dm @dm ||= DataModel.for(self, nodifier) end @@ -630,30 +626,6 @@ def add_warning(*args) warnings << args end - def needs_arguments_cte?(args) - impala? && arguments.length > 5000 - end - - def arguments_fix(db, args = nil) - args ||= arguments - return args unless needs_arguments_cte?(args) - args = args.dup - first_arg = Sequel.expr(args.shift).as(:arg) - args.unshift(first_arg) - args = args.map { |v| [v] } - args_cte = db.values(args) - args_cte_name = cte_name(:args) - - if args_cte_name.is_a?(Sequel::SQL::QualifiedIdentifier) - args_cte_name = Sequel.identifier(args_cte_name.column) - end - - # CTE here only used on impala due to needs_arguments_cte? above - db[args_cte_name] - .with(args_cte_name, args_cte, :no_temp_table=>true) - .select(:arg) - end - def include_counts?(db, opts) !(no_db?(db, opts) || opts[:skip_counts]) end diff --git a/lib/conceptql/operators/vocabulary.rb b/lib/conceptql/operators/vocabulary.rb index e9ad393b..001a2981 100644 --- a/lib/conceptql/operators/vocabulary.rb +++ b/lib/conceptql/operators/vocabulary.rb @@ -120,7 +120,7 @@ def where_clause(db) return { clinical_code_vocabulary_id: vocabulary_id } if select_all? where_conds = {vocabulary_id: vocabulary_id} - where_conds[:concept_code] = arguments_fix(db) + where_conds[:concept_code] = arguments concept_ids = db[:concepts].where(where_conds).select(:id) { clinical_code_concept_id: concept_ids } diff --git a/lib/conceptql/rdbms.rb b/lib/conceptql/rdbms.rb index 36edf6bc..5a2f1204 100644 --- a/lib/conceptql/rdbms.rb +++ b/lib/conceptql/rdbms.rb @@ -1,12 +1,9 @@ require_relative "rdbms/postgres" -require_relative "rdbms/impala" module ConceptQL module Rdbms def self.generate(database_type) case database_type.to_sym - when :impala - ConceptQL::Rdbms::Impala.new when :postgres ConceptQL::Rdbms::Postgres.new else diff --git a/lib/conceptql/rdbms/impala.rb b/lib/conceptql/rdbms/impala.rb deleted file mode 100644 index 484bb7c3..00000000 --- a/lib/conceptql/rdbms/impala.rb +++ /dev/null @@ -1,75 +0,0 @@ -require_relative "generic" - -module ConceptQL - module Rdbms - class Impala < Generic - SORT_BY_COLUMNS = %i(person_id window_id start_date end_date) - POSSIBLY_STATIC_COLUMNS = %i(criterion_table criterion_domain value_as_number) - - def cast_date(date) - Sequel.cast(date, DateTime) - end - - def semi_join(ds, table, *exprs) - ds = Sequel[ds] if ds.is_a?(Symbol) - table = Sequel[table] if table.is_a?(Symbol) - expr = exprs.inject(&:&) - ds.from_self(alias: :l) - .left_semi_join(table, expr, table_alias: :r) - .select_all(:l) - end - - # Impala is teh dumb in that it won't allow columns with constants to - # be part of the partition of a window function. - # - # Concatting other constants didn't seem to fix the problem - # - # Since we're partitioning by person_id at all times, it seems like a - # safe bet that we can append the person_id to any constant, making it - # no longer a constant, but still a viable column for partitioning - def partition_fix(column, qualifier=nil) - return column if ENV["CONCEPTQL_DISABLE_ORDER_BY_FIX"] == "true" - person_id = qualifier ? Sequel.qualify(qualifier, :person_id).cast_string : :person_id - person_id = Sequel.cast_string(person_id) - column = Sequel.expr(column).cast_string - column + '_' + person_id - end - - def uuid_items - items = %w(person_id criterion_id criterion_table).map do |column| - Sequel.cast_string(column.to_sym) - end - items << Sequel.function(:split_part, Sequel.cast_string(:start_date), " ", 1) - end - - def days_between(from_column, to_column) - Sequel.function(:datediff, cast_date(to_column), cast_date(from_column)) - end - - def create_options(scope, ds) - opts = { parquet: true } - if ENV["CONCEPTQL_SORT_TEMP_TABLES"] == "true" - sort_by_columns = if ds.opts[:values] - ds.opts[:values].first.map(&:alias) - else - SORT_BY_COLUMNS & scope.query_columns - end - opts[:sort_by] = sort_by_columns - end - opts - end - - def post_create(db, table_name) - db.compute_stats(table_name) - end - - def drop_options - { - cascade: true, - purge: true - } - end - end - end -end - diff --git a/lib/conceptql/utils.rb b/lib/conceptql/utils.rb index 767c8423..8341ca2b 100644 --- a/lib/conceptql/utils.rb +++ b/lib/conceptql/utils.rb @@ -70,7 +70,7 @@ def timed_capture(*commands) opts = extract_opts!(commands) timeout = opts.fetch(:timeout) - stdin, stdout, stderr, wait_thread = Open3.popen3(*commands) + stdin, stdout, _, wait_thread = Open3.popen3(*commands) wait_thread[:timed_out] = false stdin.puts opts[:stdin_data] if opts[:stdin_data] stdin.close @@ -124,9 +124,6 @@ def assemble_date(*symbols) concatted_strings = Sequel.join(strings_with_dashes) date = concatted_strings - if opts[:database_type] == :impala - date = Sequel.cast(Sequel.function(:concat_ws, '-', *strings), DateTime) - end date end diff --git a/lib/conceptql/vocabularies/behaviors/omopish.rb b/lib/conceptql/vocabularies/behaviors/omopish.rb index 27ce8689..4d83b62a 100644 --- a/lib/conceptql/vocabularies/behaviors/omopish.rb +++ b/lib/conceptql/vocabularies/behaviors/omopish.rb @@ -9,7 +9,7 @@ def tables def where_clause(db) conds = { dm.source_vocabulary_id(domain) => vocabulary_id.to_i } - conds[dm.source_value_column(domain)] = arguments_fix(db) unless select_all? + conds[dm.source_value_column(domain)] = arguments unless select_all? conds end diff --git a/test/annotations_test.rb b/test/annotations_test.rb index 77a83915..08f6e6d4 100644 --- a/test/annotations_test.rb +++ b/test/annotations_test.rb @@ -14,7 +14,7 @@ seq_db = Sequel.connect(DB.opts.merge(search_path: 'bad_path')) db = ConceptQL::Database.new(seq_db) query = db.query(["union",["cpt", "00000", "99213"],["icd9", "000.00", "412"]]) - query.scope_annotate(skip_counts: true).must_equal( + _(query.scope_annotate(skip_counts: true)).must_equal( {:errors=>{}, :warnings=>{"cpt"=>[["unknown code(s)", "00000"]], "icd9"=>[["unknown code(s)", "000.00"]]}, :counts=>{"cpt"=>{:procedure_occurrence=>{:rows=>0, :n=>0}}, @@ -27,7 +27,7 @@ seq_db = Sequel.connect(DB.opts.merge(search_path: 'bad_path')) db = ConceptQL::Database.new(seq_db) query = db.query(["union",["cpt","0000"],["icd9", "00.00"]]) - query.scope_annotate(skip_counts: true).must_equal( + _(query.scope_annotate(skip_counts: true)).must_equal( {:errors=>{}, :warnings=>{"cpt"=>[["improperly formatted code", "0000"]], "icd9"=>[["improperly formatted code", "00.00"]]}, :counts=>{"cpt"=>{:procedure_occurrence=>{:rows=>0, :n=>0}}, diff --git a/test/db.rb b/test/db.rb index 8208715f..e183ce4c 100644 --- a/test/db.rb +++ b/test/db.rb @@ -1,29 +1,6 @@ require 'sequelizer' DB = Object.new.extend(Sequelizer).db unless defined?(DB) -if DB.database_type == :impala - # Make sure to issue USE statement for every new connection - ac = DB.pool.after_connect - DB.pool.after_connect = proc do |conn, server, db| - DB.send(:log_connection_execute, conn, "USE #{DB.opts[:database]}") - ac.call(conn, server, db) if ac - end - - # Remove existing connnections, so that the next query will use a new connection - # that the USE statement has been executed on - DB.disconnect -end - -if %w(omopv4 omopv4_plus).include?(ENV['CONCEPTQL_DATA_MODEL']) && !DB.table_exists?(:source_to_concept_map) - $stderr.puts <ENV["CONCEPTQL_DATA_MODEL"].to_sym) +CDB = ConceptQL::Database.new(DB, :data_model=>(ENV["CONCEPTQL_DATA_MODEL"] || ConceptQL::DEFAULT_DATA_MODEL).to_sym) DB.extension :error_sql PRINT_CONCEPTQL = ENV["CONCEPTQL_PRINT_SQL"] @@ -91,7 +91,7 @@ def check_output(test_name, results, statement, has_windows = false) message += " (with windows)" if has_windows message += PP.pp(statement, "".dup, 10) if PRINT_CONCEPTQL - JSON.parse(results.to_json).must_equal(JSON.parse(expected), message) + _(JSON.parse(results.to_json)).must_equal(JSON.parse(expected), message) results end diff --git a/test/db_setup.rb b/test/db_setup.rb deleted file mode 100644 index 2e2abdbd..00000000 --- a/test/db_setup.rb +++ /dev/null @@ -1,408 +0,0 @@ -require_relative 'db' -require 'csv' - -if DB.opts[:database] && DB.opts[:database] !~ /test/ - $stderr.puts <true) do - Bignum :care_site_id, :primary_key=>true - Bignum :location_id, :null=>false - Bignum :organization_id, :null=>false - Bignum :place_of_service_concept_id - String :care_site_source_value, :size=>50 - String :place_of_service_source_value, :size=>50, :null=>false -end - -DB.create_table?(:cohort, :ignore_index_errors=>true) do - Bignum :cohort_id, :primary_key=>true - Bignum :cohort_concept_id, :null=>false - Date :cohort_start_date, :null=>false - Date :cohort_end_date - Bignum :subject_id, :null=>false - String :stop_reason, :size=>20 -end - -DB.create_table?(:location, :ignore_index_errors=>true) do - Bignum :location_id, :primary_key=>true - String :address_1, :size=>50 - String :address_2, :size=>50 - String :city, :size=>50 - String :state, :size=>2, :fixed=>true - String :zip, :size=>9 - String :county, :size=>20 - String :location_source_value, :size=>50 -end - -DB.create_table?(:provider, :ignore_index_errors=>true) do - Bignum :provider_id, :primary_key=>true - String :npi, :size=>20 - String :dea, :size=>20 - Bignum :specialty_concept_id - Bignum :care_site_id, :null=>false - String :provider_source_value, :size=>50, :null=>false - String :specialty_source_value, :size=>50 -end - -DB.create_table?(:organization, :ignore_index_errors=>true) do - Bignum :organization_id, :primary_key=>true - Bignum :place_of_service_concept_id - foreign_key :location_id, :location, :type=>:Bignum - String :organization_source_value, :size=>50, :null=>false - String :place_of_service_source_value, :size=>50 -end - -DB.create_table?(:person, :ignore_index_errors=>true) do - Bignum :person_id, :primary_key=>true - Bignum :gender_concept_id, :null=>false - Integer :year_of_birth, :null=>false - Integer :month_of_birth - Integer :day_of_birth - Bignum :race_concept_id - Bignum :ethnicity_concept_id - foreign_key :location_id, :location, :type=>:Bignum - foreign_key :provider_id, :provider, :type=>:Bignum - Bignum :care_site_id - String :person_source_value, :size=>50 - String :gender_source_value, :size=>50 - String :race_source_value, :size=>50 - String :ethnicity_source_value, :size=>50 -end - -DB.create_table?(:condition_era, :ignore_index_errors=>true) do - Bignum :condition_era_id, :primary_key=>true - foreign_key :person_id, :person, :type=>:Bignum, :null=>false - Bignum :condition_concept_id, :null=>false - Date :condition_era_start_date, :null=>false - Date :condition_era_end_date, :null=>false - Bignum :condition_type_concept_id, :null=>false - Integer :condition_occurrence_count -end - -DB.create_table?(:condition_occurrence, :ignore_index_errors=>true) do - Bignum :condition_occurrence_id, :primary_key=>true - foreign_key :person_id, :person, :type=>:Bignum, :null=>false - Bignum :condition_concept_id, :null=>false - Date :condition_start_date, :null=>false - Date :condition_end_date - Bignum :condition_type_concept_id, :null=>false - String :stop_reason, :size=>20 - Bignum :associated_provider_id - Bignum :visit_occurrence_id - String :condition_source_value, :size=>50 -end - -DB.create_table?(:death, :ignore_index_errors=>true) do - foreign_key :person_id, :person, :type=>:Bignum, :primary_key=>true - Date :death_date, :null=>false - Bignum :death_type_concept_id, :null=>false - Bignum :cause_of_death_concept_id - String :cause_of_death_source_value, :size=>50 -end - -DB.create_table?(:drug_era, :ignore_index_errors=>true) do - Bignum :drug_era_id, :primary_key=>true - foreign_key :person_id, :person, :type=>:Bignum, :null=>false - Bignum :drug_concept_id, :null=>false - Date :drug_era_start_date, :null=>false - Date :drug_era_end_date, :null=>false - Bignum :drug_type_concept_id, :null=>false - Integer :drug_exposure_count -end - -DB.create_table?(:drug_exposure, :ignore_index_errors=>true) do - Bignum :drug_exposure_id, :primary_key=>true - foreign_key :person_id, :person, :type=>:Bignum, :null=>false - Bignum :drug_concept_id, :null=>false - Date :drug_exposure_start_date, :null=>false - Date :drug_exposure_end_date - Bignum :drug_type_concept_id, :null=>false - String :stop_reason, :size=>20 - Integer :refills - Integer :quantity - Integer :days_supply - String :sig, :size=>500 - Bignum :prescribing_provider_id - Bignum :visit_occurrence_id - Bignum :relevant_condition_concept_id - String :drug_source_value, :size=>50 -end - -DB.create_table?(:observation, :ignore_index_errors=>true) do - Bignum :observation_id, :primary_key=>true - foreign_key :person_id, :person, :type=>:Bignum, :null=>false - Bignum :observation_concept_id, :null=>false - Date :observation_date, :null=>false - Date :observation_time - Float :value_as_number - String :value_as_string, :size=>60 - Bignum :value_as_concept_id - Bignum :unit_concept_id - Float :range_low - Float :range_high - Bignum :observation_type_concept_id, :null=>false - Bignum :associated_provider_id - Bignum :visit_occurrence_id - Bignum :relevant_condition_concept_id - String :observation_source_value, :size=>50 - String :unit_source_value, :size=>50 -end - -DB.create_table?(:observation_period, :ignore_index_errors=>true) do - Bignum :observation_period_id, :primary_key=>true - foreign_key :person_id, :person, :type=>:Bignum, :null=>false - Date :observation_period_start_date, :null=>false - Date :observation_period_end_date, :null=>false - Date :prev_ds_period_end_date -end - -DB.create_table?(:payer_plan_period, :ignore_index_errors=>true) do - Bignum :payer_plan_period_id, :primary_key=>true - foreign_key :person_id, :person, :type=>:Bignum, :null=>false - Date :payer_plan_period_start_date, :null=>false - Date :payer_plan_period_end_date, :null=>false - String :payer_source_value, :size=>50 - String :plan_source_value, :size=>50 - String :family_source_value, :size=>50 - Date :prev_ds_period_end_date -end - -DB.create_table?(:procedure_occurrence, :ignore_index_errors=>true) do - Bignum :procedure_occurrence_id, :primary_key=>true - foreign_key :person_id, :person, :type=>:Bignum, :null=>false - Bignum :procedure_concept_id, :null=>false - Date :procedure_date, :null=>false - Bignum :procedure_type_concept_id, :null=>false - Bignum :associated_provider_id - Bignum :visit_occurrence_id - Bignum :relevant_condition_concept_id - String :procedure_source_value, :size=>50 -end - -DB.create_table?(:visit_occurrence, :ignore_index_errors=>true) do - Bignum :visit_occurrence_id, :primary_key=>true - foreign_key :person_id, :person, :type=>:Bignum, :null=>false - Date :visit_start_date, :null=>false - Date :visit_end_date, :null=>false - Bignum :place_of_service_concept_id, :null=>false - Bignum :care_site_id - String :place_of_service_source_value, :size=>50 -end - -DB.create_table?(:drug_cost, :ignore_index_errors=>true) do - Bignum :drug_cost_id, :primary_key=>true - foreign_key :drug_exposure_id, :drug_exposure, :type=>:Bignum, :null=>false - Float :paid_copay - Float :paid_coinsurance - Float :paid_toward_deductible - Float :paid_by_payer - Float :paid_by_coordination_benefits - Float :total_out_of_pocket - Float :total_paid - Float :ingredient_cost - Float :dispensing_fee - Float :average_wholesale_price - Bignum :payer_plan_period_id -end - -DB.create_table?(:procedure_cost, :ignore_index_errors=>true) do - Bignum :procedure_cost_id, :primary_key=>true - foreign_key :procedure_occurrence_id, :procedure_occurrence, :type=>:Bignum, :null=>false - Float :paid_copay - Float :paid_coinsurance - Float :paid_toward_deductible - Float :paid_by_payer - Float :paid_by_coordination_benefits - Float :total_out_of_pocket - Float :total_paid - Bignum :disease_class_concept_id - Bignum :revenue_code_concept_id - Bignum :payer_plan_period_id - String :disease_class_source_value, :size=>50 - String :revenue_code_source_value, :size=>50 -end - -tables = [ - :care_site, - :cohort, - :location, - :provider, - :organization, - :person, - :condition_era, - :condition_occurrence, - :death, - :drug_era, - :drug_exposure, - :observation, - :observation_period, - :payer_plan_period, - :procedure_occurrence, - :visit_occurrence, - :drug_cost, - :procedure_cost -] - -tables.each do |t| - data = CSV.parse(File.binread("test/data/#{ENV['CONCEPTQL_DATA_MODEL']}/#{t}.csv")) - next if data.empty? - ds = DB[t] - - case ds.count - when 0 - # nothing - when data.length - next - else - ds.delete - end - - puts "Inserting data into #{t}" - - if DB.database_type == :impala - types = DB.schema(t).map{|_,s| s[:db_type]} - data = data.map do |row| - row.zip(types).map do |v, t| - Sequel.cast(v, t == 'double' ? 'float' : t) - end - end - end - - unless ds.count == data.length - ds.import(ds.columns, data, :slice=>1000) - end -end - -unless DB.database_type == :impala - puts "Creating indexes" - - DB.alter_table(:care_site) do - add_index [:location_id, :organization_id, :place_of_service_source_value], :name=>:care_site_location_id_organization_id_place_of_service_sour_key, :unique=>true, :ignore_add_index_errors=>true - add_index [:organization_id], :name=>:carsit_orgid, :ignore_add_index_errors=>true - add_index [:place_of_service_concept_id], :name=>:carsit_plaofserconid, :ignore_add_index_errors=>true - add_index [:location_id, :organization_id, :place_of_service_source_value, :care_site_id], :name=>:idx_care_site_org_sources, :ignore_add_index_errors=>true - end - - DB.alter_table(:cohort) do - add_index [:cohort_concept_id], :name=>:coh_cohconid, :ignore_add_index_errors=>true - add_index [:subject_id], :name=>:coh_subid, :ignore_add_index_errors=>true - end - - DB.alter_table(:location) do - add_index [:zip, :county], :name=>:location_zip_county_key, :unique=>true, :ignore_add_index_errors=>true - end - - DB.alter_table(:provider) do - add_index [:provider_source_value, :specialty_source_value, :provider_id, :care_site_id], :name=>:idx_provider_lkp, :ignore_add_index_errors=>true - add_index [:care_site_id], :name=>:pro_carsitid, :ignore_add_index_errors=>true - add_index [:specialty_concept_id], :name=>:pro_speconid, :ignore_add_index_errors=>true - end - - DB.alter_table(:organization) do - add_index [:location_id], :name=>:org_locid, :ignore_add_index_errors=>true - add_index [:place_of_service_concept_id], :name=>:org_plaofserconid, :ignore_add_index_errors=>true - end - - DB.alter_table(:person) do - add_index [:care_site_id], :name=>:per_carsitid, :ignore_add_index_errors=>true - add_index [:ethnicity_concept_id], :name=>:per_ethconid, :ignore_add_index_errors=>true - add_index [:gender_concept_id], :name=>:per_genconid, :ignore_add_index_errors=>true - add_index [:location_id], :name=>:per_locid, :ignore_add_index_errors=>true - add_index [:provider_id], :name=>:per_proid, :ignore_add_index_errors=>true - add_index [:race_concept_id], :name=>:per_racconid, :ignore_add_index_errors=>true - end - - DB.alter_table(:condition_era) do - add_index [:condition_concept_id], :name=>:conera_conconid, :ignore_add_index_errors=>true - add_index [:condition_type_concept_id], :name=>:conera_contypconid, :ignore_add_index_errors=>true - add_index [:person_id], :name=>:conera_perid, :ignore_add_index_errors=>true - end - - DB.alter_table(:condition_occurrence) do - add_index [:condition_concept_id], :name=>:cci, :ignore_add_index_errors=>true - add_index [:associated_provider_id], :name=>:conocc_assproid, :ignore_add_index_errors=>true - add_index [:condition_concept_id], :name=>:conocc_conconid, :ignore_add_index_errors=>true - add_index [:condition_source_value], :name=>:conocc_consouval, :ignore_add_index_errors=>true - add_index [:condition_type_concept_id], :name=>:conocc_contypconid, :ignore_add_index_errors=>true - add_index [:person_id], :name=>:conocc_perid, :ignore_add_index_errors=>true - add_index [:visit_occurrence_id], :name=>:conocc_visoccid, :ignore_add_index_errors=>true - add_index [:visit_occurrence_id], :name=>:voi, :ignore_add_index_errors=>true - end - - DB.alter_table(:death) do - add_index [:cause_of_death_concept_id], :name=>:dea_cauofdeaconid, :ignore_add_index_errors=>true - add_index [:death_type_concept_id], :name=>:dea_deatypconid, :ignore_add_index_errors=>true - end - - DB.alter_table(:drug_era) do - add_index [:drug_concept_id], :name=>:druera_druconid, :ignore_add_index_errors=>true - add_index [:drug_type_concept_id], :name=>:druera_drutypconid, :ignore_add_index_errors=>true - add_index [:person_id], :name=>:druera_perid, :ignore_add_index_errors=>true - end - - DB.alter_table(:drug_exposure) do - add_index [:drug_concept_id], :name=>:druexp_druconid, :ignore_add_index_errors=>true - add_index [:drug_type_concept_id], :name=>:druexp_drutypconid, :ignore_add_index_errors=>true - add_index [:person_id], :name=>:druexp_perid, :ignore_add_index_errors=>true - add_index [:prescribing_provider_id], :name=>:druexp_preproid, :ignore_add_index_errors=>true - add_index [:relevant_condition_concept_id], :name=>:druexp_relconconid, :ignore_add_index_errors=>true - add_index [:visit_occurrence_id], :name=>:druexp_visoccid, :ignore_add_index_errors=>true - end - - DB.alter_table(:observation) do - add_index [:associated_provider_id], :name=>:obs_assproid, :ignore_add_index_errors=>true - add_index [:observation_concept_id], :name=>:obs_obsconid, :ignore_add_index_errors=>true - add_index [:observation_type_concept_id], :name=>:obs_obstypconid, :ignore_add_index_errors=>true - add_index [:person_id], :name=>:obs_perid, :ignore_add_index_errors=>true - add_index [:relevant_condition_concept_id], :name=>:obs_relconconid, :ignore_add_index_errors=>true - add_index [:unit_concept_id], :name=>:obs_uniconid, :ignore_add_index_errors=>true - add_index [:value_as_concept_id], :name=>:obs_valasconid, :ignore_add_index_errors=>true - add_index [:visit_occurrence_id], :name=>:obs_visoccid, :ignore_add_index_errors=>true - end - - DB.alter_table(:observation_period) do - add_index [:person_id, :observation_period_start_date, :observation_period_end_date], :name=>:idx_observation_period_lkp, :ignore_add_index_errors=>true - add_index [:person_id], :name=>:obsper_perid, :ignore_add_index_errors=>true - end - - DB.alter_table(:payer_plan_period) do - add_index [:person_id, :plan_source_value, :payer_plan_period_start_date, :payer_plan_period_end_date], :name=>:idx_payer_plan_period_lkp, :ignore_add_index_errors=>true - add_index [:person_id], :name=>:payplaper_perid, :ignore_add_index_errors=>true - end - - DB.alter_table(:procedure_occurrence) do - add_index [:associated_provider_id], :name=>:proocc_assproid, :ignore_add_index_errors=>true - add_index [:person_id], :name=>:proocc_perid, :ignore_add_index_errors=>true - add_index [:procedure_concept_id], :name=>:proocc_proconid, :ignore_add_index_errors=>true - add_index [:procedure_source_value], :name=>:proocc_prosouval, :ignore_add_index_errors=>true - add_index [:procedure_type_concept_id], :name=>:proocc_protypconid, :ignore_add_index_errors=>true - add_index [:relevant_condition_concept_id], :name=>:proocc_relconconid, :ignore_add_index_errors=>true - add_index [:visit_occurrence_id], :name=>:proocc_visoccid, :ignore_add_index_errors=>true - end - - DB.alter_table(:visit_occurrence) do - add_index [:person_id, :visit_start_date, :place_of_service_concept_id], :name=>:visit_occurrence_person_id_visit_start_date_place_of_servic_key, :ignore_add_index_errors=>true - add_index [:care_site_id], :name=>:visocc_carsitid, :ignore_add_index_errors=>true - add_index [:person_id], :name=>:visocc_perid, :ignore_add_index_errors=>true - add_index [:place_of_service_concept_id], :name=>:visocc_plaofserconid, :ignore_add_index_errors=>true - end - - DB.alter_table(:drug_cost) do - add_index [:drug_exposure_id], :name=>:drucos_druexpid, :ignore_add_index_errors=>true - add_index [:payer_plan_period_id], :name=>:drucos_payplaperid, :ignore_add_index_errors=>true - end - - DB.alter_table(:procedure_cost) do - add_index [:disease_class_concept_id], :name=>:procos_disclaconid, :ignore_add_index_errors=>true - add_index [:payer_plan_period_id], :name=>:procos_payplaperid, :ignore_add_index_errors=>true - add_index [:procedure_occurrence_id], :name=>:procos_prooccid, :ignore_add_index_errors=>true - add_index [:revenue_code_concept_id], :name=>:procos_revcodconid, :ignore_add_index_errors=>true - end -end diff --git a/test/db_teardown.rb b/test/db_teardown.rb deleted file mode 100644 index e1a77465..00000000 --- a/test/db_teardown.rb +++ /dev/null @@ -1,33 +0,0 @@ -require_relative 'db' - -if DB.opts[:database] && DB.opts[:database] !~ /test/ - $stderr.puts < "38004458" } ]).sql.must_match(/associated_provider_id/) + _(db.query([ "provider_filter", [ "icd9", "412" ], { "specialties" => "38004458" } ]).sql).must_match(/associated_provider_id/) end it "should find associated_provider_id in procedure_occurrence table" do db = ConceptQL::Database.new(Sequel.mock(host: :postgres), data_model: dm) - db.query([ "provider_filter", [ "cpt", "99214" ], { "specialties" => "38004458" } ]).sql.must_match(/associated_provider_id/) + _(db.query([ "provider_filter", [ "cpt", "99214" ], { "specialties" => "38004458" } ]).sql).must_match(/associated_provider_id/) end it "should add no columns in person table" do db = ConceptQL::Database.new(Sequel.mock(host: :postgres), data_model: dm) - db.query([ "provider_filter", [ "person" ], { "specialties" => "38004458" } ]).sql.wont_match(/"provider_id"\s*AS/i) + _(db.query([ "provider_filter", [ "person" ], { "specialties" => "38004458" } ]).sql).wont_match(/"provider_id"\s*AS/i) end it "should find associated_provider_id in procedure_occurrence table" do db = ConceptQL::Database.new(Sequel.mock(host: :postgres), data_model: dm) - db.query([ "provider_filter", [ "ndc", "0123456789" ], { "specialties" => "38004458" } ]).sql.must_match(/prescribing_provider_id/) + _(db.query([ "provider_filter", [ "ndc", "0123456789" ], { "specialties" => "38004458" } ]).sql).must_match(/prescribing_provider_id/) end end end diff --git a/test/lib/conceptql/operators/read_test.rb b/test/lib/conceptql/operators/read_test.rb index 18528a43..c75e652e 100644 --- a/test/lib/conceptql/operators/read_test.rb +++ b/test/lib/conceptql/operators/read_test.rb @@ -2,18 +2,18 @@ describe ConceptQL::Operators::Read do it "be present in list of operators" do - ConceptQL::Operators.operators[:omopv4_plus]["read"].must_equal ConceptQL::Operators::Read + _(ConceptQL::Operators.operators[:omopv4_plus]["read"]).must_equal ConceptQL::Operators::Read end it "should produce correct SQL under omopv4_plus" do db = ConceptQL::Database.new(Sequel.mock(host: :postgres), data_model: :omopv4_plus) - db.query(["read", "xyz"]).sql.must_match %Q{"observation_source_vocabulary_id" = 17} - db.query(["read", "xyz"]).sql.must_match %Q{"observation_source_value" IN ('xyz')} + _(db.query(["read", "xyz"]).sql).must_match %Q{"observation_source_vocabulary_id" = 17} + _(db.query(["read", "xyz"]).sql).must_match %Q{"observation_source_value" IN ('xyz')} end it "should include measurement columns under GDM" do db = ConceptQL::Database.new(Sequel.mock(host: :postgres), data_model: :gdm) - db.query(["read", "xyz"]).operator.required_columns.must_include(:range_high) + _(db.query(["read", "xyz"]).operator.required_columns).must_include(:range_high) end end diff --git a/test/lib/conceptql/operators/vocabulary_test.rb b/test/lib/conceptql/operators/vocabulary_test.rb index d6381c22..fc8727ee 100644 --- a/test/lib/conceptql/operators/vocabulary_test.rb +++ b/test/lib/conceptql/operators/vocabulary_test.rb @@ -25,11 +25,6 @@ assert ConceptQL::Operators.operators[:gdm]["ndc"].to_metadata("icd9cm")[:predominant_domains] == [["drug_exposure"]] end - it "should populate known vocabularies from file in omopv4_plus" do - op_names = ConceptQL::Nodifier.new(data_model: :omopv4_plus).to_metadata.map { |_, v| v[:preferred_name] } - op_names.must_include("ATC") - end - describe "under gdm" do let :cdb do ConceptQL::Database.new(Sequel.mock(host: :postgres), data_model: :gdm) @@ -41,27 +36,17 @@ end it "should produce correct SQL" do - cdb.query(["admsrce", "12"]).sql.must_equal "SELECT * FROM (SELECT * FROM (SELECT \"patient_id\" AS \"person_id\", \"id\" AS \"criterion_id\", CAST('clinical_codes' AS text) AS \"criterion_table\", CAST('condition_occurrence' AS text) AS \"criterion_domain\", \"start_date\", \"end_date\", CAST(\"clinical_code_source_value\" AS text) AS \"source_value\", CAST(\"clinical_code_vocabulary_id\" AS text) AS \"source_vocabulary_id\" FROM \"clinical_codes\" WHERE (\"clinical_code_concept_id\" IN (SELECT \"id\" FROM \"concepts\" WHERE ((\"vocabulary_id\" = 'ADMSRCE') AND (\"concept_code\" IN ('12')))))) AS \"t1\") AS \"t1\"" + _(cdb.query(["admsrce", "12"]).sql).must_equal "SELECT * FROM (SELECT * FROM (SELECT \"patient_id\" AS \"person_id\", \"id\" AS \"criterion_id\", CAST('clinical_codes' AS text) AS \"criterion_table\", CAST('condition_occurrence' AS text) AS \"criterion_domain\", \"start_date\", \"end_date\", CAST(\"clinical_code_source_value\" AS text) AS \"source_value\", CAST(\"clinical_code_vocabulary_id\" AS text) AS \"source_vocabulary_id\" FROM \"clinical_codes\" WHERE (\"clinical_code_concept_id\" IN (SELECT \"id\" FROM \"concepts\" WHERE ((\"vocabulary_id\" = 'ADMSRCE') AND (\"concept_code\" IN ('12')))))) AS \"t1\") AS \"t1\"" end it "should produce correct SQL for older selection operators" do - cdb.query(["icd9", "412"]).sql.must_equal "SELECT * FROM (SELECT * FROM (SELECT \"patient_id\" AS \"person_id\", \"id\" AS \"criterion_id\", CAST('clinical_codes' AS text) AS \"criterion_table\", CAST('condition_occurrence' AS text) AS \"criterion_domain\", \"start_date\", \"end_date\", CAST(\"clinical_code_source_value\" AS text) AS \"source_value\", CAST(\"clinical_code_vocabulary_id\" AS text) AS \"source_vocabulary_id\" FROM \"clinical_codes\" WHERE (\"clinical_code_concept_id\" IN (SELECT \"id\" FROM \"concepts\" WHERE ((\"vocabulary_id\" = 'ICD9CM') AND (\"concept_code\" IN ('412')))))) AS \"t1\") AS \"t1\"" + _(cdb.query(["icd9", "412"]).sql).must_equal "SELECT * FROM (SELECT * FROM (SELECT \"patient_id\" AS \"person_id\", \"id\" AS \"criterion_id\", CAST('clinical_codes' AS text) AS \"criterion_table\", CAST('condition_occurrence' AS text) AS \"criterion_domain\", \"start_date\", \"end_date\", CAST(\"clinical_code_source_value\" AS text) AS \"source_value\", CAST(\"clinical_code_vocabulary_id\" AS text) AS \"source_vocabulary_id\" FROM \"clinical_codes\" WHERE (\"clinical_code_concept_id\" IN (SELECT \"id\" FROM \"concepts\" WHERE ((\"vocabulary_id\" = 'ICD9CM') AND (\"concept_code\" IN ('412')))))) AS \"t1\") AS \"t1\"" end it "should produce correct SQL for select all" do - cdb.query(["atc", "*"]).sql.must_match %Q{"clinical_code_vocabulary_id" = 'ATC'} - end - - end - - describe "under omopv4_plus" do - let :cdb do - ConceptQL::Database.new(Sequel.mock(host: :postgres), data_model: :omopv4_plus) + _(cdb.query(["atc", "*"]).sql).must_match %Q{"clinical_code_vocabulary_id" = 'ATC'} end - it "should produce correct SQL for select all under omopv4_plus" do - cdb.query(["atc", "*"]).sql.must_match %Q{"drug_source_vocabulary_id" = 21} - end end it "should read operators from a custom file" do @@ -70,8 +55,8 @@ end) Tempfile.create("blah.csv") do |f| CSV(f) do |csv| - csv << %w(id omopv4_vocabulary_id vocabulary_full_name vocabulary_short_name domain hidden format_regexp) - csv << %w(test 0 test_full test_short test_domain) + [nil, nil] + csv << %w(id vocabulary_full_name vocabulary_short_name domain hidden format_regexp) + csv << %w(test test_full test_short test_domain) + [nil, nil] end f.rewind ConceptQL.stub(:custom_vocabularies_file_path, Pathname.new(f.path)) do @@ -90,12 +75,13 @@ before do db.create_table!(:vocabularies) do - String :omopv4_id - String :omopv5_id + String :id String :vocabulary_name String :domain end - db[:vocabularies].multi_insert([{ omopv4_id: "10000000", omopv5_id: "EXAMPLE", vocabulary_name: "Example Vocabulary", domain: "measurement" }]) + db[:vocabularies].multi_insert([ + { id: "EXAMPLE", vocabulary_name: "Example Vocabulary", domain: "measurement" } + ]) end it "should read from lexicon" do diff --git a/test/lib/conceptql/query_test.rb b/test/lib/conceptql/query_test.rb index cfd4e10d..56d10ff5 100644 --- a/test/lib/conceptql/query_test.rb +++ b/test/lib/conceptql/query_test.rb @@ -2,21 +2,21 @@ describe ConceptQL::Query do it "should handle errors in the root operator" do - query( + _(query( [:foo] - ).annotate.must_equal( + ).annotate).must_equal( ["foo", {:annotation=>{:counts=>{:invalid=>{:rows=>0, :n=>0}}, :errors=>[["invalid operator", "foo"]]}, :name=>"Invalid"}] ) end it "should handle query_cols for non-CDM tables" do - query( + _(query( [:from, "other_table"] - ).query_cols.must_equal(ConceptQL::Scope::DEFAULT_COLUMNS.keys) + ).query_cols).must_equal(ConceptQL::Scope::DEFAULT_COLUMNS.keys) end it "should raise error if attempting to execute invalid recall" do - proc do + _(proc do criteria_ids( ["after", {:left=>["during", @@ -30,7 +30,7 @@ {:left=>["cpt", "84156", "84166", "86335", "84155", "84165", "86334", "83883", "81264", "82784", "82785", "82787", "82040", "82232", "77074", "77075", "83615", {"label"=>"Other Tests"}], :right=>["recall", "Meyloma 90-day Lookback"]}]]]}] ) - end.must_raise + end).must_raise end describe "#sql(:formatted)" do @@ -39,11 +39,11 @@ end it "should produce formatted SQL" do - cdb.query([:icd9, "412"]).sql(:formatted).must_match(" ") + _(cdb.query([:icd9, "412"]).sql(:formatted)).must_match(" ") end it "should timeout after 10 seconds if can't parse" do - cdb.query(json_fixture(:sqlformat_killer)).sql(:formatted).wont_match(/ /) + _(cdb.query(json_fixture(:sqlformat_killer)).sql(:formatted)).wont_match(/ /) end describe "with temp tables" do @@ -52,7 +52,7 @@ end it "should use CREATE TABLE statements" do - cdb.query([:icd9, "412", label: "l"]).sql(:formatted, :create_tables).must_match(/CREATE TABLE/) + _(cdb.query([:icd9, "412", label: "l"]).sql(:formatted, :create_tables)).must_match(/CREATE TABLE/) end end @@ -65,7 +65,7 @@ if ENV["CONCEPTQL_AVOID_CTES"] == "true" skip else - cdb.query([:icd9, "412", label: "l"]).sql(:formatted).must_match(/WITH/) + _(cdb.query([:icd9, "412", label: "l"]).sql(:formatted)).must_match(/WITH/) end end end @@ -88,13 +88,13 @@ "ICD-9 CM 250.02" ] end - query.code_list.map(&:to_s).must_equal(expected) + _(query.code_list.map(&:to_s)).must_equal(expected) end it "should handle nil for preferred name" do db = ConceptQL::Database.new(nil) query = db.query(["revenue_code", "0100"]) - query.code_list.map(&:to_s).must_equal([ + _(query.code_list.map(&:to_s)).must_equal([ "Revenue Code 0100: All-Inclusive Room and Board Plus Ancillary" ]) end @@ -116,14 +116,14 @@ "ICD-9 CM 250.02" ] end - query.code_list.map(&:to_s).must_equal(expected) + _(query.code_list.map(&:to_s)).must_equal(expected) end it "should return asterisk when selecting all" do seq_db = Sequel.connect(DB.opts.merge(search_path: 'bad_path')) db = ConceptQL::Database.new(seq_db) query = db.query(["union",["cpt","*"],["icd9", "250.00", "*"]]) - query.code_list(seq_db).map(&:to_s).must_equal([ + _(query.code_list(seq_db).map(&:to_s)).must_equal([ "CPT *: ALL CODES", "ICD-9 CM *: ALL CODES" ]) @@ -132,7 +132,7 @@ it "should return codes from vocabulary-based operators even with no db" do db = ConceptQL::Database.new(nil) query = db.query(["union", ["cpt_or_hcpcs","99214"], ["ATC", "*"]]) - query.code_list(nil).map(&:to_s).must_equal([ + _(query.code_list(nil).map(&:to_s)).must_equal([ "CPT or HCPCS 99214: Office or other outpatient visit for the evaluation and management of an established patient, which requires at least 2 of these 3 key components: A detailed history; A detailed examination; Medical decision making of moderate complexity. Counseling and/o", "WHO ATC *: ALL CODES" ]) @@ -140,7 +140,7 @@ it "should return codes even if the code doesn't exist in the database" do query = CDB.query(["hcpcs", "A0000", "A0021"]) - query.code_list.map(&:code).must_equal(["A0000", "A0021"]) + _(query.code_list.map(&:code)).must_equal(["A0000", "A0021"]) end end end diff --git a/test/lib/conceptql/rdbms/impala_test.rb b/test/lib/conceptql/rdbms/impala_test.rb deleted file mode 100644 index 871cc66f..00000000 --- a/test/lib/conceptql/rdbms/impala_test.rb +++ /dev/null @@ -1,44 +0,0 @@ -require_relative '../../../helper' - -describe ConceptQL::Rdbms::Impala do - let(:db) { Sequel.mock(host: :impala) } - let(:rdbms) { ConceptQL::Rdbms::Impala.new } - - describe "#days_between" do - it "should use datediff function" do - result = db.literal(rdbms.days_between('2001-01-01', :date_column)) - - result.must_match("`date_column`") - result.must_match("CAST('2001-01-01' AS timestamp)") - result.must_match("datediff") - end - end - - describe "#partition_fix" do - it "should apply fix by default" do - result = db.literal(rdbms.partition_fix(:some_column)) - - result.must_match("concat") - result.must_match("AS string") - result.must_match("'_'") - result.must_match("some_column") - result.must_match("person_id") - end - - it "should NOT apply fix if CONCEPTQL_DISABLE_ORDER_BY_FIX is set to 'true'" do - begin - save_disable = ENV["CONCEPTQL_DISABLE_ORDER_BY_FIX"] - ENV["CONCEPTQL_DISABLE_ORDER_BY_FIX"] = "true" - result = db.literal(rdbms.partition_fix(:some_column)) - - result.must_match("some_column") - result.wont_match("concat") - result.wont_match("AS string") - result.wont_match("'_'") - result.wont_match("person_id") - ensure - ENV["CONCEPTQL_DISABLE_ORDER_BY_FIX"] = save_disable - end - end - end -end diff --git a/test/lib/conceptql/rdbms/postgres_test.rb b/test/lib/conceptql/rdbms/postgres_test.rb index f0e652f2..39279ea5 100644 --- a/test/lib/conceptql/rdbms/postgres_test.rb +++ b/test/lib/conceptql/rdbms/postgres_test.rb @@ -8,9 +8,9 @@ it "should use subtractionb between two dates" do result = db.literal(rdbms.days_between('2001-01-01', :date_column)) - result.must_match('"date_column"') - result.must_match("CAST('2001-01-01' AS date)") - result.must_match("-") + _(result).must_match('"date_column"') + _(result).must_match("CAST('2001-01-01' AS date)") + _(result).must_match("-") end end end diff --git a/test/lib/conceptql/scope_test.rb b/test/lib/conceptql/scope_test.rb index b22e1a76..c6275659 100755 --- a/test/lib/conceptql/scope_test.rb +++ b/test/lib/conceptql/scope_test.rb @@ -7,7 +7,7 @@ def check_sequel(query, name) file.dirname.mkpath file.write(actual_sql) end - actual_sql.must_equal file.read + _(actual_sql).must_equal(file.read) end describe ConceptQL::Scope do @@ -25,7 +25,7 @@ def check_sequel(query, name) skip else db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :gdm, force_temp_tables: true, scratch_database: "jigsaw_temp") - db.query(["ADMSRCE", "12", {label: "test label"}], opts).sql.must_match /jtemp123456/ + _(db.query(["ADMSRCE", "12", {label: "test label"}], opts).sql).must_match /jtemp123456/ end end end @@ -93,32 +93,32 @@ def check_sequel(query, name) it "should not limit selection on person table" do db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - db.query(["person", true], opts).sql.wont_match(/EXISTS/) + _(db.query(["person", true], opts).sql).wont_match(/EXISTS/) end it "should limit selection on condition_occurrence table" do db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - db.query(["visit_occurrence", true], opts).sql.must_match(/inner join/i) + _(db.query(["visit_occurrence", true], opts).sql).must_match(/inner join/i) end it "should not apply to inner query of revenue code operator" do db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - db.query(["revenue_code", "0450"], opts).sql.downcase.scan(/inner join/i).count.must_equal 1 + _(db.query(["revenue_code", "0450"], opts).sql.downcase.scan(/inner join/i).count).must_equal 1 end it "should have revenue code search revenue_code_source_value" do db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - db.query(["revenue_code", "0450"], opts).sql.downcase.must_match(/revenue_code_source_value/i) + _(db.query(["revenue_code", "0450"], opts).sql.downcase).must_match(/revenue_code_source_value/i) end it "should not apply to inner query of drg operator" do db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - db.query(["drg", "829"], opts).sql.downcase.scan(/inner join/i).count.must_equal 1 + _(db.query(["drg", "829"], opts).sql.downcase.scan(/inner join/i).count).must_equal 1 end it "should have drg code search disease_class_source_value" do db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - db.query(["drg", "829"], opts).sql.downcase.must_match(/disease_class_source_value/i) + _(db.query(["drg", "829"], opts).sql.downcase).must_match(/disease_class_source_value/i) end end @@ -153,161 +153,4 @@ def check_sequel(query, name) end end end - - describe "using impala" do - - let(:host) { :impala } - - describe "with date literal windows" do - let(:opts) do - { scope_opts: { start_date: "2001-01-01", end_date: "2001-12-31" } } - end - - it "should limit selection by date range under gdm" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :gdm) - check_sequel(db.query(["ADMSRCE", "12"], opts), :impala_date_range_under_gdm) - end - - it "should limit selection by date range under omopv4_plus for source vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - check_sequel(db.query(["icd9", "412"], opts), :impala_date_range_under_omopv4_plus_source_vocab) - end - - it "should limit selection by date range under gdm with old source vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :gdm) - check_sequel(db.query(["icd9", "412"], opts), :impala_date_range_under_gdm_source_vocab) - end - - it "should limit selection by date range under omopv4_plus for standard vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - check_sequel(db.query(["cpt", "99214"], opts), :impala_date_range_under_omopv4_plus_standard_vocab) - end - - it "should limit selection by date range under gdm with old standard vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :gdm) - check_sequel(db.query(["cpt", "99214"], opts), :impala_date_range_under_gdm_standard_vocab) - end - end - - describe "with windows from another table" do - let(:opts) do - { scope_opts: { window_table: :jtemp } } - end - - it "should limit selection by date range under gdm" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :gdm) - check_sequel(db.query(["ADMSRCE", "12"], opts), :impala_window_table_under_gdm) - end - - it "should limit selection by date range under omopv4_plus for source vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - check_sequel(db.query(["icd9", "412"], opts), :impala_window_table_under_omopv4_plus_source_vocab) - end - - it "should limit selection by date range under gdm with old source vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :gdm) - check_sequel(db.query(["icd9", "412"], opts), :impala_window_table_under_gdm_source_vocab) - end - - it "should limit selection by date range under omopv4_plus for standard vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - check_sequel(db.query(["cpt", "99214"], opts), :impala_window_table_under_omopv4_plus_standard_vocab) - end - - it "should limit selection by date range under gdm with old standard vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :gdm) - check_sequel(db.query(["cpt", "99214"], opts), :impala_window_table_under_gdm_standard_vocab) - end - - it "should limit selection on person table" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - db.query(["person", true], opts).sql.must_match(/JOIN/) - end - - it "should limit selection on condition_occurrence table" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - db.query(["visit_occurrence", true], opts).sql.must_match(/JOIN/) - end - - it "avoids infinite recursion when processing CTEs" do - statement = [:any_overlap, - {:left=> - [:from, - Sequel[:jigsaw_temp][:jtemp1c4qd4w_baseline_windows], - {:query_cols=> - [:person_id, - :criterion_id, - :criterion_table, - :criterion_domain, - :start_date, - :end_date, - :source_value, - :source_vocabulary_id, - :uuid]}], - :right=> - ["after", - {"left"=> - ["one_in_two_out", - ["during", - {"left"=>["union", ["icd9", "198.5"], ["icd10", "C79.51"]], - "right"=> - ["date_range", {"start"=>"2015-01-01", "end"=>"2016-12-31"}]}], - {"inpatient_return_date"=>"Discharge Date", - "outpatient_minimum_gap"=>"1y", - "outpatient_event_to_return"=>"Initial Event", - "outpatient_maximum_gap"=>"2y"}], - "right"=>["time_window", ["person"], {"start"=>"+18y-1d", "end"=>""}]}]}] - - opts = { - force_temp_tables: true, - scratch_database: :scratch, - scope_opts: { - window_table: :baseline_windows - } - } - - q = ConceptQL::Database.new(Sequel.mock(host: :impala), opts).query(statement) - - begin - result = Timeout::timeout(2) do - q.sql_statements - end - pass - rescue Timeout::Error - flunk - end - end - end - - describe "with windows from another table, along with adjustments" do - let(:opts) do - { scope_opts: { window_table: :jtemp, adjust_window_start: "-30d", adjust_window_end: "1m"} } - end - - it "should limit selection by date range under gdm" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :gdm) - check_sequel(db.query(["ADMSRCE", "12"], opts), :impala_window_table_with_adjustments_under_gdm) - end - - it "should limit selection by date range under omopv4_plus for source vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - check_sequel(db.query(["icd9", "412"], opts), :impala_window_table_with_adjustments_under_omopv4_plus_source_vocab) - end - - it "should limit selection by date range under gdm with old source vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :gdm) - check_sequel(db.query(["icd9", "412"], opts), :impala_window_table_with_adjustments_under_gdm_source_vocab) - end - - it "should limit selection by date range under omopv4_plus for standard vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :omopv4_plus) - check_sequel(db.query(["cpt", "99214"], opts), :impala_window_table_with_adjustments_under_omopv4_plus_standard_vocab) - end - - it "should limit selection by date range under gdm with old standard vocab operator" do - db = ConceptQL::Database.new(Sequel.mock(host: host), data_model: :gdm) - check_sequel(db.query(["cpt", "99214"], opts), :impala_window_table_with_adjustments_under_gdm_standard_vocab) - end - end - end end diff --git a/test/lib/conceptql/utils_test.rb b/test/lib/conceptql/utils_test.rb index 6b18278c..fdb11f01 100644 --- a/test/lib/conceptql/utils_test.rb +++ b/test/lib/conceptql/utils_test.rb @@ -13,7 +13,7 @@ :b => [ "c", :d => "e" ] } } - ConceptQL::Utils.rekey(h).must_equal expected + _(ConceptQL::Utils.rekey(h)).must_equal expected end it "should symbolize all values if option is set" do @@ -27,7 +27,7 @@ :b => [ :c, :d => :e ] } } - ConceptQL::Utils.rekey(h, rekey_values: true).must_equal expected + _(ConceptQL::Utils.rekey(h, rekey_values: true)).must_equal expected end end @@ -81,13 +81,6 @@ "SELECT (coalesce(lpad(CAST(tab.year AS varchar(255)), 2, '0'), '01') || '-' || coalesce(lpad(CAST(tab.month AS varchar(255)), 2, '0'), '01') || '-' || coalesce(lpad(CAST(tab.day AS varchar(255)), 2, '0'), '01')) AS new_col FROM tab", query.sql) end - - it "should modify behavior if impala is database_type" do - query = db[:tab].select(ConceptQL::Utils.assemble_date(:year, :month, :day, database_type: :impala).as(:new_col)) - assert_equal( - "SELECT CAST(concat_ws('-', coalesce(lpad(CAST(year AS varchar(255)), 2, '0'), '01'), coalesce(lpad(CAST(month AS varchar(255)), 2, '0'), '01'), coalesce(lpad(CAST(day AS varchar(255)), 2, '0'), '01')) AS timestamp) AS new_col FROM tab", - query.sql) - end end end