diff --git a/.github/workflows/ruby-unit-tests.yml b/.github/workflows/ruby-unit-tests.yml index db0ae30..b8de9aa 100644 --- a/.github/workflows/ruby-unit-tests.yml +++ b/.github/workflows/ruby-unit-tests.yml @@ -6,22 +6,25 @@ on: jobs: test: + strategy: + fail-fast: false + matrix: + backend: ['ruby', 'ruby-agraph'] # ruby runs tests with 4store backend and ruby-agraph runs with AllegroGraph backend runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - name: copy config file - run: cp config/config.test.rb config/config.rb + - uses: actions/checkout@v4 + - name: copy config.rb file from template + run: cp config/config.test.rb config/config.rb - name: Build docker compose run: docker compose build - name: Run unit tests - # unit tests are run inside a container - # http://docs.codecov.io/docs/testing-with-docker run: | ci_env=`bash <(curl -s https://codecov.io/env)` - docker compose run $ci_env -e CI --rm ruby bundle exec rake test TESTOPTS='-v' + docker compose run $ci_env -e CI --rm ${{ matrix.backend }} bundle exec rake test TESTOPTS='-v' - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: + token: ${{ secrets.CODECOV_TOKEN }} flags: unittests verbose: true fail_ci_if_error: false # optional (default = false) diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 0000000..818bd47 --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +3.0.6 diff --git a/Dockerfile b/Dockerfile index 195d22d..910720f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -ARG RUBY_VERSION +ARG RUBY_VERSION=3.0 ARG DISTRO_NAME=bullseye FROM ruby:$RUBY_VERSION-$DISTRO_NAME @@ -8,14 +8,16 @@ RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends \ raptor2-utils \ && rm -rf /var/lib/apt/lists/* -# The Gemfile Caching Trick -RUN mkdir -p /srv/ontoportal/ncbo_ontology_recommender -COPY *.gemspec Gemfile* /srv/ontoportal/ncbo_ontology_recommender/ +ENV BUNDLE_PATH=/usr/local/bundle \ + BUNDLE_BIN=/usr/local/bundle/bin \ + BUNDLE_APP_CONFIG=/usr/local/bundle/config -WORKDIR /srv/ontoportal/ncbo_ontology_recommender +WORKDIR /usr/src/app -RUN gem install bundler -ENV BUNDLE_PATH /bundle +COPY Gemfile* *.gemspec ./ + +RUN gem update --system RUN bundle install -COPY . /srv/ontoportal/ncbo_ontology_recommender -CMD ["/bin/bash"] + +COPY . . +CMD ["irb"] diff --git a/Gemfile b/Gemfile index f62ffbd..20e4d61 100644 --- a/Gemfile +++ b/Gemfile @@ -1,7 +1,7 @@ source 'https://rubygems.org' gem 'cube-ruby' -gem 'faraday', '~> 2.0' +gem 'faraday' gem 'ffi' gem 'oj', '~> 3.0' gem 'rake', '~> 10.0' diff --git a/Gemfile.lock b/Gemfile.lock index 8543a75..ea5f1ff 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/ncbo/goo.git - revision: 33583fd1c1d72b449cd6bb91815e3422b8448983 + revision: 74a012eebb9433d031eb00df5abbe488cb8b4512 branch: develop specs: goo (0.0.2) @@ -16,7 +16,7 @@ GIT GIT remote: https://github.com/ncbo/ncbo_annotator.git - revision: f9a1be204e81d72f408927b8f60c0be208611929 + revision: b2731c98bfef0958fecb440c8648b7755a670895 branch: develop specs: ncbo_annotator (0.0.1) @@ -27,7 +27,7 @@ GIT GIT remote: https://github.com/ncbo/ontologies_linked_data.git - revision: da1642ed8f9f04db2e427e35d88f235a55138d03 + revision: 812dd78f02b77c9c6d579b0febf1c2a42d513022 branch: develop specs: ontologies_linked_data (0.0.1) @@ -64,34 +64,34 @@ GEM multi_json (~> 1.3) thread_safe (~> 0.1) tzinfo (~> 0.3.37) - addressable (2.8.6) - public_suffix (>= 2.0.2, < 6.0) - base64 (0.2.0) + addressable (2.8.7) + public_suffix (>= 2.0.2, < 7.0) bcrypt (3.1.20) - bigdecimal (3.1.7) - builder (3.2.4) + bigdecimal (3.1.8) + builder (3.3.0) coderay (1.1.3) - concurrent-ruby (1.2.3) + concurrent-ruby (1.3.4) connection_pool (2.4.1) cube-ruby (0.0.3) date (3.3.4) - docile (1.4.0) + docile (1.4.1) domain_name (0.6.20240107) - faraday (2.8.1) - base64 - faraday-net_http (>= 2.0, < 3.1) - ruby2_keywords (>= 0.0.4) - faraday-net_http (3.0.2) - ffi (1.16.3) + faraday (2.12.0) + faraday-net_http (>= 2.0, < 3.4) + json + logger + faraday-net_http (3.3.0) + net-http + ffi (1.17.0-aarch64-linux-gnu) http-accept (1.7.0) - http-cookie (1.0.5) + http-cookie (1.0.7) domain_name (~> 0.5) i18n (0.9.5) concurrent-ruby (~> 1.0) json (2.7.2) json_pure (2.7.2) libxml-ruby (5.0.3) - logger (1.6.0) + logger (1.6.1) macaddr (1.7.2) systemu (~> 2.6.5) mail (2.8.1) @@ -100,14 +100,17 @@ GEM net-pop net-smtp method_source (1.1.0) - mime-types (3.5.2) + mime-types (3.6.0) + logger mime-types-data (~> 3.2015) - mime-types-data (3.2024.0305) + mime-types-data (3.2024.1001) mini_mime (1.1.5) minitest (4.7.5) multi_json (1.15.0) + net-http (0.4.1) + uri net-http-persistent (2.9.4) - net-imap (0.4.10) + net-imap (0.4.16) date net-protocol net-pop (0.1.2) @@ -117,39 +120,40 @@ GEM net-smtp (0.5.0) net-protocol netrc (0.11.0) - oj (3.16.3) + oj (3.16.6) bigdecimal (>= 3.0) + ostruct (>= 0.2) omni_logger (0.1.4) logger + ostruct (0.6.0) pony (1.13.1) mail (>= 2.0) pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) - public_suffix (5.0.5) - rack (3.0.10) + public_suffix (6.0.1) + rack (3.1.7) rack-test (2.1.0) rack (>= 1.3) rake (10.5.0) rdf (1.0.8) addressable (>= 2.2) - redis (5.2.0) + redis (5.3.0) redis-client (>= 0.22.0) - redis-client (0.22.1) + redis-client (0.22.2) connection_pool - request_store (1.6.0) + request_store (1.7.0) rack (>= 1.4) rest-client (2.1.0) http-accept (>= 1.7.0, < 2.0) http-cookie (>= 1.0.2, < 2.0) mime-types (>= 1.16, < 4.0) netrc (~> 0.8) - rexml (3.2.6) + rexml (3.3.8) rsolr (2.6.0) builder (>= 2.1.2) faraday (>= 0.9, < 3, != 2.0.0) ruby-xxHash (0.4.0.2) - ruby2_keywords (0.0.5) rubyzip (2.3.2) simplecov (0.22.0) docile (~> 1.1) @@ -158,26 +162,22 @@ GEM simplecov-cobertura (2.1.0) rexml simplecov (~> 0.19) - simplecov-html (0.12.3) + simplecov-html (0.13.1) simplecov_json_formatter (0.1.4) systemu (2.6.5) thread_safe (0.3.6) timeout (0.4.1) tzinfo (0.3.62) + uri (0.13.1) uuid (2.3.9) macaddr (~> 1.0) PLATFORMS aarch64-linux - arm64-darwin-22 - ruby - x86_64-darwin-18 - x86_64-darwin-21 - x86_64-linux DEPENDENCIES cube-ruby - faraday (~> 2.0) + faraday ffi goo! minitest (~> 4.0) @@ -192,4 +192,4 @@ DEPENDENCIES sparql-client! BUNDLED WITH - 2.4.22 + 2.5.21 diff --git a/config/config.test.rb b/config/config.test.rb index e1553d9..d3a150b 100644 --- a/config/config.test.rb +++ b/config/config.test.rb @@ -1,27 +1,40 @@ # frozen_string_literal: true -# # This file is designed for used for unit testing # # All the defaults are set in # https://github.com/ncbo/ontologies_linked_data/blob/master/lib/ontologies_linked_data/config/config.rb ### -GOO_HOST = ENV.include?('GOO_HOST') ? ENV['GOO_HOST'] : 'localhost' -GOO_PORT = ENV.include?('GOO_PORT') ? ENV['GOO_PORT'] : 9000 -REDIS_HOST = ENV.include?('REDIS_HOST') ? ENV['REDIS_HOST'] : 'localhost' -REDIS_PORT = ENV.include?('REDIS_PORT') ? ENV['REDIS_PORT'] : 6379 -SOLR_HOST = ENV.include?('SOLR_HOST') ? ENV['SOLR_HOST'] : 'localhost' -MGREP_HOST = ENV.include?('MGREP_HOST') ? ENV['MGREP_HOST'] : 'localhost' -MGREP_PORT = ENV.include?('MGREP_PORT') ? ENV['MGREP_PORT'] : 55555 +GOO_BACKEND_NAME = ENV.include?("GOO_BACKEND_NAME") ? ENV["GOO_BACKEND_NAME"] : "4store" +GOO_PATH_QUERY = ENV.include?("GOO_PATH_QUERY") ? ENV["GOO_PATH_QUERY"] : "/sparql/" +GOO_PATH_DATA = ENV.include?("GOO_PATH_DATA") ? ENV["GOO_PATH_DATA"] : "/data/" +GOO_PATH_UPDATE = ENV.include?("GOO_PATH_UPDATE") ? ENV["GOO_PATH_UPDATE"] : "/update/" +GOO_PORT = ENV.include?("GOO_PORT") ? ENV["GOO_PORT"] : 9000 +GOO_HOST = ENV.include?("GOO_HOST") ? ENV["GOO_HOST"] : "localhost" +REDIS_HOST = ENV.include?("REDIS_HOST") ? ENV["REDIS_HOST"] : "localhost" +REDIS_PORT = ENV.include?("REDIS_PORT") ? ENV["REDIS_PORT"] : 6379 +MGREP_HOST = ENV.include?('MGREP_HOST') ? ENV['MGREP_HOST'] : 'localhost' +MGREP_PORT = ENV.include?('MGREP_PORT') ? ENV['MGREP_PORT'] : 55556 +SOLR_TERM_SEARCH_URL = ENV.include?("SOLR_TERM_SEARCH_URL") ? ENV["SOLR_TERM_SEARCH_URL"] : "http://localhost:8983/solr/term_search_core1" +SOLR_PROP_SEARCH_URL = ENV.include?("SOLR_PROP_SEARCH_URL") ? ENV["SOLR_PROP_SEARCH_URL"] : "http://localhost:8983/solr/prop_search_core1" LinkedData.config do |config| - config.goo_host = GOO_HOST.to_s - config.goo_port = GOO_PORT.to_i + config.goo_backend_name = GOO_BACKEND_NAME.to_s + config.goo_host = GOO_HOST.to_s + config.goo_port = GOO_PORT.to_i + config.goo_path_query = GOO_PATH_QUERY.to_s + config.goo_path_data = GOO_PATH_DATA.to_s + config.goo_path_update = GOO_PATH_UPDATE.to_s + config.goo_redis_host = REDIS_HOST.to_s + config.goo_redis_port = REDIS_PORT.to_i + config.http_redis_host = REDIS_HOST.to_s + config.http_redis_port = REDIS_PORT.to_i config.ontology_analytics_redis_host = REDIS_HOST.to_s config.ontology_analytics_redis_port = REDIS_PORT.to_i - config.search_server_url = "http://#{SOLR_HOST}:8983/solr/term_search_core1".to_s - config.property_search_server_url = "http://#{SOLR_HOST}:8983/solr/prop_search_core1".to_s + config.search_server_url = SOLR_TERM_SEARCH_URL.to_s + config.property_search_server_url = SOLR_PROP_SEARCH_URL.to_s end + Annotator.config do |config| config.mgrep_host = MGREP_HOST.to_s config.mgrep_port = MGREP_PORT.to_i diff --git a/dip.yml b/dip.yml new file mode 100644 index 0000000..92d63ce --- /dev/null +++ b/dip.yml @@ -0,0 +1,55 @@ +version: '7.1' + +# Define default environment variables to pass +# to Docker Compose +#environment: +# RAILS_ENV: development + +compose: + files: + - docker-compose.yml + +interaction: + # This command spins up a ontologies_api container with the required dependencies (solr, 4store, etc), + # and opens a terminal within it. + runner: + description: Open a Bash shell within a ontologies_api container (with dependencies up) + service: ruby + command: /bin/bash + + # Run a Rails container without any dependent services (useful for non-Rails scripts) + bash: + description: Run an arbitrary script within a container (or open a shell without deps) + service: ruby + command: /bin/bash + compose_run_options: [ no-deps ] + + # A shortcut to run Bundler commands + bundle: + description: Run Bundler commands + service: ruby + command: bundle + compose_run_options: [ no-deps ] + + # A shortcut to run unit tests + test: + description: Run minitest unit tests + service: ruby + command: bundle exec rake test + + test-ag: + description: Run minitest unit tests + service: ruby-ag + command: bundle exec rake test + + + 'redis-cli': + description: Run Redis console + service: redis-ut + command: redis-cli -h redis-ut + +#provision: + #- dip compose down --volumes + #- dip compose up -d solr 4store + #- dip bundle install + #- dip bash -c bin/setup diff --git a/docker-compose.yml b/docker-compose.yml index 2960866..39cd3eb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,40 +1,93 @@ -version: '3.8' +x-app: &app + image: ncbo_ontology_recommender:0.0.1 + build: + context: . + args: + RUBY_VERSION: '3.0' + environment: &env + BUNDLE_PATH: /usr/local/bundle + # default bundle config resolves to /usr/local/bundle/config inside of the container + # we are setting it to local app directory if we need to use 'bundle config local' + BUNDLE_APP_CONFIG: /usr/local/bundle/config + COVERAGE: 'true' # enable simplecov code coverage + REDIS_HOST: redis-ut + REDIS_PORT: 6379 + MGREP_HOST: mgrep-ut + MGREP_PORT: 55556 + SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr/term_search_core1 + SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr/prop_search_core1 + volumes: + - bundle:/usr/local/bundle + - .:/usr/src/app + stdin_open: true + tty: true + command: "bundle exec rake test TESTOPTS='-v'" + depends_on: &depends_on + solr-ut: + condition: service_healthy + redis-ut: + condition: service_healthy + mgrep-ut: + condition: service_healthy services: ruby: - build: - context: . - args: - RUBY_VERSION: '2.7' - volumes: - - bundle:/bundle - - .:/srv/ontoportal/ncbo_ontology_recommender + <<: *app environment: - - BUNDLE_PATH=/bundle - - GOO_BACKEND_NAME=4store - - GOO_PORT=9000 - - GOO_HOST=4store-ut - - REDIS_HOST=redis-ut - - REDIS_PORT=6379 - - SOLR_HOST=solr-ut - - MGREP_HOST=mgrep-ut - - MGREP_PORT=55556 + <<: *env + GOO_BACKEND_NAME: 4store + GOO_PORT: 9000 + GOO_HOST: 4store-ut depends_on: - - solr-ut - - redis-ut - - 4store-ut - - mgrep-ut - command: "bundle exec rake test TESTOPTS='-v'" + <<: *depends_on + 4store-ut: + condition: service_started + profiles: + - 4store + + ruby-agraph: + <<: *app + environment: + <<: *env + GOO_BACKEND_NAME: ag + GOO_PORT: 10035 + GOO_HOST: agraph-ut + GOO_PATH_QUERY: /repositories/ontoportal_test + GOO_PATH_DATA: /repositories/ontoportal_test/statements + GOO_PATH_UPDATE: /repositories/ontoportal_test/statements + depends_on: + <<: *depends_on + agraph-ut: + condition: service_healthy + profiles: + - agraph solr-ut: - image: ontoportal/solr-ut:0.0.2 + image: ontoportal/solr-ut:0.2.0 + healthcheck: + test: ["CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1"] + start_period: 3s + interval: 10s + timeout: 5s + retries: 5 redis-ut: image: redis + healthcheck: + test: redis-cli ping + interval: 10s + timeout: 3s + retries: 10 mgrep-ut: image: ontoportal/mgrep:0.0.2 platform: linux/amd64 + healthcheck: + test: ["CMD", "nc", "-z", "-v", "127.0.0.1", "55556"] + start_period: 3s + interval: 10s + timeout: 5s + retries: 5 4store-ut: image: bde2020/4store @@ -43,6 +96,30 @@ services: bash -c "4s-backend-setup --segments 4 ontoportal_kb && 4s-backend ontoportal_kb && 4s-httpd -D -s-1 -p 9000 ontoportal_kb" + profiles: + - 4store + + agraph-ut: + image: franzinc/agraph:v8.1.0 + platform: linux/amd64 + environment: + - AGRAPH_SUPER_USER=test + - AGRAPH_SUPER_PASSWORD=xyzzy + shm_size: 1g + command: > + bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start + ; agtool repos create --supersede ontoportal_test + ; agtool users add anonymous + ; agtool users grant anonymous root:ontoportal_test:rw + ; tail -f /agraph/data/agraph.log" + healthcheck: + test: ["CMD-SHELL", "agtool storage-report ontoportal_test || exit 1"] + start_period: 30s + interval: 20s + timeout: 10s + retries: 10 + profiles: + - agraph volumes: bundle: diff --git a/test/test_case.rb b/test/test_case.rb index c8c1864..81f38ba 100644 --- a/test/test_case.rb +++ b/test/test_case.rb @@ -21,7 +21,7 @@ end # Check to make sure you want to run if not pointed at localhost -safe_host = Regexp.new(/localhost|-ut|ncbo-dev*|ncbo-unittest*/) +safe_host = Regexp.new(/localhost|-ut/) unless LinkedData.settings.goo_host.match(safe_host) && LinkedData.settings.search_server_url.match(safe_host) && Annotator.settings.annotator_redis_host.match(safe_host) @@ -92,7 +92,7 @@ def self.ontologies # Code to run before the very first test def before_suites LinkedData::SampleData::Ontology.delete_ontologies_and_submissions - @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies + @@ontologies = LinkedData::SampleData::Ontology.sample_owl_ontologies(process_submission: true) @@sty = LinkedData::SampleData::Ontology.load_semantic_types_ontology annotator = Annotator::Models::NcboAnnotator.new annotator.init_redis_for_tests