forked from ontoportal/ontologies_api
-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
set up multiple triple store test environment
1 parent
37408c5
commit d30be5b
Showing
6 changed files
with
269 additions
and
122 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,129 +1,154 @@ | ||
x-app: &app | ||
build: | ||
context: . | ||
args: | ||
RUBY_VERSION: '2.7' | ||
# Increase the version number in the image tag every time Dockerfile or its arguments is changed | ||
image: ontologies_api:0.0.1 | ||
environment: &env | ||
BUNDLE_PATH: /srv/ontoportal/bundle | ||
# default bundle config resolves to /usr/local/bundle/config inside of the container | ||
# we are setting it to local app directory if we need to use 'bundle config local' | ||
BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle | ||
COVERAGE: 'true' | ||
GOO_REDIS_HOST: redis-ut | ||
REDIS_HOST: redis-ut | ||
REDIS_PORT: 6379 | ||
SOLR_HOST: solr-ut | ||
SOLR_TERM_SEARCH_URL: http://solr-ut:8983/solr/term_search_core1 | ||
SOLR_PROP_SEARCH_URL: http://solr-ut:8983/solr/prop_search_core1 | ||
MGREP_HOST: mgrep-ut | ||
MGREP_PORT: 55555 | ||
stdin_open: true | ||
tty: true | ||
command: "bundle exec rackup -o 0.0.0.0 --port 9393" | ||
ports: | ||
- 9393:9393 | ||
volumes: | ||
# bundle volume for hosting gems installed by bundle; it helps in local development with gem udpates | ||
- bundle:/srv/ontoportal/bundle | ||
# api code | ||
- .:/srv/ontoportal/ontologies_api | ||
# mount directory containing development version of the gems if you need to use 'bundle config local' | ||
#- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal | ||
depends_on: | ||
- solr-ut | ||
- redis-ut | ||
- mgrep-ut | ||
build: | ||
context: . | ||
args: | ||
RUBY_VERSION: '2.7' | ||
# Increase the version number in the image tag every time Dockerfile or its arguments is changed | ||
image: ontologies_ld-dev:0.0.2 | ||
environment: &env | ||
# default bundle config resolves to /usr/local/bundle/config inside of the container | ||
# we are setting it to local app directory if we need to use 'bundle config local' | ||
BUNDLE_APP_CONFIG: /srv/ontoportal/ontologies_api/.bundle | ||
BUNDLE_PATH: /srv/ontoportal/bundle | ||
COVERAGE: 'true' # enable simplecov code coverage | ||
REDIS_HOST: redis-ut | ||
REDIS_PORT: 6379 | ||
SOLR_TERM_SEARCH_URL: http://solr-term-ut:8983/solr/term_search_core1 | ||
SOLR_PROP_SEARCH_URL: http://solr-prop-ut:8983/solr/prop_search_core1 | ||
stdin_open: true | ||
tty: true | ||
command: /bin/bash | ||
volumes: | ||
# bundle volume for hosting gems installed by bundle; it speeds up gem install in local development | ||
- bundle:/srv/ontoportal/bundle | ||
- .:/srv/ontoportal/ontologies_linked_data | ||
# mount directory containing development version of the gems if you need to use 'bundle config local' | ||
#- /Users/alexskr/ontoportal:/Users/alexskr/ontoportal | ||
depends_on: &depends_on | ||
solr-prop-ut: | ||
condition: service_healthy | ||
solr-term-ut: | ||
condition: service_healthy | ||
redis-ut: | ||
condition: service_healthy | ||
|
||
services: | ||
api: | ||
<<: *app | ||
environment: | ||
<<: *env | ||
GOO_BACKEND_NAME: 4store | ||
GOO_PORT: 9000 | ||
GOO_HOST: 4store-ut | ||
GOO_PATH_QUERY: /sparql/ | ||
GOO_PATH_DATA: /data/ | ||
GOO_PATH_UPDATE: /update/ | ||
profiles: | ||
- 4store | ||
depends_on: | ||
- solr-ut | ||
- redis-ut | ||
- mgrep-ut | ||
- 4store-ut | ||
|
||
api-agraph: | ||
<<: *app | ||
environment: | ||
<<: *env | ||
GOO_BACKEND_NAME: ag | ||
GOO_PORT: 10035 | ||
GOO_HOST: agraph-ut | ||
GOO_PATH_QUERY: /repositories/bioportal_test | ||
GOO_PATH_DATA: /repositories/bioportal_test/statements | ||
GOO_PATH_UPDATE: /repositories/bioportal_test/statements | ||
profiles: | ||
- agraph | ||
depends_on: | ||
- solr-ut | ||
- redis-ut | ||
- mgrep-ut | ||
- agraph-ut | ||
|
||
redis-ut: | ||
image: redis | ||
ports: | ||
- 6379:6379 | ||
- "6379:6379" | ||
command: [ "redis-server", "--save", "", "--appendonly", "no" ] | ||
healthcheck: | ||
test: redis-cli ping | ||
interval: 10s | ||
timeout: 3s | ||
retries: 10 | ||
|
||
4store-ut: | ||
image: bde2020/4store | ||
#volume: fourstore:/var/lib/4store | ||
ports: | ||
- 9000:9000 | ||
command: > | ||
bash -c "4s-backend-setup --segments 4 ontoportal_kb | ||
&& 4s-backend ontoportal_kb | ||
&& 4s-httpd -D -s-1 -p 9000 ontoportal_kb" | ||
ports: | ||
- "9000:9000" | ||
profiles: | ||
- 4store | ||
|
||
- fs | ||
|
||
solr-ut: | ||
solr-term-ut: | ||
image: solr:8 | ||
volumes: | ||
- ./test/solr/configsets:/configsets:ro | ||
ports: | ||
- "8983:8983" | ||
command: > | ||
bash -c "precreate-core term_search_core1 /configsets/term_search | ||
&& precreate-core prop_search_core1 /configsets/property_search | ||
&& solr-foreground" | ||
command: [ "solr-precreate", "term_search_core1", "/configsets/term_search" ] | ||
healthcheck: | ||
test: [ "CMD-SHELL", "curl -sf http://localhost:8983/solr/term_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1" ] | ||
start_period: 5s | ||
interval: 10s | ||
timeout: 5s | ||
retries: 5 | ||
|
||
mgrep-ut: | ||
image: ontoportal/mgrep-ncbo:0.1 | ||
solr-prop-ut: | ||
image: solr:8 | ||
volumes: | ||
- ./test/solr/configsets:/configsets:ro | ||
ports: | ||
- "55556:55555" | ||
- "8984:8983" | ||
command: [ "solr-precreate", "prop_search_core1", "/configsets/property_search" ] | ||
healthcheck: | ||
test: [ "CMD-SHELL", "curl -sf http://localhost:8983/solr/prop_search_core1/admin/ping?wt=json | grep -iq '\"status\":\"OK\"}' || exit 1" ] | ||
start_period: 5s | ||
interval: 10s | ||
timeout: 5s | ||
retries: 5 | ||
|
||
agraph-ut: | ||
image: franzinc/agraph:v7.3.0 | ||
image: franzinc/agraph:v8.0.0.rc1 | ||
platform: linux/amd64 | ||
environment: | ||
- AGRAPH_SUPER_USER=test | ||
- AGRAPH_SUPER_PASSWORD=xyzzy | ||
shm_size: 1g | ||
# ports: | ||
# - 10035:10035 | ||
ports: | ||
# - 10035:10035 | ||
- 10000-10035:10000-10035 | ||
volumes: | ||
- agdata:/agraph/data | ||
# - ./agraph/etc:/agraph/etc | ||
command: > | ||
bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start | ||
; agtool repos create bioportal_test | ||
; agtool users add anonymous | ||
; agtool users grant anonymous root:bioportal_test:rw | ||
; tail -f /agraph/data/agraph.log" | ||
bash -c "/agraph/bin/agraph-control --config /agraph/etc/agraph.cfg start | ||
; agtool repos create ontoportal_test --supersede | ||
; agtool users add anonymous | ||
; agtool users grant anonymous root:ontoportal_test:rw | ||
; tail -f /agraph/data/agraph.log" | ||
# healthcheck: | ||
# test: ["CMD-SHELL", "curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1"] | ||
# start_period: 10s | ||
# interval: 10s | ||
# timeout: 5s | ||
# retries: 5 | ||
profiles: | ||
- ag | ||
|
||
virtuoso-ut: | ||
image: tenforce/virtuoso:virtuoso7.2.5 | ||
platform: linux/amd64 | ||
environment: | ||
- SPARQL_UPDATE=true | ||
ports: | ||
- 1111:1111 | ||
- 8890:8890 | ||
profiles: | ||
- vo | ||
healthcheck: | ||
test: [ "CMD-SHELL", "curl -sf http://localhost:8890/sparql || exit 1" ] | ||
start_period: 10s | ||
interval: 60s | ||
timeout: 5s | ||
retries: 3 | ||
|
||
graphdb-ut: | ||
image: ontotext/graphdb:10.3.3 | ||
platform: linux/amd64 | ||
privileged: true | ||
environment: | ||
GDB_HEAP_SIZE: 5G | ||
GDB_JAVA_OPTS: >- | ||
-Xms5g -Xmx5g | ||
ports: | ||
- 7200:7200 | ||
- 7300:7300 | ||
volumes: | ||
- ./test/data/graphdb-repo-config.ttl:/opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl | ||
- ./test/data/graphdb-test-load.nt:/opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt | ||
|
||
entrypoint: > | ||
bash -c " importrdf load -f -c /opt/graphdb/dist/configs/templates/data/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/data/graphdb-test-load.nt ; graphdb -Ddefault.min.distinct.threshold=3000 " | ||
profiles: | ||
- agraph | ||
- gb | ||
|
||
volumes: | ||
bundle: | ||
#fourstore: | ||
agdata: |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,107 @@ | ||
# Rake tasks for running unit tests with backend services running as docker containers | ||
|
||
desc 'Run unit tests with docker based backend' | ||
namespace :test do | ||
namespace :docker do | ||
task :up do | ||
system("docker compose up -d") || abort("Unable to start docker containers") | ||
end | ||
task :down do | ||
#system("docker compose --profile fs --profile ag stop") | ||
#system("docker compose --profile fs --profile ag kill") | ||
end | ||
desc "run tests with docker AG backend" | ||
task :ag do | ||
ENV["GOO_BACKEND_NAME"]="AG" | ||
ENV["GOO_PORT"]="10035" | ||
ENV["GOO_PATH_QUERY"]="/repositories/ontoportal_test" | ||
ENV["GOO_PATH_DATA"]="/repositories/ontoportal_test/statements" | ||
ENV["GOO_PATH_UPDATE"]="/repositories/ontoportal_test/statements" | ||
ENV["COMPOSE_PROFILES"]="ag" | ||
Rake::Task["test:docker:up"].invoke | ||
# AG takes some time to start and create databases/accounts | ||
# TODO: replace system curl command with native ruby code | ||
unless system("curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1") | ||
printf("waiting for AllegroGraph container to initialize") | ||
sec = 0 | ||
until system("curl -sf http://127.0.0.1:10035/repositories/ontoportal_test/status | grep -iqE '(^running|^lingering)' || exit 1") do | ||
sleep(1) | ||
printf(".") | ||
sec += 1 | ||
end | ||
end | ||
puts | ||
system("docker compose ps") # TODO: remove after GH actions troubleshooting is complete | ||
Rake::Task["test"].invoke | ||
Rake::Task["test:docker:down"].invoke | ||
end | ||
|
||
desc "run tests with docker 4store backend" | ||
task :fs do | ||
ENV["GOO_PORT"]="9000" | ||
ENV["COMPOSE_PROFILES"]='fs' | ||
Rake::Task["test:docker:up"].invoke | ||
Rake::Task["test"].invoke | ||
Rake::Task["test:docker:down"].invoke | ||
end | ||
|
||
desc "run tests with docker Virtuoso backend" | ||
task :vo do | ||
ENV["GOO_BACKEND_NAME"]="VO" | ||
ENV["GOO_PORT"]="8890" | ||
ENV["GOO_PATH_QUERY"]="/sparql" | ||
ENV["GOO_PATH_DATA"]="/sparql" | ||
ENV["GOO_PATH_UPDATE"]="/sparql" | ||
ENV["COMPOSE_PROFILES"]="vo" | ||
Rake::Task["test:docker:up"].invoke | ||
# | ||
unless system("curl -sf http://localhost:8890/sparql || exit 1") | ||
printf("waiting for Virtuoso container to initialize") | ||
sec = 0 | ||
until system("curl -sf http://localhost:8890/sparql || exit 1") do | ||
sleep(1) | ||
printf(".") | ||
sec += 1 | ||
if sec > 30 | ||
system("docker compose logs virtuoso-ut") | ||
abort(" Virtuoso container hasn't initialized properly") | ||
end | ||
end | ||
end | ||
Rake::Task["test"].invoke | ||
Rake::Task["test:docker:down"].invoke | ||
end | ||
|
||
|
||
desc "run tests with docker GraphDb backend" | ||
task :gb do | ||
ENV["GOO_BACKEND_NAME"]="GB" | ||
ENV["GOO_PORT"]="7200" | ||
ENV["GOO_PATH_QUERY"]="/repositories/ontoportal" | ||
ENV["GOO_PATH_DATA"]="/repositories/ontoportal/statements" | ||
ENV["GOO_PATH_UPDATE"]="/repositories/ontoportal/statements" | ||
ENV["COMPOSE_PROFILES"]="gb" | ||
Rake::Task["test:docker:up"].invoke | ||
|
||
#system("docker compose cp ./test/data/graphdb-repo-config.ttl graphdb:/opt/graphdb/dist/configs/templates/graphdb-repo-config.ttl") | ||
#system("docker compose cp ./test/data/graphdb-test-load.nt graphdb:/opt/graphdb/dist/configs/templates/graphdb-test-load.nt") | ||
#system('docker compose exec graphdb sh -c "importrdf load -f -c /opt/graphdb/dist/configs/templates/graphdb-repo-config.ttl -m parallel /opt/graphdb/dist/configs/templates/graphdb-test-load.nt ;"') | ||
unless system("curl -sf http://localhost:7200/repositories || exit 1") | ||
printf("waiting for Graphdb container to initialize") | ||
sec = 0 | ||
until system("curl -sf http://localhost:7200/repositories || exit 1") do | ||
sleep(1) | ||
printf(".") | ||
sec += 1 | ||
if sec > 30 | ||
system("docker compose logs graphdb") | ||
abort(" Graphdb container hasn't initialized properly") | ||
end | ||
end | ||
end | ||
Rake::Task["test"].invoke | ||
Rake::Task["test:docker:down"].invoke | ||
end | ||
|
||
end | ||
end |