From 55fbd02f42d5529bdc75582fe7135ef47a28354a Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 26 Nov 2018 12:04:48 -0700 Subject: [PATCH 1/5] run :work, disable work on web container --- docker/server/start-server.sh | 3 ++- docker/server/start-workers.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/server/start-server.sh b/docker/server/start-server.sh index 56015c90e..1f94649f7 100755 --- a/docker/server/start-server.sh +++ b/docker/server/start-server.sh @@ -14,6 +14,7 @@ cd /opt/openstudio/server && bundle exec rake db:mongoid:create_indexes ruby /usr/local/lib/memfix-controller.rb start # AP hack for now to ensure both resque and nginx/rails are running on web -bundle exec rake environment resque:work & +# NL: I don't think we want to run work on the web container. +# bundle exec rake environment resque:work & /opt/nginx/sbin/nginx diff --git a/docker/server/start-workers.sh b/docker/server/start-workers.sh index 55cf57b1a..aaee0f224 100644 --- a/docker/server/start-workers.sh +++ b/docker/server/start-workers.sh @@ -6,4 +6,5 @@ echo "Waiting for Mongo to start" echo "Waiting for Redis to start" /usr/local/bin/wait-for-it --strict -t 0 queue:6379 -cd /opt/openstudio/server && bundle exec rake environment resque:workers +# Only start a single worker when calling this script (do not use resque:workers). +cd /opt/openstudio/server && bundle exec rake environment resque:work From 538e421f532bc382885b0af17ca15a0c09a7706a Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 3 Dec 2018 15:28:45 -0700 Subject: [PATCH 2/5] fix unexpected return when launching, update docs --- README.md | 4 +- docker/server/start-server.sh | 6 +-- server/README.md | 51 +++++------------------ server/config/environments/development.rb | 3 ++ server/config/environments/docker-dev.rb | 1 - server/config/environments/docker-test.rb | 1 - server/config/environments/docker.rb | 1 - 7 files changed, 18 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 36295897d..28b242918 100644 --- a/README.md +++ b/README.md @@ -81,11 +81,11 @@ docker-compose rm -f docker volume rm osdata dbdata docker volume create --name=osdata docker volume create --name=dbdata -docker-compose up +OS_SERVER_NUMBER_OF_WORKERS=N docker-compose up docker-compose service scale worker=N # Or one line -docker-compose rm -f && docker-compose build && docker volume rm osdata dbdata && docker volume create --name=osdata && docker volume create --name=dbdata && OS_SERVER_NUMBER_OF_WORKERS=5 docker-compose up && docker-compose service scale worker=N +docker-compose rm -f && docker-compose build && docker volume rm osdata dbdata && docker volume create --name=osdata && docker volume create --name=dbdata && OS_SERVER_NUMBER_OF_WORKERS=N docker-compose up && docker-compose service scale worker=N ``` Congratulations! Visit `http://localhost:8080` to see the OpenStudio Server Management Console. diff --git a/docker/server/start-server.sh b/docker/server/start-server.sh index 1f94649f7..c9cded9bf 100755 --- a/docker/server/start-server.sh +++ b/docker/server/start-server.sh @@ -13,8 +13,8 @@ cd /opt/openstudio/server && bundle exec rake db:mongoid:create_indexes # https://github.com/NREL/OpenStudio-server/issues/348 ruby /usr/local/lib/memfix-controller.rb start -# AP hack for now to ensure both resque and nginx/rails are running on web -# NL: I don't think we want to run work on the web container. -# bundle exec rake environment resque:work & +# AP hack for now to ensure both resque and nginx/rails are running on web. +# Note that this will only run the analysis background queue as it is set in the docker-compose env file. +bundle exec rake environment resque:work & /opt/nginx/sbin/nginx diff --git a/server/README.md b/server/README.md index df6207c5e..d45666329 100644 --- a/server/README.md +++ b/server/README.md @@ -43,6 +43,15 @@ COUNT=4 QUEUES=simulations bundle exec rake environment resque:workers QUEUES=analysis_wrappers bundle exec rake environment resque:work ``` +### Running Simulations for development in the foreground + +OpenStudio Server now runs the simulations through the OpenStudio CLI. In order for this to work on local development, +the gems need to be installed in the OpenStudio installation directory. Run the following for OSX: + +```bash +cd /Applications/OpenStudio-x.y.z/Ruby +bundle install --path ./gems +``` ## Starting Rserve for development in the foreground @@ -69,21 +78,9 @@ gem install nokogiri -- --use-system-libraries --with-xml2-include=/usr/include/ gem install libxml-ruby -- --with-xml2-include=/usr/include/libxml2 --with-xml2-lib=/usr/lib/ ``` -# Testing using Docker-compose - -``` -docker volume create --name=osdata -docker-compose -f docker-compose.test.yml build -docker-compose -f docker-compose.test.yml up - -# One line -docker-compose rm -f && docker-compose -f docker-compose.test.yml build && docker-compose -f docker-compose.test.yml up -``` - # Testing -If running the tests on a local machine, then make sure to install -geckodriver to run the webpage. +If running the tests on a local machine, then make sure to install geckodriver to run the webpage. ``` bundle install @@ -108,31 +105,3 @@ publishing scripts are run on TravisCI. These are only run the develop and maste * Write tests for each analysis (expand existing SPEA test) * Add CLI path to config.yml * Add tests for embedded files on analysis model. Test result of R code that pushed to analysis model (i.e. best_point.json) - -# AWS Elastic Container Service - -It is possible to use Amazon's Elastic Container Service but it will -be limited to only running one machine and it is not possible to add -more worker nodes. The preferred approach is to use docker-machine and -docker-swarm or to custom deploy. - -``` -ecs-cli configure --cluster openstudio -ecs-cli up --keypair --capability-iam --size 1 --instance-type t2.medium --port 8080 -ecs-cli compose -f docker-compose.deploy.yml up -# Get the IP address from the console `ecs-cli ps` -ecs-cli down --force -``` - -# Docker-Machine - -## AWS - -The easiest approach to using Docker-Machine is to export your keys as -environment variables. - -``` -export $AWS_ACCESS_KEY_ID= -export $AWS_SECRET_ACCESS_KEY= -``` - diff --git a/server/config/environments/development.rb b/server/config/environments/development.rb index 90bde3972..929e4ec38 100644 --- a/server/config/environments/development.rb +++ b/server/config/environments/development.rb @@ -76,4 +76,7 @@ # Use a different logger for distributed setups. # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) + + # TODO: Remove assumption that we are developing on a machine that has which installed! Make cross platform... + config.os_gemfile_path = File.expand_path(File.join(File.dirname(`which openstudio`), '../Ruby')) end diff --git a/server/config/environments/docker-dev.rb b/server/config/environments/docker-dev.rb index 8687ef8e3..72f123989 100644 --- a/server/config/environments/docker-dev.rb +++ b/server/config/environments/docker-dev.rb @@ -78,6 +78,5 @@ # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) # detect os version build path to gemfile. this is passed to cli as --bundle - os_version = `openstudio openstudio_version` config.os_gemfile_path = "/var/oscli" end diff --git a/server/config/environments/docker-test.rb b/server/config/environments/docker-test.rb index 134b5b9c1..ba6ab7e92 100644 --- a/server/config/environments/docker-test.rb +++ b/server/config/environments/docker-test.rb @@ -79,6 +79,5 @@ # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) # detect os version build path to gemfile. this is passed to cli as --bundle - os_version = `openstudio openstudio_version` config.os_gemfile_path = "/var/oscli" end diff --git a/server/config/environments/docker.rb b/server/config/environments/docker.rb index 120a9165b..2e8730856 100644 --- a/server/config/environments/docker.rb +++ b/server/config/environments/docker.rb @@ -110,6 +110,5 @@ config.active_support.deprecation = :notify # detect os version build path to gemfile. this is passed to cli as --bundle - os_version = `openstudio openstudio_version` config.os_gemfile_path = "/var/oscli" end From 3108ce75e41849688c7cd22ad177e4ca67d1392e Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Mon, 3 Dec 2018 16:09:14 -0700 Subject: [PATCH 3/5] missed resque:workers --- local_setup_scripts/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local_setup_scripts/docker-compose.yml b/local_setup_scripts/docker-compose.yml index 20ed6ab87..75c93f3de 100644 --- a/local_setup_scripts/docker-compose.yml +++ b/local_setup_scripts/docker-compose.yml @@ -70,7 +70,7 @@ services: environment: - QUEUES=simulations - COUNT=1 - command: bundle exec rake environment resque:workers + command: bundle exec rake environment resque:work volumes: - /mnt/openstudio depends_on: From eb43eb9f2449a407d83d404b25552b2169128772 Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Tue, 4 Dec 2018 09:39:21 -0700 Subject: [PATCH 4/5] add travis wait to osx install of mongo --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d204dad68..ae90c3d68 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ gemfile: # Set the default scripts that are used for the majority of the tests in the matrix before_install: - - ./ci/travis/setup.sh + - travis_wait ./ci/travis/setup.sh before_script: - sudo Xvfb :99 -ac -screen 0 1024x768x8 & - sleep 3 From 7d4e528e32cf18845af32bcd7532dba5dcc907ed Mon Sep 17 00:00:00 2001 From: Nicholas Long Date: Tue, 4 Dec 2018 09:55:40 -0700 Subject: [PATCH 5/5] brew install mongodb is timing out --- .travis.yml | 4 ++-- ci/travis/setup.sh | 14 ++++++++++---- server/config/environments/development.rb | 7 +++++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index ae90c3d68..ef890e248 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,7 @@ gemfile: # Set the default scripts that are used for the majority of the tests in the matrix before_install: - - travis_wait ./ci/travis/setup.sh + - ./ci/travis/setup.sh before_script: - sudo Xvfb :99 -ac -screen 0 1024x768x8 & - sleep 3 @@ -22,7 +22,7 @@ after_failure: ./ci/travis/print_logs.sh # Services for linux -- all instances services: - - mongod + - mongodb - redis-server - docker diff --git a/ci/travis/setup.sh b/ci/travis/setup.sh index 79b2d18e8..93605a5b1 100755 --- a/ci/travis/setup.sh +++ b/ci/travis/setup.sh @@ -14,12 +14,18 @@ if [ "${BUILD_TYPE}" == "docker" ]; then else if [ "${TRAVIS_OS_NAME}" == "osx" ]; then brew update > /Users/travis/build/NREL/OpenStudio-server/spec/files/logs/brew-update.log - # AP: do we need mongo install here ? seems to be handled by service defined in travis yml - brew install mongodb@3.4 pv tree - ln -s /usr/local/opt/mongodb@3.4/bin/* /usr/local/bin + # AP: do we need mongo install here ? seems to be handled by service defined in travis yml. + # NL: Services are not handled in osx + brew install pv tree + + # Install mongodb from a download. Brew is hanging and requires building mongo. This also speeds up the builds. + curl -SLO https://fastdl.mongodb.org/osx/mongodb-osx-ssl-x86_64-3.4.18.tgz + tar xvzf mongodb-osx-ssl-x86_64-3.4.18.tgz + cp mongodb-osx-x86_64-3.4.18/bin/* /usr/local/bin/ + + # Install openstudio -- Use the install script that is in this repo now, the one on OpenStudio/develop has changed curl -SLO --insecure https://s3.amazonaws.com/openstudio-builds/$OPENSTUDIO_VERSION/OpenStudio-$OPENSTUDIO_VERSION.$OPENSTUDIO_VERSION_SHA-Darwin.zip unzip OpenStudio-$OPENSTUDIO_VERSION.$OPENSTUDIO_VERSION_SHA-Darwin.zip - # Use the install script that is in this repo now, the one on OpenStudio/develop has changed sed -i -e "s|REPLACEME|$HOME/openstudio|" ci/travis/install-mac.qs rm -rf $HOME/openstudio # Will install into $HOME/openstudio and RUBYLIB will be $HOME/openstudio/Ruby diff --git a/server/config/environments/development.rb b/server/config/environments/development.rb index 929e4ec38..1c1c385a5 100644 --- a/server/config/environments/development.rb +++ b/server/config/environments/development.rb @@ -77,6 +77,9 @@ # Use a different logger for distributed setups. # config.logger = ActiveSupport::TaggedLogging.new(SyslogLogger.new) - # TODO: Remove assumption that we are developing on a machine that has which installed! Make cross platform... - config.os_gemfile_path = File.expand_path(File.join(File.dirname(`which openstudio`), '../Ruby')) + # If you need to use custom gems in your development environment, then uncomment the line below. You will + # also need to run `bundle install` in your local openstudio Gemfile path. For example: + # cd /Applications/OpenStudio-x.y.z/Ruby + # bundle install --path ./gems + # config.os_gemfile_path = File.expand_path(File.join(File.dirname(`which openstudio`), '../Ruby')) end