Merge pull request #535 from NREL/develop

Updating to 2.9.1 official

.travis.yml

@@ -5,7 +5,7 @@ rvm: 2.2.4
 env:
   global:
     - USE_TESTING_TIMEOUTS: "true"
-    - OPENSTUDIO_VERSION=2.9.1 OPENSTUDIO_VERSION_SHA=3472e8b799 OPENSTUDIO_VERSION_EXT=-rc1
+    - OPENSTUDIO_VERSION=2.9.1 OPENSTUDIO_VERSION_SHA=3472e8b799 OPENSTUDIO_VERSION_EXT=
     - DOCKER_COMPOSE_VERSION=1.21.1
 
 gemfile:

appveyor.yml

@@ -4,7 +4,7 @@ environment:
   USE_TESTING_TIMEOUTS: "true"
   OPENSTUDIO_VERSION: 2.9.1
   OPENSTUDIO_VERSION_SHA: 3472e8b799
-  OPENSTUDIO_VERSION_EXT: -rc1
+  OPENSTUDIO_VERSION_EXT:
   OPENSTUDIO_TEST_EXE: C:\projects\openstudio\bin\openstudio.exe
 
 

local_setup_scripts/README.md

@@ -0,0 +1,10 @@
+# Local Setup Scripts
+
+These are useful scripts which are used to setup local deployments are described below:
+
+* *rebuild_sr.sh*: This script will rebuild the docker containers using the code in this repo and deploy the swarm.
+* *redploy.sh*: This script will redploy an existing docker swarm.
+* *nuke.sh*: This script is useful for deleting all docker images and reinstalling a fresh deployment from published containers on dockerhub.  It takes as an argument the tag for the containers to install.
+* *docker_logs.sh*: This script is useful for getting the docker logs from the swarm containers.
+
+The win64 directory contains windows version of these files for setting up a deployment on windows using Docker Desktop.

local_setup_scripts/win64/docker-compose.yml

@@ -0,0 +1,107 @@
+version: '3.4'
+services:
+  db:
+    image: 127.0.0.1:5000/mongo
+    ports:
+      - "27017:27017"
+    volumes:
+      - dbdata:/data/db
+    deploy:
+      placement:
+        constraints:
+          - node.role == manager
+  queue:
+    image: 127.0.0.1:5000/redis
+    ports:
+      - "6379:6379"
+    deploy:
+      placement:
+        constraints:
+          - node.role == manager
+  web:
+    image: 127.0.0.1:5000/openstudio-server
+    ports:
+      - "8080:80"
+      - "80:80"
+      - "443:443"
+    environment:
+      - OS_SERVER_NUMBER_OF_WORKERS=1
+      - MAX_REQUESTS=10
+      - MAX_POOL=10
+      - QUEUES=analysis_wrappers
+    volumes:
+      - osdata:/mnt/openstudio
+    depends_on:
+      - db
+      - queue
+    deploy:
+      placement:
+        constraints:
+          - node.role == manager
+    command: /usr/local/bin/start-server
+  web-background:
+    image: 127.0.0.1:5000/openstudio-server
+    environment:
+      - OS_SERVER_NUMBER_OF_WORKERS=1
+      - QUEUES=background,analyses
+    volumes:
+      - osdata:/mnt/openstudio
+    depends_on:
+      - db
+      - web
+      - queue
+    deploy:
+      placement:
+        constraints:
+          - node.role == manager
+    command: /usr/local/bin/start-web-background
+  worker:
+    image: 127.0.0.1:5000/openstudio-server
+    environment:
+      - QUEUES=simulations
+      - COUNT=1
+    volumes:
+      - /mnt/openstudio
+    depends_on:
+      - web
+      - web-background
+      - db
+      - queue
+      - rserve
+    command: /usr/local/bin/start-workers
+  rserve:
+    image: 127.0.0.1:5000/openstudio-rserve
+    volumes:
+      - osdata:/mnt/openstudio
+    depends_on:
+      - web
+      - web-background
+      - db
+    deploy:
+      placement:
+        constraints:
+          - node.role == manager
+  notebook:
+    image: 127.0.0.1:5000/openstudio-jupyter
+    ports:
+      - "8888:8888"
+    volumes:
+      - osdata:/mnt/openstudio
+    depends_on:
+      - web
+      - web-background
+      - db
+      - queue
+      - rserve  
+volumes:
+  osdata:
+    external: true
+  dbdata:
+    external: true
+networks:
+  default:
+    driver: overlay
+    ipam:
+      driver: default
+      config:
+        - subnet: 172.28.0.0/16

local_setup_scripts/win64/docker_logs.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+declare -a list="$(docker service ls -q)"
+echo "$list"
+for app in $list; do
+  echo "$app"
+  docker service logs "$app"
+done

local_setup_scripts/win64/docker_logs2.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+declare -a list="$(docker container ls -q)"
+echo "$list"
+for app in $list; do
+  echo "$app"
+  docker logs "$app"
+done

local_setup_scripts/win64/nuke.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash -e
+echo "full redploy with OS version: $1"
+
+docker stack rm osserver || true
+docker service rm $(docker service ls -q) || true
+docker swarm leave -f || true
+echo "docker rm ps"
+docker rm -f $(docker ps -aq) || true
+echo "docker volume rm"
+docker volume rm $(docker volume ls -q) || true
+echo "docker image rm"
+docker image rm -f $(docker image ls -aq) || true
+
+echo "pull images"
+docker pull registry:2.6
+docker pull nrel/openstudio-server:$1
+docker pull nrel/openstudio-rserve:$1
+docker pull mongo:3.4.10
+docker pull redis:4.0.6
+
+echo "create registry"
+docker volume create --name=regdata
+docker swarm init
+docker service create --name registry --publish 5000:5000 --mount type=volume,source=regdata,destination=//c//Projects//OS-Server-fmu//local_setup_scripts registry:2.6
+sleep 10
+echo "tag"
+docker tag nrel/openstudio-server:$1 127.0.0.1:5000/openstudio-server
+docker tag nrel/openstudio-rserve:$1 127.0.0.1:5000/openstudio-rserve
+docker tag mongo:3.4.10 127.0.0.1:5000/mongo
+docker tag redis:4.0.6 127.0.0.1:5000/redis
+sleep 3
+echo "push"
+docker push 127.0.0.1:5000/openstudio-server
+docker push 127.0.0.1:5000/openstudio-rserve
+docker push 127.0.0.1:5000/mongo
+docker push 127.0.0.1:5000/redis
+
+echo "deploy"
+docker stack deploy osserver --compose-file=//c//Projects//OS-Server-fmu//local_setup_scripts//win64//docker-compose.yml &
+wait $!
+while ( nc -zv 127.0.0.1 80 3>&1 1>&2- 2>&3- ) | awk -F ":" '$3 != " Connection refused" {exit 1}'; do sleep 5; done
+docker service scale osserver_worker=1
+echo 'osserver stack redeployed'
+

local_setup_scripts/win64/rebuild_sr.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash -e
+cd /C/Projects/OS-Server-cli_debug
+docker stack rm osserver || true
+
+docker volume rm osdata -f || true
+docker volume rm dbdata -f || true
+
+while [ $(docker ps -q | wc -l) != 1 ]; do sleep 5; done
+docker image rm 127.0.0.1:5000/openstudio-server -f
+docker build . -t="127.0.0.1:5000/openstudio-server"
+docker push 127.0.0.1:5000/openstudio-server
+cd /C/Projects/OS-Server-cli_debug/docker/R
+docker image rm 127.0.0.1:5000/openstudio-rserve -f
+docker build . -t="127.0.0.1:5000/openstudio-rserve"
+docker push 127.0.0.1:5000/openstudio-rserve
+docker pull mongo:3.4.10
+docker tag mongo:3.4.10 127.0.0.1:5000/mongo
+docker push 127.0.0.1:5000/mongo
+#docker image rm mongo || true
+docker pull redis:4.0.6
+docker tag redis:4.0.6 127.0.0.1:5000/redis
+docker push 127.0.0.1:5000/redis
+cd /C/Projects/OS-Server-cli_debug/local_setup_scripts/win64
+docker stack deploy osserver --compose-file=docker-compose.yml
+while ( nc -zv 127.0.0.1 80 3>&1 1>&2- 2>&3- ) | awk -F ":" '$3 != " Connection refused" {exit 1}'; do sleep 5; done
+docker service scale osserver_worker=1
+echo 'osserver stack rebuilt and redeployed'
+

local_setup_scripts/win64/rebuild_sr_no_rm.sh

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash -e
+cd /C/Projects/OS-Server-fmu
+docker stack rm osserver || true
+
+docker volume rm osdata -f || true
+docker volume rm dbdata -f || true
+
+while [ $(docker ps -q | wc -l) != 1 ]; do sleep 5; done
+#docker image rm 127.0.0.1:5000/openstudio-server -f
+docker build . -t="127.0.0.1:5000/openstudio-server"
+docker push 127.0.0.1:5000/openstudio-server
+cd /C/Projects/OS-Server-fmu/docker/R
+#docker image rm 127.0.0.1:5000/openstudio-rserve -f
+docker build . -t="127.0.0.1:5000/openstudio-rserve"
+docker push 127.0.0.1:5000/openstudio-rserve
+docker pull mongo:3.4.10
+docker tag mongo:3.4.10 127.0.0.1:5000/mongo
+docker push 127.0.0.1:5000/mongo
+#docker image rm mongo || true
+docker pull redis:4.0.6
+docker tag redis:4.0.6 127.0.0.1:5000/redis
+docker push 127.0.0.1:5000/redis
+cd /C/Projects/OS-Server-fmu/local_setup_scripts/win64
+docker stack deploy osserver --compose-file=docker-compose.yml
+while ( nc -zv 127.0.0.1 80 3>&1 1>&2- 2>&3- ) | awk -F ":" '$3 != " Connection refused" {exit 1}'; do sleep 5; done
+docker service scale osserver_worker=1
+echo 'osserver stack rebuilt and redeployed'
+

local_setup_scripts/win64/redeploy.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash -e
+cd /C/Projects/OS-Server-fmu/
+docker stack rm osserver || true
+while [ $(docker ps -q | wc -l) != 1 ]; do sleep 5; done
+sleep 10
+echo 'docker volume rm osdata and dbdata'
+docker volume rm osdata -f || true
+docker volume rm dbdata -f || true
+docker stack deploy osserver --compose-file=/C/Projects/OS-Server-fmu/local_setup_scripts/win64/docker-compose.yml
+while ( nc -zv 127.0.0.1 80 3>&1 1>&2- 2>&3- ) | awk -F ":" '$3 != " Connection refused" {exit 1}'; do sleep 5; done
+docker service scale osserver_worker=1
+echo 'osserver stack redeployed'
+

server/app/jobs/dj_jobs/run_simulate_data_point.rb

@@ -185,14 +185,22 @@ def perform
       @sim_logger.info "Directory is #{simulation_dir}"
       run_log_file = File.join(run_dir, 'run.log')
       @sim_logger.info "Opening run.log file '#{run_log_file}'"
-
+      # add check for valid CLI option or ""
+      unless ['','--debug'].include?(@data_point.analysis.cli_debug)
+        @sim_logger.warn "CLI_Debug option: #{@data_point.analysis.cli_debug} is not valid.  Using --debug instead."
+        @data_point.analysis.cli_debug = '--debug'
+      end
+      unless ['','--verbose'].include?(@data_point.analysis.cli_verbose)
+        @sim_logger.warn "CLI_Verbose option: #{@data_point.analysis.cli_verbose} is not valid.  Using --verbose instead."
+        @data_point.analysis.cli_verbose = '--verbose'
+      end
       # Fail gracefully if the datapoint errors out by returning the zip and out.osw
       begin
         # Make sure to pass in preserve_run_dir
         run_result = nil
         File.open(run_log_file, 'a') do |run_log|
           begin
-            cmd = "#{Utility::Oss.oscli_cmd(@sim_logger)} --verbose run --workflow '#{osw_path}' --debug"
+            cmd = "#{Utility::Oss.oscli_cmd(@sim_logger)} #{@data_point.analysis.cli_verbose} run --workflow '#{osw_path}' #{@data_point.analysis.cli_debug}"
             process_log = File.join(simulation_dir, 'oscli_simulation.log')
             @sim_logger.info "Running workflow using cmd #{cmd} and writing log to #{process_log}"
             oscli_env_unset = Hash[Utility::Oss::ENV_VARS_TO_UNSET_FOR_OSCLI.collect{|x| [x,nil]}]

server/app/lib/openstudio_server/version.rb

@@ -37,6 +37,6 @@ module OpenstudioServer
   VERSION = '2.9.1'.freeze
   # format should be ^.*\-{1}[a-z]+[0-9]+
   # for example: -rc1, -beta6, -customusecase0
-  VERSION_EXT = '-rc1'.freeze # with preceding - or +
+  VERSION_EXT = ''.freeze # with preceding - or +
   OS_SHA = '3472e8b799'
 end

server/app/models/analysis.rb

@@ -49,6 +49,8 @@ class Analysis
   field :description, type: String
   field :run_flag, type: Boolean, default: false
   field :exit_on_guideline_14, type: Integer, default: 0
+  field :cli_debug, type: String, default: "--debug"  #set default to --debug so CI tests pass
+  field :cli_verbose, type: String, default: "--verbose"  #set default to --verbose to CI tests pass
 
   # Hash of the jobs to run for the analysis
   # field :jobs, type: Array, default: [] # very specific format