Elasticsearch 6 upgrade

Vagrantfile

@@ -36,6 +36,14 @@ if not plugins_to_install.empty?
   end
 end
 
+if ENV['VAGRANT_VM_NAME'] == nil
+  ENV['VAGRANT_VM_NAME'] = 'dendroVagrantDemo'
+end
+
+if ENV['VAGRANT_VM_IP'] == nil
+  ENV['VAGRANT_VM_IP'] = '192.168.56.249'
+end
+
 ###Configuration
 puts "Configuring Vagrant VM #{ENV['VAGRANT_VM_NAME']} on IP #{ENV['VAGRANT_VM_IP']}."
 
@@ -44,7 +52,7 @@ if "#{ENV['JENKINS_BUILD']}" == '1'
 end
 
 Vagrant.configure("2") do |config|
-  
+
   if "#{ENV['VAGRANT_USE_SQUID_PROXY_VM']}" == 'true'
     puts "Using Squid proxy VM for quicker deployments..."
     # Enable caching web server
@@ -75,10 +83,10 @@ Vagrant.configure("2") do |config|
   else
     config.vm.boot_timeout= 600
   end
-  
+
   config.vm.box = "ubuntu/xenial64"
   config.vm.box_version = "20170922.0.0"
-  
+
 
   #set maximum size of the main hard drive
   config.disksize.size = '100GB'
@@ -119,8 +127,8 @@ Vagrant.configure("2") do |config|
      else
 
      end
-      vb.cpus = 2     
-      vb.memory = "2048"
+      vb.cpus = 2
+      vb.memory = "4096"
   end
 
   time = sanitize_filename(Time.new.inspect)

install.sh

@@ -100,7 +100,7 @@ then
   source ./halt_vm.sh
   VBoxManage snapshot $active_deployment_setting restore $snapshot_id
 else
-  take_vm_snapshot $active_deployment_setting $snapshot_name
+  take_vm_snapshot $active_deployment_setting "before_$snapshot_name"
 fi
 
 if [[ "$use_proxy_vm" == "true" ]]; then
@@ -144,6 +144,9 @@ else
   die "There were errors installing Dendro."
 fi
 
+info "Taking VM snapshot after operation..."
+take_vm_snapshot $active_deployment_setting "after_$snapshot_name"
+
 info "Cleaning up..."
 rm ./scripts.tar.gz
 success "Deleted temporary scripts package."

scripts/Checks/load_nvm.sh

@@ -2,12 +2,6 @@
 
 node_version=$1
 
-load_nvm()
-{
-  export NVM_DIR="$HOME/.nvm" &&
-  [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && echo "NVM loaded."
-}
-
 add_line_at_end_of_file_if_tag_not_present()
 {
 	local file=$1
@@ -40,7 +34,8 @@ install_nvm()
 
   #install NVM, use node version
   curl -o- https://raw.githubusercontent.com/creationix/nvm/v0.33.2/install.sh | bash > /dev/null &&
-  load_nvm
+  export NVM_DIR="$HOME/.nvm" &&
+  [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && echo "NVM loaded."
 }
 
 install_node()
@@ -101,7 +96,8 @@ then
     install_nvm
   else
     echo "User $(whoami) has nvm installed. Trying to load it..."
-    load_nvm
+    export NVM_DIR="$HOME/.nvm" &&
+    [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" && echo "NVM loaded."
   fi
 fi
 

scripts/Dependencies/elasticsearch.sh

@@ -8,7 +8,7 @@ else
 	source ./constants.sh
 fi
 
-info "Installing ElasticSearch 2.2.3......"
+info "Installing ElasticSearch 6.2.2......"
 
 #save current dir
 setup_dir=$(pwd)
@@ -24,13 +24,11 @@ sudo apt-get install --yes oracle-java8-installer
 sudo sh -c "yes \"\" | apt-get -f install"
 
 #install elasticsearch
-cd $temp_downloads_folder
-sudo wget --progress=bar:force https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.3.3/elasticsearch-2.3.3.deb
-sudo dpkg -i elasticsearch-2.3.3.deb
-
-
+cd $temp_downloads_folder || die "Unable to cd to $temp_downloads_folder"
+sudo wget --progress=bar:force https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.2.2.deb
+sudo dpkg -i elasticsearch-6.2.2.deb || die "Error installing ElasticSearch!"
 
 #go back to initial dir
-cd $setup_dir
+cd $setup_dir || die "Unable to cd to $setup_dir"
 
-success "Installed ElasticSearch 2.2.3."
+success "Installed ElasticSearch 6.2.2."

scripts/Fixes/set_dev_mode.sh

@@ -20,9 +20,9 @@ file_exists file_exists_flag $mongodb_conf_file
 if [[ "$file_exists_flag" == "true" ]]; then
 	info "File $mongodb_conf_file exists..."
 	patch_file $mongodb_conf_file "bindIp: 127.0.0.1" "#bindIp: 127.0.0.1" "mongodb_dendro_dev_patch" && success "Opened MongoDB." || die "Unable to patch mongodb configuration file."
-	sudo systemctl enable mongod.service && 
+	sudo systemctl enable mongod.service &&
 	sudo systemctl daemon-reload &&
-	sudo service mongod start && 
+	sudo service mongod start &&
 	sudo service mongod restart || die "Unable to restart mongodb service."
 else
 	die "File $mongodb_conf_file does not exist."
@@ -33,24 +33,28 @@ info "Trying to open ElasticSearch to ANY remote connection."
 file_exists file_exists_flag $elasticsearch_conf_file
 if [[ "$file_exists_flag" == "true" ]]; then
 	info "File $elasticsearch_conf_file exists..."
-	patch_file $elasticsearch_conf_file "# network.host: 192.168.0.1" "network.host: 0.0.0.0" "elasticsearch_dendro_dev_patch_network_host" && success "Set ElasticSearch HOST." || die "Unable to patch ElasticSearch configuration file (hostname)."
+	patch_file $elasticsearch_conf_file "#network.host: 192.168.0.1" "network.host: 0.0.0.0" "elasticsearch_dendro_dev_patch_network_host" && success "Set ElasticSearch HOST." || die "Unable to patch ElasticSearch configuration file (hostname)."
 	patch_file $elasticsearch_conf_file "http.port: 9200" "http.port: 9200" "elasticsearch_dendro_dev_patch_network_port" && success "Set ElasticSearch PORT." || die "Unable to patch ElasticSearch configuration file (port)."  &&
-	sudo service elasticsearch start && 
+	sudo service elasticsearch start &&
 	sudo service elasticsearch restart || die "Unable to restart ElasticSearch service."
 else
 	die "File $elasticsearch_conf_file does not exist."
 fi
 
-#install management gui
-sudo /usr/share/elasticsearch/bin/plugin install royrusso/elasticsearch-HQ/v2.0.3
+# # install kibana
+# wget https://artifacts.elastic.co/downloads/kibana/kibana-6.2.2-amd64.deb
+# shasum -a 512 kibana-6.2.2-amd64.deb
+# sudo dpkg -i kibana-6.2.2-amd64.deb
+# sudo systemctl enable kibana.service
+# sudo service kibana start
 
 ##Redis
 info "Trying to open Redis to ANY remote connection."
 file_exists file_exists_flag $redis_conf_file
 if [[ "$file_exists_flag" == "true" ]]; then
 	info "File $redis_conf_file exists..."
 	patch_file $redis_conf_file "bind 127.0.0.1" "bind 0.0.0.0" "redis_dendro_dev_patch"  && success "Opened Redis." || die "Unable to patch Redis configuration file." &&
- 	sudo service redis start && 
+ 	sudo service redis start &&
 	sudo service redis restart || die "Unable to restart Redis service."
 	#setup multiple redis instances
 	source ./Dependencies/Redis/setup_redis_instances.sh

scripts/SQLCommands/grant_commands.sh

@@ -15,30 +15,60 @@ else
 	running_folder=$script_dir/SQLCommands
 fi
 
-function wait_for_virtuoso_to_boot()
+function wait_for_server_to_boot_on_port()
 {
-    echo "Waiting for virtuoso to boot up..."
-    attempts=0
-    max_attempts=30
-    while ( nc 127.0.0.1 8890 < /dev/null || nc 127.0.0.1 1111 < /dev/null )  && [[ $attempts < $max_attempts ]] ; do
+    local port=$1
+    local attempts=0
+    local max_attempts=60
+
+    echo "Waiting for server on port $port to boot up..."
+
+    while (
+						nc 127.0.0.1 "$port" < /dev/null ||
+						telnet 127.0.0.1 $port
+					) && [[ $attempts < $max_attempts ]] ; do
         attempts=$((attempts+1))
-        sleep 1;
-        echo "waiting... (${attempts}/${max_attempts})"
+        if [[ "$attempts" == "$max_attempts" ]]
+        then
+            break;
+        else
+            sleep 1;
+            echo "waiting... (${attempts}/${max_attempts})"
+        fi
     done
+
+		if [[ "$attempts" == "$max_attempts" ]]
+    then
+				echo "Server on port $port failed to start after $max_attempts"
+		else
+			echo "Server on port $port started successfully at attempt (${attempts}/${max_attempts})"
+    fi
 }
 
-wait_for_virtuoso_to_boot
+sudo service virtuoso start
+
+# echo "sleeping for 30 seconds before attempting to load ontologies into virtuoso"
+# sleep 30
+
+wait_for_server_to_boot_on_port 8890
+wait_for_server_to_boot_on_port 1111
+
+echo "trying to run ontology loading commands in virtuoso"
+
+/usr/local/virtuoso-opensource/bin/isql 1111 "$virtuoso_dba_user" "$virtuoso_dba_password" < $running_folder/interactive_sql_commands.sql
+/usr/local/virtuoso-opensource/bin/isql 1111 "$virtuoso_dba_user" "$virtuoso_dba_password" < $running_folder/declare_namespaces.sql
+
+# flush OS buffers
+sync
+
+# force virtuoso to save the data to the hard drive
+/usr/local/virtuoso-opensource/bin/isql 1111 "$virtuoso_dba_user" "$virtuoso_dba_password" "EXEC=checkpoint;"
 
 # change the default password if it is set as default and the password is different
 if [[ "${virtuoso_dba_password}" != "dba" ]]
 then
-	echo "set password dba ${virtuoso_dba_password};" | /usr/local/virtuoso-opensource/bin/isql 127.0.0.1 "dba" "dba"
+	echo "set password dba ${virtuoso_dba_password};" | /usr/local/virtuoso-opensource/bin/isql 127.0.0.1 "dba" "dba" || die "Unable to set the custom password for virtuoso user $virtuoso_dba_user"
 fi
 
-/usr/local/virtuoso-opensource/bin/isql 1111 "$virtuoso_dba_user" "$virtuoso_dba_password" < $running_folder/interactive_sql_commands.sql || die "Unable to load ontologies into Virtuoso."
-/usr/local/virtuoso-opensource/bin/isql 1111 "$virtuoso_dba_user" "$virtuoso_dba_password" < $running_folder/declare_namespaces.sql || die "Unable to setup namespaces"
-
-success "Installed base ontologies in virtuoso."
-
 #go back to initial dir
 cd $setup_dir

scripts/Services/elasticsearch.sh

@@ -15,8 +15,8 @@ setup_dir=$(pwd)
 
 #set elasticsearch startup service
 sudo systemctl daemon-reload
-sudo systemctl enable elasticsearch
-sudo systemctl start elasticsearch
+sudo systemctl enable elasticsearch &&
+sudo systemctl start elasticsearch || die "Error starting ElasticSearch service! Maybe it did not install correctly!"
 
 #install elasticsearch gui client
 sudo /usr/share/elasticsearch/bin/plugin install jettro/elasticsearch-gui

scripts/constants.sh

@@ -79,7 +79,7 @@ recommender_installation_path='/dendro_recommender'
 
 	#dependencies
 		#nodejs version
-		node_version="8.9.0"
+		node_version="8.10.0"
 
 		#elasticsearch
 		elasticsearch_port=9200
@@ -167,7 +167,7 @@ recommender_installation_path='/dendro_recommender'
 
 #multi-core configuration
 	num_cpus=1
-	
+
 #Vagrant-specific configuration
 
 #VAGRANT_VM_SSH_USERNAME="vagrant"

scripts/install.sh

@@ -156,7 +156,11 @@ warning "Installing NVM as $(whoami) in order to install node version $node_vers
 #This can fail without dying because not always we have a 'ubuntu' user
 sudo su - "ubuntu" -c "$setup_dir/Checks/load_nvm.sh $node_version"
 
-if [ "${set_dev_mode}" != "true" ] && [ "${unset_dev_mode}" != "true" ] && [ "$install_jenkins" != "true" ] && [ "$install_teamcity" != "true" ] && [ "$install_teamcity_agent" != "true" ]
+if 	[ "${set_dev_mode}" != "true" ] && \
+		[ "${unset_dev_mode}" != "true" ] && \
+		[ "$install_jenkins" != "true" ] && \
+		[ "$install_teamcity" != "true" ] && \
+		[ "$install_teamcity_agent" != "true" ]
 then
 	info "Running the Dendro User Setup."
 
@@ -188,13 +192,6 @@ then
 				source ./Services/virtuoso.sh
 			fi
 
-			timeout=45
-			info "Waiting for virtuoso service to start. Installing base ontologies in virtuoso in $timeout seconds..."
-			for (( i = 0; i < $timeout; i++ )); do
-				echo -ne $[$timeout-i]...
-				sleep 1s
-			done
-
 			source ./SQLCommands/grant_commands.sh
 
 			# Install MongoDB
@@ -203,7 +200,7 @@ then
 
 			#source ./Dependencies/drawing_to_text.sh #TODO this crashes still with GCC 5.8+. Commenting
 			source ./Dependencies/Redis/setup_redis_instances.sh
-			
+
 
 			if [[ "$dendro_recommender_active" == "true" ]]
 			then
@@ -214,7 +211,7 @@ then
 
 			source ./Dependencies/elasticsearch.sh
 			source ./Services/elasticsearch.sh
-			
+
 			#source ./Checks/check_services_status.sh
 		fi