Feat: Create Elasticsearch elasticsearch_protocol Setting

README.md

@@ -67,8 +67,10 @@ _In a different terminal window `cd` into root of **common-workflow-service** fo
 
 For development we tend to create our own separate build script `<personal-dev.sh>` (firstinitial-lastname.sh), i.e.:`jsmith.sh`, that calls `dev.sh`. Here's an template for your personal build script that will work for development on a local machine:
 
-* Correctly set the Elasticsearch configuration within your personal script by including the proper protocol `http://` or `https://` in the Elasticsearch endpoint.
-    * Example: `ES_HOST="http://localhost"`
+* Correctly set the Elasticsearch configuration within your personal script by assigning the proper protocol, `HTTP` or `HTTPS`, to `ES_PROTOCOL` with Elasticsearch hostname assigned to `ES_HOST`.
+    * Example: 
+      * `ES_PROTOCOL="HTTP"`
+      * `ES_HOST="locahost"`
 
 ```
 #File: jsmith.sh
@@ -103,7 +105,8 @@ ADMIN_LAST="{last}"
 ADMIN_EMAIL="{email}"
 
 # ES config
-ES_HOST="http://localhost"
+ES_PROTOCOL="HTTP"  # options: 'HTTP' or 'HTTPS'
+ES_HOST="localhost"
 ES_PORT=9200
 ES_USE_AUTH=n
 ES_USERNAME="na"
@@ -117,7 +120,7 @@ NUM_WORKERS=1
 WORKER_ABANDONED_DAYS=1
 
 # Run the dev script
-./dev.sh `pwd` ${USER} ${DB_TYPE} ${DB_HOST} ${DB_PORT} ${DB_NAME} ${DB_USER} ${DB_PASS} ${ES_HOST} ${ES_PORT} ${ES_USE_AUTH} ${ES_USERNAME} ${ES_PASSWORD} ${CLOUD} ${SECURITY} ${HOSTNAME} ${EMAIL_LIST} ${ADMIN_FIRST} ${ADMIN_LAST} ${ADMIN_EMAIL} ${NUM_WORKERS} ${WORKER_ABANDONED_DAYS}
+./dev.sh `pwd` ${USER} ${DB_TYPE} ${DB_HOST} ${DB_PORT} ${DB_NAME} ${DB_USER} ${DB_PASS} ${ES_PROTOCOL} ${ES_HOST} ${ES_PORT} ${ES_USE_AUTH} ${ES_USERNAME} ${ES_PASSWORD} ${CLOUD} ${SECURITY} ${HOSTNAME} ${EMAIL_LIST} ${ADMIN_FIRST} ${ADMIN_LAST} ${ADMIN_EMAIL} ${NUM_WORKERS} ${WORKER_ABANDONED_DAYS}
 ```
 
 ###### Run Personal Dev Script

create_server_dist.sh

@@ -218,7 +218,6 @@ cp ${INSTALL_DIR}/launch_ls.sh                         ${CWS}
 
 print 'Copying Modeller scripts and libraries...'
 cp -R ${INSTALL_DIR}/modeler                    ${CWS}
-cp ${ROOT}/cws-modeler/install/modeler-config/* ${CWS}/modeler/
 
 print 'Installing context.xml to Tomcat...'
 cp ${INSTALL_DIR}/context.xml ${CWS_TOMCAT_ROOT}/conf/context.xml

cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java

@@ -169,7 +169,10 @@ public class CwsInstaller {
 	private static String cws_security_filter_class;
 	private static String startup_autoregister_process_defs;
 	private static String cws_token_expiration_hours;
+	private static String elasticsearch_protocol;
+	private static String elasticsearch_protocol_init;
 	private static String elasticsearch_host;
+	private static String elasticsearch_host_init;
 	private static String elasticsearch_port;
 	private static String elasticsearch_use_auth;
 	private static String elasticsearch_username;
@@ -1064,33 +1067,85 @@ private static void setupLogstash() {
 
 	private static void setupElasticsearch() {
 
+		// PROMPT USER FOR ELASTICSEARCH PROTOCOL
+		elasticsearch_protocol = getPreset("elasticsearch_protocol");
+
+		if (cws_installer_mode.equals("interactive")) {
+			if (elasticsearch_protocol == null) {
+
+				String read_elasticsearch_protocol = "";
+				while (!read_elasticsearch_protocol.toLowerCase().startsWith("https") &&
+					!read_elasticsearch_protocol.toLowerCase().startsWith("http")) {
+					read_elasticsearch_protocol = readRequiredLine("Enter the Elasticsearch protocol (be sure to use HTTP or HTTPS):  ",
+						"You must enter a protocol");
+				}
+
+				elasticsearch_protocol_init = read_elasticsearch_protocol;
+				elasticsearch_protocol = read_elasticsearch_protocol.toLowerCase();
+				if (elasticsearch_protocol.startsWith("https")) {
+					elasticsearch_protocol = "https";
+				}
+				if (elasticsearch_protocol.startsWith("http")) {
+					elasticsearch_protocol = "http";
+				}
+			} else {
+				elasticsearch_protocol = readLine("Enter the Elasticsearch protocol. " + "Default is " + elasticsearch_protocol + ": ", elasticsearch_protocol);
+			}
+		} else {
+			if (elasticsearch_protocol == null) {
+				bailOutMissingOption("elasticsearch_protocol");
+			}
+
+			elasticsearch_protocol_init = elasticsearch_protocol;
+			elasticsearch_protocol = elasticsearch_protocol.toLowerCase();
+			if (elasticsearch_protocol.startsWith("https")) {
+				elasticsearch_protocol = "https";
+			} else if (elasticsearch_protocol.startsWith("http")) {
+				elasticsearch_protocol = "http";
+			} else {
+				bailOutWithMessage("ERROR: elasticsearch_protocol config input is '" +  elasticsearch_protocol
+					+ "' ... Be sure to use 'HTTP' or 'HTTPS' for elasticsearch_protocol configuration.");
+			}
+		}
+
+		log.debug("elasticsearch_protocol: " + elasticsearch_protocol);
+
+
 		// PROMPT USER FOR ELASTICSEARCH HOST
 		elasticsearch_host = getPreset("elasticsearch_host");
 
 		if (cws_installer_mode.equals("interactive")) {
 			if (elasticsearch_host == null) {
 
 				String read_elasticsearch_host = "";
-
-				while (!read_elasticsearch_host.startsWith("https://") &&
-						!read_elasticsearch_host.startsWith("http://")) {
-					read_elasticsearch_host = readRequiredLine("Enter the Elasticsearch host (be sure to include protocol in URL: http:// or https://):  ",
+				read_elasticsearch_host = readRequiredLine("Enter the Elasticsearch host:  ",
 							"You must enter a hostname");
-				}
 
+				elasticsearch_host_init = read_elasticsearch_host;
 				elasticsearch_host = read_elasticsearch_host.toLowerCase();
+				if (elasticsearch_host.startsWith("http:/") || elasticsearch_host.startsWith("http://") ||
+					elasticsearch_host.startsWith("https:/") || elasticsearch_host.startsWith("https://")) {
+					elasticsearch_host = elasticsearch_host.replaceAll("http://", "").replaceAll("http:/","").replaceAll("https://","").replaceAll("https:/","");
+				}
 			} else {
-				elasticsearch_host = readLine("Enter the Elasticsearch host. " +
-						"Default is " + elasticsearch_host + ": ", elasticsearch_host);
+				elasticsearch_host = readLine("Enter the Elasticsearch host. " + "Default is " + elasticsearch_host + ": ", elasticsearch_host);
 			}
 		} else {
 			if (elasticsearch_host == null) {
 				bailOutMissingOption("elasticsearch_host");
 			}
+
+			elasticsearch_host_init = elasticsearch_host;
+			elasticsearch_host = elasticsearch_host.toLowerCase();
+			if (elasticsearch_host.startsWith("http:/") || elasticsearch_host.startsWith("http://") ||
+				elasticsearch_host.startsWith("https:/") || elasticsearch_host.startsWith("https://")) {
+				elasticsearch_host = elasticsearch_host.replaceAll("http://", "").replaceAll("http:/","").replaceAll("https://","").replaceAll("https:/","");
+			}
 		}
 
 		log.debug("elasticsearch_host: " + elasticsearch_host);
 
+
 		// PROMPT USER FOR ELASTICSEARCH PORT
 		elasticsearch_port = getPreset("elasticsearch_port");
 
@@ -1489,7 +1544,8 @@ private static void showInstallationInfo() {
 		print("SMTP host                     = " + cws_smtp_hostname);
 		print("SMTP port                     = " + cws_smtp_port);
 		print("....................................................................................");
-		print("Elasticsearch URL             = " + elasticsearch_host);
+		print("Elasticsearch Protocol        = " + elasticsearch_protocol);
+		print("Elasticsearch Host            = " + elasticsearch_host);
 		print("Elasticsearch Port            = " + elasticsearch_port);
 		if (elasticsearch_use_auth.equalsIgnoreCase("Y")) {
 			print("Elasticsearch User            = " + elasticsearch_username);
@@ -1854,14 +1910,32 @@ else if (Integer.valueOf(cws_amq_port) < 1024) {
 	 *
 	 */
 	private static int validateElasticsearch() {
-		print("checking that user provided Elasticsearch (" + elasticsearch_host + ":" + elasticsearch_port + ") is running...");
+		print("checking that user provided Elasticsearch (" + elasticsearch_protocol + "://" + elasticsearch_host + ":" + elasticsearch_port + ") is running...");
 
 		try {
-			String[] cmdArray = new String[] {"curl", "--fail", elasticsearch_host + ":" + elasticsearch_port + "/_cluster/health"};
+			if (!(elasticsearch_protocol.startsWith("http") || elasticsearch_protocol.startsWith("https")) ) {
+				print("   [WARNING]");
+				print("       It was determined that the user provided Elasticsearch endpoint protocol '" + elasticsearch_protocol + "' did not properly set or protocol to 'HTTP' OR 'HTTPS'");
+				print("");
+				return 1;
+			}
+
+			if (elasticsearch_protocol == "http" && elasticsearch_host_init.toLowerCase().startsWith("https") ||
+				elasticsearch_protocol == "https" && elasticsearch_host_init.toLowerCase().startsWith("http")) {
+				print("   [SETUP RESOLUTION]");
+				print("       It was determined that the user provided elasticsearch_protocol and elasticsearch_host have mismatched protocol identifiers.");
+				print("          elasticsearch_protocol=" + elasticsearch_protocol_init + "  ");
+				print("          elasticsearch_host=" + elasticsearch_host_init + "  ");
+				print("");
+				print("       CWS Installation will default to using given elasticsearch_protocol value: " + elasticsearch_protocol_init + " ");
+				print("");
+			}
+
+			String[] cmdArray = new String[] {"curl", "--fail", elasticsearch_protocol + "://" + elasticsearch_host + ":" + elasticsearch_port + "/_cluster/health"};
 
 			if (elasticsearch_use_auth.equalsIgnoreCase("Y")) {
 				// Add auth to curl
-				cmdArray = new String[] {"curl", "--fail", "-u", elasticsearch_username + ":" + elasticsearch_password, elasticsearch_host + ":" + elasticsearch_port + "/_cluster/health"};
+				cmdArray = new String[] {"curl", "--fail", "-u", elasticsearch_username + ":" + elasticsearch_password, elasticsearch_protocol + "://" + elasticsearch_host + ":" + elasticsearch_port + "/_cluster/health"};
 			}
 
 			Process p = Runtime.getRuntime().exec(cmdArray);
@@ -1873,19 +1947,20 @@ private static int validateElasticsearch() {
 			if (p.exitValue() != 0) {
 				print("   [WARNING]");
 				print("       It was determined that the user provided Elasticsearch is not running or is inaccessible.");
+				print("       .........................................................................................");
+				print("           [ELASTICSEARCH]: Configuration Details");
+				print("               elasticsearch_protocol=" + elasticsearch_protocol_init + "  ");
+				print("               elasticsearch_host=" + elasticsearch_host_init + "  ");
+				print("               elasticsearch_port=" + elasticsearch_port + "  ");
+				print("       .........................................................................................");
 				print("");
-				return 1;
-			}
 
-			if (!(elasticsearch_host.startsWith("https://") || elasticsearch_host.startsWith("http://")) ) {
-				print("   [WARNING]");
-				print("       It was determined that the user provided Elasticsearch endpoint '" + elasticsearch_host + "' did not properly set or include protocol 'http://' OR 'https://'");
-				print("");
 				return 1;
 			}
 
 			print("   [OK]");
 			print("");
+
 			return 0; // no warnings
 
 		} catch (Exception e) {
@@ -2277,6 +2352,7 @@ private static void updateCwsUiProperties() throws IOException {
 		content = content.replace("__CWS_DB_USERNAME__",                 cws_db_username);
 		content = content.replace("__CWS_DB_PASSWORD__",                 cws_db_password);
 		content = content.replace("__CWS_CONSOLE_SSL_PORT__",            cws_console_ssl_port);
+		content = content.replace("__CWS_ES_PROTOCOL__",                 elasticsearch_protocol);
 		content = content.replace("__CWS_ES_HOST__",                     elasticsearch_host);
 		content = content.replace("__CWS_ES_PORT__",                     elasticsearch_port);
 		content = content.replace("__CWS_ES_USE_AUTH__",                 elasticsearch_use_auth);
@@ -2365,6 +2441,7 @@ private static void updateCwsUiConfig() throws IOException {
 		// Update clean_es_history.sh file
 		path = Paths.get(config_work_dir + SEP + "clean_es_history.sh");
 		content = getFileContents(path);
+		content = content.replace("__ES_PROTOCOL__",      			elasticsearch_protocol);
 		content = content.replace("__ES_HOST__",      				elasticsearch_host);
 		content = content.replace("__ES_PORT__",  					elasticsearch_port);
 		content = content.replace("__ES_USE_AUTH__",                 elasticsearch_use_auth);
@@ -2507,6 +2584,7 @@ private static void installLogstash() throws IOException {
 		catalinaLogPath = catalinaLogPath.replace("\\", "/");
 		logstashContent = logstashContent.replace("__CWS_CATALINA_OUT_PATH__", catalinaLogPath);
 
+		logstashContent = logstashContent.replace("__CWS_ES_PROTOCOL__", elasticsearch_protocol);
 		logstashContent = logstashContent.replace("__CWS_ES_HOST__", elasticsearch_host);
 		logstashContent = logstashContent.replace("__CWS_ES_PORT__", elasticsearch_port);
 		if (elasticsearch_use_auth.equalsIgnoreCase(("Y"))) {
@@ -2575,6 +2653,7 @@ private static void writeOutConfigurationFile() {
 		setPreset("metrics_publishing_interval", metrics_publishing_interval);
 		setPreset("cws_notification_emails", cws_notification_emails);
 		setPreset("cws_token_expiration_hours", cws_token_expiration_hours);
+		setPreset("elasticsearch_protocol", elasticsearch_protocol);
 		setPreset("elasticsearch_host", elasticsearch_host);
 		setPreset("elasticsearch_port", elasticsearch_port);
 		setPreset("elasticsearch_use_auth", elasticsearch_use_auth);

cws-service/src/main/java/jpl/cws/controller/RestService.java

@@ -108,7 +108,8 @@ public class RestService extends MvcCore {
 
 	@Value("${cws.console.app.root}") private String appRoot;
 	@Value("${cws.install.hostname}") private String hostName;
-
+
+	@Value("${cws.elasticsearch.protocol}") private String elasticsearchProtocolName;
 	@Value("${cws.elasticsearch.hostname}") private String elasticsearchHostname;
 	@Value("${cws.elasticsearch.port}") private String elasticsearchPort;
 
@@ -405,7 +406,7 @@ private String doDeployProcessDefinition(MultipartFile file) {
 	 * @return fully constructed elasticsearch URL string
 	 */
 	private String constructElasticsearchUrl(String subPath) {
-		String urlString = elasticsearchHostname + ":" + elasticsearchPort + subPath;
+		String urlString = elasticsearchProtocolName + "://" + elasticsearchHostname + ":" + elasticsearchPort + subPath;
 		return urlString;
 	}
 

dev.sh

@@ -12,20 +12,21 @@ DB_PORT=${5}
 DB_NAME=${6}
 DB_USER=${7}
 DB_PASS=${8}
-ES_HOST=${9}
-ES_PORT=${10}
-ES_USE_AUTH=${11}
-ES_USERNAME=${12}
-ES_PASSWORD=${13}
-ENABLE_CLOUD_AS=${14}
-SECURITY_SCHEME=${15}
-THIS_HOSTNAME=${16}
-NOTIFICATION_EMAILS=${17}
-ADMIN_FIRSTNAME=${18}
-ADMIN_LASTNAME=${19}
-ADMIN_EMAIL=${20}
-NUM_WORKERS=${21}
-WORKER_ABANDONED_DAYS=${22}
+ES_PROTOCOL=${9}
+ES_HOST=${10}
+ES_PORT=${11}
+ES_USE_AUTH=${12}
+ES_USERNAME=${13}
+ES_PASSWORD=${14}
+ENABLE_CLOUD_AS=${15}
+SECURITY_SCHEME=${16}
+THIS_HOSTNAME=${17}
+NOTIFICATION_EMAILS=${18}
+ADMIN_FIRSTNAME=${19}
+ADMIN_LASTNAME=${20}
+ADMIN_EMAIL=${21}
+NUM_WORKERS=${22}
+WORKER_ABANDONED_DAYS=${23}
 
 source ${ROOT}/utils.sh
 

install/clean_es_history.sh

@@ -9,6 +9,7 @@ ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 
 source ${ROOT}/utils.sh
 
+ES_PROTOCOL='__ES_PROTOCOL__'
 ES_HOST='__ES_HOST__'
 ES_PORT='__ES_PORT__'
 ES_USE_AUTH='__ES_USE_AUTH__'

install/cws-ui/cws-ui.properties

@@ -16,6 +16,7 @@ cws.db.port=__CWS_DB_PORT__
 cws.db.username=__CWS_DB_USERNAME__
 cws.db.password=__CWS_DB_PASSWORD__
 
+cws.elasticsearch.protocol=__CWS_ES_PROTOCOL__
 cws.elasticsearch.hostname=__CWS_ES_HOST__
 cws.elasticsearch.port=__CWS_ES_PORT__
 cws.elasticsearch.use.auth=__CWS_ES_USE_AUTH__

install/docker/worker-ls/cws-logstash.conf

@@ -53,5 +53,5 @@ filter {
 }
 
 output {
-  elasticsearch { hosts => ["__ES_HOST__:9200"] }
+  elasticsearch { hosts => ["__ES_PROTOCOL__://__ES_HOST__:9200"] }
 }

install/docker/worker-ls/docker-compose.yml

@@ -39,6 +39,7 @@ services:
        - DB_HOST=__DB_HOST__
        - DB_USER=root
        - DB_PW=test
+       - ES_PROTOCOL=__ES_PROTOCOL__
        - ES_HOST=__ES_HOST__
        - ES_PORT=9200
     volumes:

install/example-cws-configuration.properties

@@ -53,7 +53,7 @@ database_password=[YourPassword]
 # The LDAP username for the user that will be the CWS Administrator.
 # This User will have the initial permissions in CWS, and will have the ability
 # to add users and modify permissions.
-# Example:  ghollins
+# Example:  jsmith
 admin_user=[YourJPLUsername]
 
 # The first and last name, and email address of the CWS Administrator.
@@ -116,6 +116,13 @@ cws_notification_emails=[AdminEmailAddresses(comma separated)]
 # This allows CWS to use a user provided Elasticsearch service.
 # If you wish to use an unsecured Elasticsearch host, you may do so
 # by specifying elasticsearch_use_auth=n below.
+# Also specify if your provided Elasticsearch is encrypted(HTTPS) or non-encrypted(HTTP)
+# by setting elasticsearch_protocol to HTTPS or HTTP.
+# Examples:   elasticsearch_protocol=http
+# or          elasticsearch_protocol=https
+# or          elasticsearch_protocol=HTTP
+#             elasticsearch_protocol=HTTPS
+elasticsearch_protocol=[YourElasticsearchProtocol]
 elasticsearch_host=[YourElasticsearchHost]
 elasticsearch_port=9200
 elasticsearch_use_auth=n

install/logging/cws-logstash.conf

@@ -55,7 +55,7 @@ filter {
 
 output {
   elasticsearch {
-    hosts => ["__CWS_ES_HOST__:__CWS_ES_PORT__"]
+    hosts => ["__CWS_ES_PROTOCOL____CWS_ES_HOST__:__CWS_ES_PORT__"]
     ilm_enabled => false
     ssl => __LOGSTASH_ES_USE_SSL__
     __LOGSTASH_ES_USERNAME__