diff --git a/CHANGELOG.md b/CHANGELOG.md index fb130d61..e92668f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [X.Y.Z](https://github.com/NASA-AMMOS//releases/tag/X.Y.Z) - 2023-MM-DD -### Added - -- +### Added +- IDS-10140 + - Configure Elasticsearch Index name with Prefix setting: `elasticsearch_index_prefix` +### Changed +### Deprecated +### Removed +### Fixed +### Security \ No newline at end of file diff --git a/ci/ci.sh b/ci/ci.sh index 297156a1..fb0ed3a5 100755 --- a/ci/ci.sh +++ b/ci/ci.sh @@ -12,20 +12,21 @@ DB_PASS=${8} ES_PROTOCOL=${9} ES_HOST=${10} -ES_PORT=${11} -ES_USE_AUTH=${12} -ES_USERNAME=${13} -ES_PASSWORD=${14} +ES_INDEX_PREFIX=${11} +ES_PORT=${12} +ES_USE_AUTH=${13} +ES_USERNAME=${14} +ES_PASSWORD=${15} -CLOUD=${15} -SECURITY=${16} -HOSTNAME=${17} -EMAIL_LIST=${18} -ADMIN_FIRST=${19} -ADMIN_LAST=${20} -ADMIN_EMAIL=${21} -NUM_WORKERS=${22} -WORKER_ABANDONED_DAYS=${23} +CLOUD=${16} +SECURITY=${17} +HOSTNAME=${18} +EMAIL_LIST=${19} +ADMIN_FIRST=${20} +ADMIN_LAST=${21} +ADMIN_EMAIL=${22} +NUM_WORKERS=${23} +WORKER_ABANDONED_DAYS=${24} source ${ROOT}/../utils.sh @@ -105,6 +106,7 @@ project_webapp_root=proj cws_enable_cloud_autoscaling=n elasticsearch_protocol=${ES_PROTOCOL} elasticsearch_host=${ES_HOST} +elasticsearch_index_prefix=${ES_INDEX_PREFIX} elasticsearch_use_auth=${ES_USE_AUTH} elasticsearch_port=${ES_PORT} elasticsearch_username=${ES_USERNAME} @@ -115,6 +117,7 @@ default_cws_ldap_url=ldap://localhost:389 cws_ldap_url=ldap://localhost:389 default_elasticsearch_use_auth=n aws_cloudwatch_endpoint=monitoring.us-west-1.amazonaws.com +default_elasticsearch_index_prefix=cws-index default_elasticsearch_port=9200 default_aws_cloudwatch_endpoint=monitoring.us-west-1.amazonaws.com aws_sqs_dispatcher_msgFetchLimit=1 @@ -167,11 +170,13 @@ email_subject=[CWS] You have been assigned a task (CWS_TASK_NAME) email_body=fn:CWS_USER_FIRSTNAME
ln:CWS_USER_LASTNAME,
tn:(CWS_TASK_NAME), em:CWS_USER_EMAIL elasticsearch_protocol=${ES_PROTOCOL} elasticsearch_host=${ES_HOST} +elasticsearch_index_prefix=${ES_INDEX_PREFIX} elasticsearch_use_auth=${ES_USE_AUTH} elasticsearch_port=${ES_PORT} elasticsearch_username=${ES_USERNAME} elasticsearch_password=${ES_PASSWORD} smtp_hostname=smtp.localhost +default_elasticsearch_index_prefix=cws-index default_smtp_hostname=smtp.localhost default_cws_ldap_url=ldap://localhost:389 cws_ldap_url=ldap://localhost:389 diff --git a/ci/run_ci.sh b/ci/run_ci.sh index 7474bfac..dd607c05 100755 --- a/ci/run_ci.sh +++ b/ci/run_ci.sh @@ -31,6 +31,7 @@ ADMIN_EMAIL="/" ES_PROTOCOL="HTTP" ES_HOST="http://localhost" +ES_INDEX_PREFIX="cws-index" ES_PORT=9200 ES_USE_AUTH=n ES_USERNAME="na" @@ -44,4 +45,4 @@ NUM_WORKERS=${2} # parameter is passed as an env through workflow file WORKER_ABANDONED_DAYS=1 # Run the ci script -./ci.sh `pwd` ${USER} ${DB_TYPE} ${DB_HOST} ${DB_PORT} ${DB_NAME} ${DB_USER} ${DB_PASS} ${ES_PROTOCOL} ${ES_HOST} ${ES_PORT} ${ES_USE_AUTH} ${ES_USERNAME} ${ES_PASSWORD} ${CLOUD} ${SECURITY} ${HOSTNAME} ${EMAIL_LIST} ${ADMIN_FIRST} ${ADMIN_LAST} ${ADMIN_EMAIL} ${NUM_WORKERS} ${WORKER_ABANDONED_DAYS} +./ci.sh `pwd` ${USER} ${DB_TYPE} ${DB_HOST} ${DB_PORT} ${DB_NAME} ${DB_USER} ${DB_PASS} ${ES_PROTOCOL} ${ES_HOST} ${ES_INDEX_PREFIX} ${ES_PORT} ${ES_USE_AUTH} ${ES_USERNAME} ${ES_PASSWORD} ${CLOUD} ${SECURITY} ${HOSTNAME} ${EMAIL_LIST} ${ADMIN_FIRST} ${ADMIN_LAST} ${ADMIN_EMAIL} ${NUM_WORKERS} ${WORKER_ABANDONED_DAYS} diff --git a/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java b/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java index 71da8ac9..53fb57e8 100755 --- a/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java +++ b/cws-installer/src/main/java/jpl/cws/task/CwsInstaller.java @@ -222,6 +222,7 @@ public class CwsInstaller { private static String elasticsearch_protocol_init; private static String elasticsearch_host; private static String elasticsearch_host_init; + private static String elasticsearch_index_prefix; private static String elasticsearch_port; private static String elasticsearch_use_auth; private static String elasticsearch_username; @@ -1368,6 +1369,21 @@ private static void setupElasticsearch() { log.debug("elasticsearch_port: " + elasticsearch_port); + // PROMPT USER FOR ELASTICSEARCH INDEX + elasticsearch_index_prefix = getPreset("elasticsearch_index_prefix"); + + if (elasticsearch_index_prefix == null) { + elasticsearch_index_prefix = getPreset("default_elasticsearch_index_prefix"); + } + + if (cws_installer_mode.equals("interactive")) { + elasticsearch_index_prefix = readLine("Enter the Elasticsearch Index. " + + "Default is " + elasticsearch_index_prefix + ": ", elasticsearch_index_prefix); + } + + log.debug("elasticsearch_index_prefix: " + elasticsearch_index_prefix); + + // PROMPT USER ELASTICSEARCH AUTH elasticsearch_use_auth = getPreset("elasticsearch_use_auth"); @@ -1783,7 +1799,8 @@ private static void showInstallationInfo() { print("...................................................................................."); print("Elasticsearch Protocol = " + elasticsearch_protocol); print("Elasticsearch Host = " + elasticsearch_host); - print("Elasticsearch Port = " + elasticsearch_port); + print("Elasticsearch Index Prefix = " + elasticsearch_index_prefix); + print("Elasticsearch Port = " + elasticsearch_port); if (elasticsearch_use_auth.equalsIgnoreCase("Y")) { print("Elasticsearch User = " + elasticsearch_username); print("Elasticsearch Password = ****** (hidden) "); @@ -2442,6 +2459,7 @@ private static int validateElasticsearch() { print(" [ELASTICSEARCH]: Configuration Details"); print(" elasticsearch_protocol=" + elasticsearch_protocol_init + " "); print(" elasticsearch_host=" + elasticsearch_host_init + " "); + print(" elasticsearch_index_prefix=" + elasticsearch_index_prefix + " "); print(" elasticsearch_port=" + elasticsearch_port + " "); print(" ........................................................................................."); print(""); @@ -2964,6 +2982,7 @@ private static void updateCwsUiProperties() throws IOException { content = content.replace("__CWS_CONSOLE_SSL_PORT__", cws_console_ssl_port); content = content.replace("__CWS_ES_PROTOCOL__", elasticsearch_protocol); content = content.replace("__CWS_ES_HOST__", elasticsearch_host); + content = content.replace("__CWS_ES_INDEX_PREFIX__", elasticsearch_index_prefix); content = content.replace("__CWS_ES_PORT__", elasticsearch_port); content = content.replace("__CWS_ES_USE_AUTH__", elasticsearch_use_auth); content = content.replace("__CWS_ENABLE_CLOUD_AUTOSCALING__", cws_enable_cloud_autoscaling); @@ -3086,8 +3105,9 @@ private static void updateCwsUiConfig() throws IOException { content = getFileContents(path); content = content.replace("__ES_PROTOCOL__", elasticsearch_protocol); content = content.replace("__ES_HOST__", elasticsearch_host); + content = content.replace("__ES_INDEX_PREFIX__", elasticsearch_index_prefix); content = content.replace("__ES_PORT__", elasticsearch_port); - content = content.replace("__ES_USE_AUTH__", elasticsearch_use_auth); + content = content.replace("__ES_USE_AUTH__", elasticsearch_use_auth); if (elasticsearch_use_auth.equalsIgnoreCase("Y")) { content = content.replace("__ES_USERNAME__", elasticsearch_username); content = content.replace("__ES_PASSWORD__", elasticsearch_password); @@ -3330,6 +3350,7 @@ private static void installLogstash() throws IOException { logstashContent = logstashContent.replace("__CWS_ES_PROTOCOL__", elasticsearch_protocol); logstashContent = logstashContent.replace("__CWS_ES_HOST__", elasticsearch_host); + logstashContent = logstashContent.replace("__CWS_ES_INDEX_PREFIX__", elasticsearch_index_prefix); logstashContent = logstashContent.replace("__CWS_ES_PORT__", elasticsearch_port); if (elasticsearch_use_auth.equalsIgnoreCase(("Y"))) { // Construct the auth config for logstash @@ -3415,6 +3436,7 @@ private static void writeOutConfigurationFile() throws IOException { setPreset("cws_token_expiration_hours", cws_token_expiration_hours); setPreset("elasticsearch_protocol", elasticsearch_protocol); setPreset("elasticsearch_host", elasticsearch_host); + setPreset("elasticsearch_index_prefix", elasticsearch_index_prefix); setPreset("elasticsearch_port", elasticsearch_port); setPreset("elasticsearch_use_auth", elasticsearch_use_auth); setPreset("elasticsearch_username", elasticsearch_username); diff --git a/cws-service/src/main/java/jpl/cws/controller/MvcCore.java b/cws-service/src/main/java/jpl/cws/controller/MvcCore.java index 40ce565c..31ea6a81 100755 --- a/cws-service/src/main/java/jpl/cws/controller/MvcCore.java +++ b/cws-service/src/main/java/jpl/cws/controller/MvcCore.java @@ -19,29 +19,33 @@ @Component public class MvcCore { private static final Logger log = LoggerFactory.getLogger(MvcCore.class); - + @Autowired protected CamundaExecutionService cwsExecutionService; @Autowired protected SchedulerQueueUtils cwsSchedulerUtils; @Autowired private CwsConsoleService cwsConsoleService; - - @Value("${cws.console.app.root}") private String appRoot; - @Value("${cws.version}") private String version; - @Value("${cws.db.type}") private String dbType; - @Value("${cws.db.host}") private String dbHost; - @Value("${cws.db.name}") private String dbName; - @Value("${cws.db.port}") private String dbPort; - @Value("${cws.auth.scheme}") private String authScheme; - @Value("${cws.install.dir}") private String installDir; - @Value("${cws.tomcat.lib}") private String tomcatLib; - @Value("${cws.tomcat.bin}") private String tomcatBin; - @Value("${cws.tomcat.home}") private String tomcatHome; - @Value("${cws.tomcat.webapps}") private String tomcatWebapps; - @Value("${cws.project.webapp.root}") private String projectWebappRoot; - @Value("${cws.history.days.to.live}") private String historyDaysToLive; - @Value("${cws.history.level}") private String historyLevel; + + @Value("${cws.console.app.root}") private String appRoot; + @Value("${cws.version}") private String version; + @Value("${cws.db.type}") private String dbType; + @Value("${cws.db.host}") private String dbHost; + @Value("${cws.db.name}") private String dbName; + @Value("${cws.db.port}") private String dbPort; + @Value("${cws.elasticsearch.protocol}") private String esProtocol; + @Value("${cws.elasticsearch.hostname}") private String esHost; + @Value("${cws.elasticsearch.index.prefix}") private String esIndexPrefix; + @Value("${cws.elasticsearch.port}") private String esPort; + @Value("${cws.auth.scheme}") private String authScheme; + @Value("${cws.install.dir}") private String installDir; + @Value("${cws.tomcat.lib}") private String tomcatLib; + @Value("${cws.tomcat.bin}") private String tomcatBin; + @Value("${cws.tomcat.home}") private String tomcatHome; + @Value("${cws.tomcat.webapps}") private String tomcatWebapps; + @Value("${cws.project.webapp.root}") private String projectWebappRoot; + @Value("${cws.history.days.to.live}") private String historyDaysToLive; + @Value("${cws.history.level}") private String historyLevel; public MvcCore() {} - + protected ModelAndView buildModel(String page, String message) { ModelAndView model = new ModelAndView(page); model.addObject("base", appRoot); @@ -49,17 +53,17 @@ protected ModelAndView buildModel(String page, String message) { if (projectWebappRoot != null && !projectWebappRoot.isEmpty()) { model.addObject("cwsProjectWebappRoot", projectWebappRoot); } - log.debug("MODEL: "+model.getModel()); + log.debug("MODEL: " + model.getModel()); return model; } - - + + protected ModelAndView buildHomeModel(String message) { ModelAndView model = new ModelAndView("home"); model.addObject("base", appRoot); model.addObject("msg", message); - - log.trace("MODEL for Home page: "+model.getModel()); + + log.trace("MODEL for Home page: " + model.getModel()); return model; } @@ -80,11 +84,11 @@ protected ModelAndView buildSummaryModel(String message) { } model.addObject("numActiveProcDefs", numActivePd); - log.trace("MODEL for Summary page: "+model.getModel()); + log.trace("MODEL for Summary page: " + model.getModel()); return model; } - - + + protected ModelAndView buildDeploymentsModel(String message) { log.trace("buildDeploymentsModel..."); ModelAndView model = new ModelAndView("deployments"); @@ -95,10 +99,9 @@ protected ModelAndView buildDeploymentsModel(String message) { // Add list of (the latest) process definitions to the model // model.addObject("procDefs", cwsExecutionService.listProcessDefinitions()); - - log.trace("MODEL for Deployments page: "+model.getModel()); - } - catch (Throwable t) { + + log.trace("MODEL for Deployments page: " + model.getModel()); + } catch (Throwable t) { log.error("Unexpected exception", t); } return model; @@ -116,31 +119,29 @@ protected ModelAndView buildLogsModel(String message) { model.addObject("procDefs", cwsExecutionService.listProcessDefinitions()); model.addObject("workerIds", cwsConsoleService.getAllWorkerIds()); - log.trace("MODEL for Logs page: "+model.getModel()); - } - catch (Throwable t) { + log.trace("MODEL for Logs page: " + model.getModel()); + } catch (Throwable t) { log.error("Unexpected exception", t); } return model; } - + protected ModelAndView buildHistoryModel(String message) { log.trace("buildHistoryModel..."); ModelAndView model = new ModelAndView("history"); try { model.addObject("base", appRoot); model.addObject("msg", message); - - log.trace("MODEL for History page: "+model.getModel()); - } - catch (Throwable t) { + + log.trace("MODEL for History page: " + model.getModel()); + } catch (Throwable t) { log.error("Unexpected exception", t); } return model; } - + /** - * + * */ protected ModelAndView buildProcessesModel(String message) { ModelAndView model = new ModelAndView("processes"); @@ -152,17 +153,16 @@ protected ModelAndView buildProcessesModel(String message) { // model.addObject("procDefs", cwsExecutionService.listProcessDefinitions()); - log.trace("MODEL for Processes page: "+model.getModel()); - } - catch (Throwable t) { + log.trace("MODEL for Processes page: " + model.getModel()); + } catch (Throwable t) { log.error("Unexpected exception", t); } return model; } - - + + /** - * + * */ protected ModelAndView buildConfigurationModel(String message) { ModelAndView model = new ModelAndView("configuration"); @@ -170,55 +170,57 @@ protected ModelAndView buildConfigurationModel(String message) { model.addObject("base", appRoot); model.addObject("msg", message); - model.addObject("version", version); - model.addObject("dbType", dbType); - model.addObject("dbHost", dbHost); - model.addObject("dbName", dbName); - model.addObject("dbPort", dbPort); - model.addObject("authScheme", authScheme); - model.addObject("installDir", installDir); - model.addObject("tomcatLib", tomcatLib); - model.addObject("tomcatBin", tomcatBin); - model.addObject("tomcatHome", tomcatHome); - model.addObject("tomcatWebapps", tomcatWebapps); + model.addObject("version", version); + model.addObject("dbType", dbType); + model.addObject("dbHost", dbHost); + model.addObject("dbName", dbName); + model.addObject("dbPort", dbPort); + model.addObject("esProtocol", esProtocol); + model.addObject("esHost", esHost); + model.addObject("esIndexPrefix", esIndexPrefix); + model.addObject("esPort", esPort); + model.addObject("authScheme", authScheme); + model.addObject("installDir", installDir); + model.addObject("tomcatLib", tomcatLib); + model.addObject("tomcatBin", tomcatBin); + model.addObject("tomcatHome", tomcatHome); + model.addObject("tomcatWebapps", tomcatWebapps); model.addObject("historyDaysToLive", historyDaysToLive); - model.addObject("historyLevel", historyLevel); + model.addObject("historyLevel", historyLevel); model.addObject("javaHome", System.getenv("JAVA_HOME")); model.addObject("javaVersion", Runtime.version().toString()); model.addObject("camundaVersion", System.getenv("CAMUNDA_VER")); DiskUsage diskUsage = cwsConsoleService.getDiskUsage(); - - model.addObject("databaseSize", diskUsage.databaseSize); - model.addObject("workersInfo", diskUsage.workers); - - log.trace("MODEL for Configuration page: "+model.getModel()); - } - catch (Throwable t) { + + model.addObject("databaseSize", diskUsage.databaseSize); + model.addObject("workersInfo", diskUsage.workers); + + log.trace("MODEL for Configuration page: " + model.getModel()); + } catch (Throwable t) { log.error("Unexpected exception", t); } return model; } - + /** - * + * */ protected ModelAndView buildModelerModel() { ModelAndView model = new ModelAndView("modeler"); try { model.addObject("base", appRoot); - - log.trace("MODEL for Modeler page: "+model.getModel()); - } - catch (Throwable t) { + + log.trace("MODEL for Modeler page: " + model.getModel()); + } catch (Throwable t) { log.error("Unexpected exception", t); } return model; } - - + + /** - * + * */ protected ModelAndView buildDocumentationModel(String message) { ModelAndView model = new ModelAndView("documentation"); @@ -226,17 +228,15 @@ protected ModelAndView buildDocumentationModel(String message) { model.addObject("base", appRoot); model.addObject("msg", message); model.addObject("cwsVersion", version); - } - catch (Throwable t) { + } catch (Throwable t) { log.error("Unexpected exception", t); } return model; } - - + + /** * Model for "Workers" web page - * */ protected ModelAndView buildWorkersModel() { ModelAndView model = new ModelAndView("workers"); @@ -247,19 +247,19 @@ protected ModelAndView buildWorkersModel() { // List procDefs = cwsExecutionService.listProcessDefinitions(); model.addObject("procDefs", procDefs); - + model.addObject("workers", cwsConsoleService.getWorkersUiDTO(procDefs)); model.addObject("externalWorkers", cwsConsoleService.getExternalWorkersUiDTO()); - + model.addObject("workersTitle", cwsConsoleService.getWorkersTitle()); - + try { Set clients = cwsSchedulerUtils.getAmqClients(); model.addObject("amqClients", clients); } catch (Exception e) { log.error("There was a problem getting listing of AMQ clients", e); } - + return model; } @@ -270,28 +270,24 @@ protected ModelAndView buildModelerModel(String message) { model.addObject("base", appRoot); model.addObject("msg", message); - log.trace("MODEL for Modeler page: "+model.getModel()); - } - catch (Throwable t) { + log.trace("MODEL for Modeler page: " + model.getModel()); + } catch (Throwable t) { log.error("Unexpected exception", t); } return model; } - protected ModelAndView buildApiDocsModel(String message) { + protected ModelAndView buildApiDocsModel(String message) { log.trace("Building apidocs's model..."); ModelAndView model = new ModelAndView("api-docs"); try { model.addObject("base", appRoot); model.addObject("msg", message); - log.trace("MODEL for Modeler page: "+model.getModel()); - } - catch (Throwable t) { + log.trace("MODEL for Modeler page: " + model.getModel()); + } catch (Throwable t) { log.error("Unexpected exception", t); } return model; } - -} - +} \ No newline at end of file diff --git a/cws-service/src/main/java/jpl/cws/controller/RestService.java b/cws-service/src/main/java/jpl/cws/controller/RestService.java index f6f14317..9f947429 100644 --- a/cws-service/src/main/java/jpl/cws/controller/RestService.java +++ b/cws-service/src/main/java/jpl/cws/controller/RestService.java @@ -103,6 +103,7 @@ public class RestService extends MvcCore { @Value("${cws.elasticsearch.protocol}") private String elasticsearchProtocolName; @Value("${cws.elasticsearch.hostname}") private String elasticsearchHostname; + @Value("${cws.elasticsearch.index.prefix}") private String elasticsearchIndexPrefix; @Value("${cws.elasticsearch.port}") private String elasticsearchPort; @Value("${cws.elasticsearch.use.auth}") private String elasticsearchUseAuth; @@ -769,8 +770,7 @@ public ModelAndView validateAndSaveSnippets( @ApiOperation(value = "Gets the total number of log rows.", tags = {"Logs"}, produces = "application/json") @RequestMapping(value="/logs/get/count", method = GET, produces="application/json") public @ResponseBody String getNumLogs() { - String urlString = constructElasticsearchUrl("/_count"); - + String urlString = constructElasticsearchUrl("/" + elasticsearchIndexPrefix + "-logstash-*/_count"); log.trace("REST getNumLogs query = " + urlString); try { @@ -804,8 +804,7 @@ public ModelAndView validateAndSaveSnippets( @RequestMapping(value = "/logs/get/noScroll", method = GET, produces="application/json") public @ResponseBody String getLogsNoScroll( @RequestParam(value = "source") String source) { - String urlString = constructElasticsearchUrl("/_search"); - + String urlString = constructElasticsearchUrl("/" + elasticsearchIndexPrefix + "-logstash-*/_search"); log.debug("REST logs/get/noScroll query = " + urlString); try { @@ -843,8 +842,7 @@ public ModelAndView validateAndSaveSnippets( @RequestMapping(value = "/logs/get", method = GET, produces="application/json") public @ResponseBody String getLogs( @RequestParam(value = "source") String source) { - String urlString = constructElasticsearchUrl("/_search?scroll=5m&source=" + source + "&source_content_type=application/json"); - + String urlString = constructElasticsearchUrl("/" + elasticsearchIndexPrefix + "-logstash-*/_search?scroll=5m&source=" + source + "&source_content_type=application/json"); log.trace("REST getLogs query = " + urlString); try { @@ -881,7 +879,7 @@ public ModelAndView validateAndSaveSnippets( HttpServletResponse response, @PathVariable String procDefKey ) { - String urlString = constructElasticsearchUrl("/*/_delete_by_query"); + String urlString = constructElasticsearchUrl("/" + elasticsearchIndexPrefix + "-logstash*/_delete_by_query"); log.debug("REST deleteLogsByProcDefKey url = " + urlString); String data = "{ \"query\": { \"bool\": { \"must\": [ { \"match\": { \"procDefKey\": \"" + procDefKey + "\" } } ] } } }"; diff --git a/cws-test/src/test/java/jpl/cws/test/integration/ui/WebTestIT.java b/cws-test/src/test/java/jpl/cws/test/integration/ui/WebTestIT.java index c104ecd2..76e07e62 100644 --- a/cws-test/src/test/java/jpl/cws/test/integration/ui/WebTestIT.java +++ b/cws-test/src/test/java/jpl/cws/test/integration/ui/WebTestIT.java @@ -221,7 +221,7 @@ public void runHelloWorldTest() { sleep(1000); findOnPage("ls"); - findOnPage("Hello World"); + findOnPage("Hello World."); findOnPage("Command 'ls' exit code: 0"); sleep(9000); diff --git a/dev.sh b/dev.sh index 4e0116b2..30be500e 100755 --- a/dev.sh +++ b/dev.sh @@ -14,20 +14,21 @@ DB_USER=${7} DB_PASS=${8} ES_PROTOCOL=${9} ES_HOST=${10} -ES_PORT=${11} -ES_USE_AUTH=${12} -ES_USERNAME=${13} -ES_PASSWORD=${14} -ENABLE_CLOUD_AS=${15} -SECURITY_SCHEME=${16} -THIS_HOSTNAME=${17} -NOTIFICATION_EMAILS=${18} -ADMIN_FIRSTNAME=${19} -ADMIN_LASTNAME=${20} -ADMIN_EMAIL=${21} -NUM_WORKERS=${22} -WORKER_MAX_NUM_RUNNING_PROCS=${23} -WORKER_ABANDONED_DAYS=${24} +ES_INDEX_PREFIX=${11} +ES_PORT=${12} +ES_USE_AUTH=${13} +ES_USERNAME=${14} +ES_PASSWORD=${15} +ENABLE_CLOUD_AS=${16} +SECURITY_SCHEME=${17} +THIS_HOSTNAME=${18} +NOTIFICATION_EMAILS=${19} +ADMIN_FIRSTNAME=${20} +ADMIN_LASTNAME=${21} +ADMIN_EMAIL=${22} +NUM_WORKERS=${23} +WORKER_MAX_NUM_RUNNING_PROCS=${24} +WORKER_ABANDONED_DAYS=${25} source ${ROOT}/utils.sh diff --git a/install/cws-ui/configuration.ftl b/install/cws-ui/configuration.ftl index a457f993..d4d073a3 100755 --- a/install/cws-ui/configuration.ftl +++ b/install/cws-ui/configuration.ftl @@ -13,59 +13,58 @@ var params = {}; var rows; var es = []; - + function formatBytes(bytes,decimals) { if(bytes == 0) return '0 Bytes'; - + var k = 1024, dm = decimals <= 0 ? 0 : decimals || 2, sizes = ['Bytes', 'KB', 'MB', 'GB', 'TB', 'PB', 'EB', 'ZB', 'YB'], i = Math.floor(Math.log(bytes) / Math.log(k)); - + return parseFloat((bytes / Math.pow(k, i)).toFixed(dm)) + ' ' + sizes[i]; } function getEsIndices() { - - - $.ajax({ + + $.ajax({ url: "/${base}/rest/stats/es/indices", success: function( data ) { - + for (var i = 0; i < data.length; i++) { - - var index = data[i]; - var health = index.health; - - if (health === 'yellow') { - health += ' (OK)'; - } - - var size = formatBytes(parseInt(index["store.size"], 10)); - - $("#es_indices").append('' + index.index + '' + health + '' + size + ''); + var index = data[i]; + if (index.index.startsWith("${esIndexPrefix}" + "-logstash")) { + + var health = index.health; + + if (health === 'yellow') { + health += ' (OK)'; + } + var size = formatBytes(parseInt(index["store.size"], 10)); + $("#es_indices").append('' + index.index + '' + health + '' + size + ''); + } } - + // $("#es_num_indices").html(data.length); }, error: function(err){ console.log("getes indices failed!", err); } }); - + } - + function refreshElasticsearchStats() { - - $.ajax({ + + $.ajax({ url: "/${base}/rest/stats/es", success: function( data ) { - + for (var property in data.nodes) { if (data.nodes.hasOwnProperty(property)) { - + var node = data.nodes[property]; - + es.push( { host: node.host, size: node.indices.store.size_in_bytes, disk_free: node.fs.total.free_in_bytes, @@ -73,31 +72,31 @@ memory_used_precent: node.os.mem.used_percent } ); } } - + $("#es_host").html(es[0].host); $("#es_disk_used").html(formatBytes(es[0].size)); $("#es_disk_free").html(formatBytes(es[0].disk_free)); $("#es_cpu").html(es[0].cpu_usage_percent + "%"); $("#es_memory").html(es[0].memory_used_precent + "%"); - + }, error: function(){ } }); } - + function updateValues() { - + var dbSize = formatBytes(parseInt('${databaseSize}'.replace(/,/g, ''), 10)); - + $("#db_size").html(dbSize); - - + + // Update table values var table = $("#diskUsage tbody"); - + table.find('tr').each(function (i, el) { - + var $tds = $(this).find('td'), node = $tds.eq(1), sizeWithCommas = node.text(); @@ -111,9 +110,9 @@ } }); } - + $( document ).ready(function() { - // + // if ($("#statusMessageDiv:contains('ERROR:')").length >= 1) { $("#statusMessageDiv").css( "color", "red" ); } @@ -123,15 +122,15 @@ $('#statusMessageDiv').fadeOut(5000, "linear"); } } - + updateValues(); getEsIndices(); refreshElasticsearchStats(); }); - + - + @@ -147,28 +146,28 @@ margin-bottom: 1em; float:left; } - + .database { display: flex; align-items: center; margin: 30px 0px 50px; } - + .database > div { font-size: 23px; } - + #db_size { border-radius: 8px; margin-left: 25px; background: #ededed; padding: 10px; } - + .margin { margin-bottom: 20px; } - + @@ -180,13 +179,13 @@
<#include "sidebar.ftl">
- +

${msg}

 - +

Configuration

- +
@@ -213,6 +212,14 @@ + + + + + + + + @@ -252,7 +259,7 @@ - + @@ -263,14 +270,14 @@

System Health

- +
Database Disk Usage
${databaseSize}
- +

System

- +
CWS Database Port ${dbPort}
CWS Elasticsearch URL${esProtocol}://${esHost}:${esPort}
CWS Elasticsearch Index Prefix${esIndexPrefix}
CWS Authentication Scheme ${authScheme}
Java Version ${javaVersion}
Java Home Path ${javaHome}
@@ -307,10 +314,10 @@

Elasticsearch Server

- +
Name
- + diff --git a/install/cws-ui/cws-ui.properties b/install/cws-ui/cws-ui.properties index 57600b5d..d5bed0df 100755 --- a/install/cws-ui/cws-ui.properties +++ b/install/cws-ui/cws-ui.properties @@ -18,6 +18,7 @@ cws.db.password=__CWS_DB_PASSWORD__ cws.elasticsearch.protocol=__CWS_ES_PROTOCOL__ cws.elasticsearch.hostname=__CWS_ES_HOST__ +cws.elasticsearch.index.prefix=__CWS_ES_INDEX_PREFIX__ cws.elasticsearch.port=__CWS_ES_PORT__ cws.elasticsearch.use.auth=__CWS_ES_USE_AUTH__ cws.elasticsearch.username=__CWS_ES_USERNAME__ diff --git a/install/example-cws-configuration.properties b/install/example-cws-configuration.properties index b936eab5..23244a5e 100644 --- a/install/example-cws-configuration.properties +++ b/install/example-cws-configuration.properties @@ -124,6 +124,7 @@ cws_notification_emails=[AdminEmailAddresses(comma separated)] # elasticsearch_protocol=HTTPS elasticsearch_protocol=[YourElasticsearchProtocol] elasticsearch_host=[YourElasticsearchHost] +elasticsearch_index_prefix=[YourElasticsearchIndexPrefix] elasticsearch_port=9200 elasticsearch_use_auth=n elasticsearch_username=[YourElasticsearchUsername] diff --git a/install/installerPresets.properties b/install/installerPresets.properties index c3a12470..978ce267 100644 --- a/install/installerPresets.properties +++ b/install/installerPresets.properties @@ -15,6 +15,7 @@ default_cws_token_expiration_hours=24 default_smtp_hostname=smtp.localhost default_smtp_port=25 default_elasticsearch_use_auth=n +default_elasticsearch_index_prefix=cws-index default_elasticsearch_port=9200 default_user_provided_logstash=n default_history_level=full diff --git a/install/logging/cws-logstash.conf b/install/logging/cws-logstash.conf index f2efd5fb..7c701236 100644 --- a/install/logging/cws-logstash.conf +++ b/install/logging/cws-logstash.conf @@ -22,7 +22,7 @@ filter { if "_grokparsefailure" in [tags] { drop { } } - + grok { match => [ "message", '%{TIMESTAMP_ISO8601:logdate} %{DATA:logLevel} +\[%{DATA:threadName}\] .* - \[%{DATA:cwsHost}\]\[%{DATA:cwsWorkerId}\]\[%{DATA:procDefKey}\]\[%{DATA:procInstId}\]\[%{DATA:actInstId}\] %{GREEDYDATA:msgBody}', @@ -32,20 +32,20 @@ filter { "message", '%{GREEDYDATA:msgBody}' ] } - + date { match => [ "logdate", "yyyy-MM-dd HH:mm:ss,SSS" ] timezone => "UTC" locale => en remove_field => [ "logdate" ] } - + if [logLevel] == "WARNING" { mutate { update => { "logLevel" => "WARN" } } } - + if [logLevel] == "SEVERE" { mutate { update => { "logLevel" => "ERROR" } @@ -57,7 +57,7 @@ output { elasticsearch { hosts => ["__CWS_ES_PROTOCOL__://__CWS_ES_HOST__:__CWS_ES_PORT__"] ilm_enabled => false - index => "logstash-%{+YYYY.MM.dd}" + index => "__CWS_ES_INDEX_PREFIX__-logstash-%{+YYYY.MM.dd}" ssl => __LOGSTASH_ES_USE_SSL__ __LOGSTASH_ES_USERNAME__ __LOGSTASH_ES_PASSWORD__ diff --git a/utils.sh b/utils.sh index 864cbcca..0ba03b6e 100755 --- a/utils.sh +++ b/utils.sh @@ -191,22 +191,23 @@ function auto_conf_data () { DB_PASS=${9} ES_PROTOCOL=${10} ES_HOST=${11} - ES_PORT=${12} - ES_USE_AUTH=${13} - ES_USERNAME=${14} - ES_PASSWORD=${15} - ENABLE_CLOUD_AS=${16} - SECURITY_SCHEME=${17} - THIS_HOSTNAME=${18} - NOTIFICATION_EMAILS=${19} - ADMIN_FIRSTNAME=${20} - ADMIN_LASTNAME=${21} - ADMIN_EMAIL=${22} - NUM_WORKERS=${23} - WORKER_MAX_NUM_RUNNING_PROCS=${24} - WORKER_ABANDONED_DAYS=${25} - - OUTPUT_FILE=${26} + ES_INDEX_PREFIX=${12} + ES_PORT=${13} + ES_USE_AUTH=${14} + ES_USERNAME=${15} + ES_PASSWORD=${16} + ENABLE_CLOUD_AS=${17} + SECURITY_SCHEME=${18} + THIS_HOSTNAME=${19} + NOTIFICATION_EMAILS=${20} + ADMIN_FIRSTNAME=${21} + ADMIN_LASTNAME=${22} + ADMIN_EMAIL=${23} + NUM_WORKERS=${24} + WORKER_MAX_NUM_RUNNING_PROCS=${25} + WORKER_ABANDONED_DAYS=${26} + + OUTPUT_FILE=${27} source ${ROOT}/utils.sh @@ -281,6 +282,7 @@ function auto_conf_data () { database_password=${DB_PASS} elasticsearch_protocol=${ES_PROTOCOL} elasticsearch_host=${ES_HOST} + elasticsearch_index_prefix=${ES_INDEX_PREFIX} elasticsearch_port=${ES_PORT} elasticsearch_use_auth=${ES_USE_AUTH} elasticsearch_username=${ES_USERNAME}
Hostname 127.0.0.1