Skip to content

Commit

Permalink
PAYARA-161 Healthcheck stuck threads check (#1796)
Browse files Browse the repository at this point in the history
* PAYARA-161 Initial stuck thread healthcheck

Includes two different attempts at check for stuck threads. One of these is using the Thread MBean to check for deadlocked threads. The other is getting the stack trace of each thread and if it is the same when the healthcheck runs again then it says it is a stuck thread. However listeners and the like are at the same point each time which is expected behaviour so it prints out a load of unnecessary stack traces.

* PAYARA-161 Use long requests to check for stuck threads

* PAYARA-161 Moved healthchecks to their own modules.

Still doesn't work, caused by OSGi wierdness

* PAYARA-161 moved packages

* PAYARA-161 server now boots

Thread ids of stuck threads are listed, now to provide more information

* PAYARA-161 healthcheck stuck threads now works

* Added description for PAYARA-1411

* Admin Console integration for Stuck Thread checker

PAYARA-1877
  • Loading branch information
Cousjava authored and smillidge committed Jul 29, 2017
1 parent c29bfa5 commit 377785b
Show file tree
Hide file tree
Showing 36 changed files with 1,496 additions and 221 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ healthcheck.configuration.machineMemoryUsageCheckerTabTitle=Machine Memory Usage
healthcheck.configuration.machineMemoryUsageCheckerTabTitleToolTip=Configures the Machine Memory Usage HealthCheck.
healthcheck.configuration.hoggingThreadsCheckerTabTitle=Hogging Threads
healthcheck.configuration.hoggingThreadsCheckerTabTitleToolTip=Configures the Hogging Threads HealthCheck.
healthcheck.configuration.stuckThreadsCheckerTabTitle=Stuck Threads
healthcheck.configuration.stuckThreadsCheckerTabTitleToolTip=Configures the Stuck Threads HealthCheck.
healthcheck.configuration.garbageCollectorCheckerTabTitle=Garbage Collector
healthcheck.configuration.garbageCollectorCheckerTabTitleToolTip=Configures the Garbage Collector HealthCheck.

Expand Down Expand Up @@ -117,5 +119,12 @@ healthcheck.checker.configuration.hoggingThreads.retryCountLabelHelpText=Number
healthcheck.checker.configuration.garbageCollector.pageTitle=Garbage Collector
healthcheck.checker.configuration.garbageCollector.pageTitleHelpText=Configuration options for the Garbage Collector HealthCheck.

healthcheck.checker.configuration.stuckThreads.pageTitle=Stuck Threads
healthcheck.checker.configuration.stuckThreads.pageTitleHelpText=Configuration options for the Stuck Threads HealthCheck.
healthcheck.checker.configuration.stuckThreads.timeThresholdLabel=Threshold time
healthcheck.checker.configuration.stuckThreads.timeThresholdLabelHelpText=Defines amount of time for the threshold.
healthcheck.checker.configuration.stuckThreads.thresholdUnitLabel=Threshold unit
healthcheck.checker.configuration.stuckThreads.thresholdUnitLabelHelpText=Defines unit of time for the threshold.

healthcheck.configuration.availableNotifiers=Avaliable Notifiers
healthcheck.configuration.selectedNotifiers=Selected Notifiers
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
<!--

Copyright (c) 2017 Payara Foundation and/or its affiliates. All rights reserved.

The contents of this file are subject to the terms of either the GNU
General Public License Version 2 only ("GPL") or the Common Development
and Distribution License("CDDL") (collectively, the "License"). You
may not use this file except in compliance with the License. You can
obtain a copy of the License at
https://github.com/payara/Payara/blob/master/LICENSE.txt
See the License for the specific
language governing permissions and limitations under the License.

When distributing the software, include this License Header Notice in each
file and include the License file at glassfish/legal/LICENSE.txt.

GPL Classpath Exception:
The Payara Foundation designates this particular file as subject to the "Classpath"
exception as provided by the Payara Foundation in the GPL Version 2 section of the License
file that accompanied this code.

Modifications:
If applicable, add the following below the License Header, with the fields
enclosed by brackets [] replaced by your own identifying information:
"Portions Copyright [year] [name of copyright owner]"

Contributor(s):
If you wish your version of this file to be governed by only the CDDL or
only the GPL Version 2, indicate your decision by adding "[Contributor]
elects to include this software in this distribution under the [CDDL or GPL
Version 2] license." If you don't indicate a single choice of license, a
recipient has the option to distribute your version of this file under
either the CDDL, the GPL Version 2 or to extend the choice of license to
its licensees as provided above. However, if you add GPL Version 2 code
and therefore, elected the GPL Version 2 license, then the option applies
only if the new code is made subject to such option by the copyright
holder.
-->

<!initPage
setResourceBundle(key="i18nhx" bundle="fish.payara.admingui.healthcheck.Strings");
/>
<!composition template="/templates/default.layout"
guiTitle="$resource{i18nhx.healthcheck.checker.configuration.garbageCollector.pageTitle}">

<!define name="content">

<event>
<!beforeCreate
getRequestValue(key="configName" value="#{pageSession.configName}");
setPageSessionAttribute(key="HEALTHCHECK_CONFIG_URL",
value="#{sessionScope.REST_URL}/configs/config/#{pageSession.configName}/health-check-service-configuration");

gf.restRequest(endpoint="#{pageSession.HEALTHCHECK_CONFIG_URL}/get-healthcheck-configuration?target=#{pageSession.configName}"
method="GET" result="#{requestScope.resp}");
setPageSessionAttribute(key="valueMap",
value="#{requestScope.resp.data.subReports.get(3).extraProperties.stuckThreads}");
mapPut(map="#{pageSession.valueMap}" key="target" value="#{pageSession.configName}");

setPageSessionAttribute(key="convertToFalseList", value={"enabled", "dynamic"});
if (#{pageSession.valueMap['enabled']}=true) {
setPageSessionAttribute(key="enabledSelected", value="true");
}

setPageSessionAttribute(key="dynamic", value="true");
/>
</event>

<sun:form id="propertyForm">
#include "/common/shared/alertMsg_1.inc"
#include "/healthcheck/healthcheck/healthcheckConfigTabs.inc
<sun:title id="propertyContentPage" title="$resource{i18nhx.healthcheck.checker.configuration.stuckThreads.pageTitle}"
helpText="$resource{i18nhx.healthcheck.checker.configuration.stuckThreads.pageTitleHelpText}" >
<!facet pageButtonsTop>
<sun:panelGroup id="topButtons">
<sun:button id="saveButton" text="$resource{i18n.button.Save}"
onClick="if (guiValidate('#{reqMsg}','#{reqInt}','#{reqPort}')) submitAndDisable(this, '$resource{i18n.button.Processing}'); return false;" >
<!command
mapPut(map="#{pageSession.valueMap}" key="enabled" value="#{pageSession.enabledSelected}");
mapPut(map="#{pageSession.valueMap}" key="dynamic" value="#{pageSession.dynamic}");
prepareSuccessfulMsg();
gf.updateEntity(endpoint="#{pageSession.HEALTHCHECK_CONFIG_URL}/healthcheck-stuckthreads-configure"
attrs="#{pageSession.valueMap}" convertToFalse="#{pageSession.convertToFalseList}");
/>
</sun:button>
</sun:panelGroup>
</facet>
</sun:title>
<sun:propertySheet id="propertySheet">
#include "/common/shared/configNameSection.inc"
<sun:propertySheetSection id="garbageCollectorCheckerProps">
<sun:property id="enabledProp" labelAlign="left" noWrap="#{true}" overlapLabel="#{false}"
label="$resource{i18nhx.healthcheck.configuration.enabledLabel}"
helpText="$resource{i18nhx.healthcheck.configuration.enabledLabelHelpText}">
<sun:checkbox id="enabledBox" selected="#{pageSession.enabledSelected}" selectedValue="true" />
</sun:property>
<sun:property id="dynamicProp" labelAlign="left" noWrap="#{true}" overlapLabel="#{false}"
label="$resource{i18nhx.healthcheck.configuration.dynamicLabel}"
helpText="$resource{i18nhx.healthcheck.configuration.dynamicLabelHelpText}">
<sun:checkbox id="dynamicBox" selected="#{pageSession.dynamic}" selectedValue="true" />
</sun:property>
<sun:property id="nameProp" labelAlign="left" noWrap="#{true}" overlapLabel="#{false}"
label="$resource{i18nhx.healthcheck.checker.configuration.nameLabel}"
helpText="$resource{i18nhx.healthcheck.checker.configuration.nameLabelHelpText}">
<sun:textField id="nameField" columns="$int{40}" maxLength="250" text="#{pageSession.valueMap['checkerName']}"/>
</sun:property>
<sun:property id="timeProp" labelAlign="left" noWrap="#{true}" overlapLabel="#{false}"
label="$resource{i18nhx.healthcheck.checker.configuration.timeLabel}"
helpText="$resource{i18nhx.healthcheck.checker.configuration.timeLabelHelpText}">
<sun:textField id="timeField" columns="$int{40}" maxLength="250" styleClass="integer"
text="#{pageSession.valueMap['time']}"/>
</sun:property>
<sun:property id="unitProp" labelAlign="left" noWrap="#{true}" overlapLabel="#{false}"
label="$resource{i18nhx.healthcheck.checker.configuration.unitLabel}"
helpText="$resource{i18nhx.healthcheck.checker.configuration.unitLabelHelpText}">
<sun:dropDown id="unitDropdown" selected="#{pageSession.valueMap['unit']}"
labels={"NANOSECONDS", "MICROSECONDS", "MILLISECONDS", "SECONDS", "MINUTES", "HOURS", "DAYS"} />
</sun:property>
<sun:property id="thresholdTimeProp" labelAlign="left" noWrap="#{true}" overlapLabel="#{false}"
label="$resource{i18nhx.healthcheck.checker.configuration.stuckThreads.timeThresholdLabel}"
helpText="$resource{i18nhx.healthcheck.checker.configuration.stuckThreads.timeThresholdLabelHelpText}">
<sun:textField id="timeField" columns="$int{40}" maxLength="250" styleClass="integer"
text="#{pageSession.valueMap['threshold']}"/>
</sun:property>
<sun:property id="thresholdUnitProp" labelAlign="left" noWrap="#{true}" overlapLabel="#{false}"
label="$resource{i18nhx.healthcheck.checker.configuration.stuckThreads.thresholdUnitLabel}"
helpText="$resource{i18nhx.healthcheck.checker.configuration.stuckThreads.thresholdUnitLabelHelpText}">
<sun:dropDown id="unitDropdown" selected="#{pageSession.valueMap['thresholdUnit']}"
labels={"NANOSECONDS", "MICROSECONDS", "MILLISECONDS", "SECONDS", "MINUTES", "HOURS", "DAYS"} />
</sun:property>
</sun:propertySheetSection>
</sun:propertySheet>
</sun:form>
</define>
</composition>
Original file line number Diff line number Diff line change
@@ -1,20 +1,41 @@
!--

DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER.
<!--
Copyright (c) 2017 Payara Foundation and/or its affiliates. All rights reserved.
The contents of this file are subject to the terms of the Common Development
The contents of this file are subject to the terms of either the GNU
General Public License Version 2 only ("GPL") or the Common Development
and Distribution License("CDDL") (collectively, the "License"). You
may not use this file except in compliance with the License. You can
obtain a copy of the License at
https://glassfish.dev.java.net/public/CDDL+GPL_1_1.html
or packager/legal/LICENSE.txt. See the License for the specific
https://github.com/payara/Payara/blob/master/LICENSE.txt
See the License for the specific
language governing permissions and limitations under the License.
When distributing the software, include this License Header Notice in each
file and include the License file at packager/legal/LICENSE.txt.
-->
file and include the License file at glassfish/legal/LICENSE.txt.
GPL Classpath Exception:
The Payara Foundation designates this particular file as subject to the "Classpath"
exception as provided by the Payara Foundation in the GPL Version 2 section of the License
file that accompanied this code.
Modifications:
If applicable, add the following below the License Header, with the fields
enclosed by brackets [] replaced by your own identifying information:
"Portions Copyright [year] [name of copyright owner]"
Contributor(s):
If you wish your version of this file to be governed by only the CDDL or
only the GPL Version 2, indicate your decision by adding "[Contributor]
elects to include this software in this distribution under the [CDDL or GPL
Version 2] license." If you don't indicate a single choice of license, a
recipient has the option to distribute your version of this file under
either the CDDL, the GPL Version 2 or to extend the choice of license to
its licensees as provided above. However, if you add GPL Version 2 code
and therefore, elected the GPL Version 2 license, then the option applies
only if the new code is made subject to such option by the copyright
holder.
-->
<sun:tabSet id="healthcheckConfigTabs" immediate="true" selected="#{sessionScope.healthcheckConfigTab}"
text="$resource{i18nhx.healthcheck.configuration.tabSetTitle}"
toolTip="$resource{i18nhx.healthcheck.configuration.tabSetTitleToolTip}">
Expand Down Expand Up @@ -75,6 +96,15 @@
gf.redirect(page="#{request.contextPath}/healthcheck/healthcheck/checkers/hoggingThreadsCheckerConfiguration.jsf?configName=#{configName}");
/>
</sun:tab>

<sun:tab id="healthcheckStuckThreadsCheckerTab" immediate="true"
text="$resource{i18nhx.healthcheck.configuration.stuckThreadsCheckerTabTitle}"
toolTip="$resource{i18nhx.healthcheck.configuration.stuckThreadsCheckerTabTitleToolTip}">
<!command
setSessionAttribute(key="healthcheckConfigTab" value="healthcheckStuckThreadsCheckerTab");
gf.redirect(page="#{request.contextPath}/healthcheck/healthcheck/checkers/stuckThreadsConfiguration.jsf?configName=#{configName}");
/>
</sun:tab>

<sun:tab id="healthcheckGarbageCollectorCheckerTab" immediate="true"
text="$resource{i18nhx.healthcheck.configuration.garbageCollectorCheckerTabTitle}"
Expand Down
7 changes: 6 additions & 1 deletion appserver/concurrent/concurrent-impl/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@
-->

<!-- Portions Copyright [2016] [Payara Foundation and/or its affiliates] -->
<!-- Portions Copyright [2016-2017] [Payara Foundation and/or its affiliates] -->

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
Expand Down Expand Up @@ -129,6 +129,11 @@
<artifactId>requesttracing-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>fish.payara.payara-modules</groupId>
<artifactId>healthcheck-stuck</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
</project>

Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
* only if the new code is made subject to such option by the copyright
* holder.
*/
// Portions Copyright [2016] [Payara Foundation]
// Portions Copyright [2016-2017] [Payara Foundation]

package org.glassfish.concurrent.runtime;

Expand All @@ -63,6 +63,7 @@

import fish.payara.nucleus.requesttracing.RequestTracingService;
import fish.payara.nucleus.requesttracing.domain.RequestEvent;
import fish.payara.nucleus.healthcheck.stuck.StuckThreadsStore;
import org.glassfish.internal.api.Globals;

public class ContextSetupProviderImpl implements ContextSetupProvider {
Expand All @@ -82,6 +83,7 @@ static enum CONTEXT_TYPE {CLASSLOADING, SECURITY, NAMING, WORKAREA}
private boolean classloading, security, naming, workArea;

private RequestTracingService requestTracing;
private StuckThreadsStore stuckThreads;

public ContextSetupProviderImpl(InvocationManager invocationManager,
Deployment deployment,
Expand All @@ -99,6 +101,12 @@ public ContextSetupProviderImpl(InvocationManager invocationManager,
logger.log(Level.INFO, "Error retrieving Request Tracing service "
+ "during initialisation of Concurrent Context - NullPointerException");
}
try {
this.stuckThreads = Globals.getDefaultHabitat().getService(StuckThreadsStore.class);
} catch (NullPointerException ex) {
logger.log(Level.INFO, "Error retrieving Stuck Threads Sore Healthcheck service "
+ "during initialisation of Concurrent Context - NullPointerException");
}

for (CONTEXT_TYPE contextType: contextTypes) {
switch(contextType) {
Expand Down Expand Up @@ -191,7 +199,9 @@ public ContextHandle setup(ContextHandle contextHandle) throws IllegalStateExcep

private RequestEvent constructConcurrentContextEvent(ComponentInvocation invocation) {
requestTracing.startTrace();

if (stuckThreads != null){
stuckThreads.registerThread(Thread.currentThread().getId());
}
RequestEvent requestEvent = new RequestEvent("ConcurrentContextTrace");

requestEvent.addProperty("App Name", invocation.getAppName());
Expand Down Expand Up @@ -241,6 +251,9 @@ public void reset(ContextHandle contextHandle) {
if (requestTracing != null && requestTracing.isRequestTracingEnabled()) {
requestTracing.endTrace();
}
if (stuckThreads != null){
stuckThreads.deregisterThread(Thread.currentThread().getId());
}
}

private boolean isApplicationEnabled(String appId) {
Expand Down
Loading

0 comments on commit 377785b

Please sign in to comment.