WOTaskd - improve logging and add comments to Properties file as docs

Applications/wotaskd/Resources/Properties

@@ -1,8 +1,14 @@
 # Properties file for wotaskd
 # Overrides the JavaWebObjects.framework Properties
 
+# Changes to this file will be overwritten when you update wotaskd. 
+# Consider storing customizations in: /etc/WebObjects/wotaskd/Properties   (or in the launchd file)
+
 #WOHost=localhost
 
+# Uncomment this to get some basic logging (like lifebeats from instances), probably to the file: /Library/WebObjects/Logs/wotaskd.log
+#_DeploymentDebugging=true
+
 WOAutoOpenInBrowser=false
 WODebuggingEnabled=false
 WODirectConnectEnabled=true
@@ -15,18 +21,61 @@ WOMonitorEnabled=false
 WOLifebeatDestinationPort=1085
 WOLifebeatEnabled=false
 
-# wotaskd specific defaults
+# Sets the IP address that the wotaskd process listens to for multicast requests from the HTTP adaptor.
 WOMulticastAddress=239.128.14.2
+
+# Determines whether the wotaskd process responds to multicast queries from the HTTP adaptor.
 WORespondsToMulticastQuery=false 
+
+# Determines whether the wotaskd process generates an HTTP adaptor configuration file.
 WOSavesAdaptorConfiguration=false
+
+# Used to determine the number of seconds that the wotaskd process waits without receiving any status messages 
+# from an instance before considering it dead. It species a multiplier against WOLifebeatInterval. 
+# If WOLifebeatInterval is 30, a wotaskd process waits 120 seconds from the last status message before 
+# determining that an instance is dead.
 WOAssumeApplicationIsDeadMultiplier=4
+
+# Controls whether wotaskd should start instances using a system-specific launcher;
+# this allows for starting applications in the background, as well as circumventing 
+# Windows-specific issues with launch output.
 WOShouldUseSpawn=true
 
+
+
+# If an app instance has been told to 1) "stop" or 2) "refuse new sessions" and it does not stop, 
+# then it can be forcefully terminated after some time period if WOTaskd.forceQuitTaskEnabled is true
+#WOTaskd.forceQuitTaskEnabled=false
+
+
+# If an app instance has been told to "refuse new sessions" and it does not stop, 
+# then it can be forcefully terminated after this number of retries.
+#WOTaskd.refuseNumRetries=3
+
+
+# If WOTaskd.forceQuitTaskEnabled is true, setup a task to check the instance. 
+# If it still doesn't die, then force a QUIT command when the timer elapses.
+# If the app was told to "stop" and does not stop, then it will be forcefully stopped after the killTimeout elapses.
+# If the app was told to "refuse new sessions" and does not stop, then it will be forcefully stopped after
+#    (killTimeout + (refuseNumRetries-1 * killTimeout)) elapses.
+# The value is specified in milliseconds. The minimum is 60000 (60 seconds).
+#WOTaskd.killTimeout=120000
+
+
+# The number of milliseconds before a timeout occurs for requests sent from WOTaskD to WOApplication instances. 
+#WOTaskd.receiveTimeout=5000
+
+
+
+
+
+
+
 log4j.rootCategory=INFO, A1
 
 # A1 is set to be a FileAppender which outputs to System.out.
 log4j.appender.A1=org.apache.log4j.ConsoleAppender
 
 # A1 uses PatternLayout.
 log4j.appender.A1.layout=er.extensions.logging.ERXPatternLayout
-log4j.appender.A1.layout.ConversionPattern=%d %-5p %V{u/f} [%t] %C{2} (%F:%L) - %m%n
+log4j.appender.A1.layout.ConversionPattern=%d %-5p %V{u/f} [%t] %C{2} (%F:%L) - %m%n

Applications/wotaskd/Resources/SpawnOfWotaskd.sh

@@ -1,3 +1,19 @@
 #!/bin/sh
 
-$@ 1>/dev/null 2>&1 & 
+# To enable logging of instance startup run the command 'touch /tmp/logWebObjects'
+
+# Log messages will be written to: 
+LOG=/Library/WebObjects/Logs/SpawnOfWotaskd.log
+
+if [ -f /tmp/logWebObjects ]; then 
+
+	echo "************" >>${LOG}
+	echo "date: `date`" >>${LOG}
+	echo "args: $@" >>${LOG}
+	$@ 1>>${LOG} 2>&1 &
+
+else
+
+	$@ 1>/dev/null 2>&1 &
+
+fi

Applications/wotaskd/Sources/com/webobjects/monitor/wotaskd/Application.java

@@ -127,7 +127,9 @@ public Application() {
             NSLog.out.setIsVerbose(true);
             NSLog.err.setIsVerbose(true);
             NSLog.allowDebugLoggingForGroups(NSLog.DebugGroupDeployment);
-            NSLog.debug.setAllowedDebugLevel(NSLog.DebugLevelDetailed);
+            if (!NSLog.debugLoggingAllowedForLevel(NSLog.DebugLevelInformational)) {
+            	NSLog.debug.setAllowedDebugLevel(NSLog.DebugLevelInformational);
+            }
         }
 
         com.webobjects.appserver._private.WOHttpIO._alwaysAppendContentLength = false;

Applications/wotaskd/Sources/com/webobjects/monitor/wotaskd/DirectAction.java

@@ -915,7 +915,7 @@ public WOResponse woconfigAction() {
         aResponse.appendContentString(xml);
         aResponse.setHeader("text/xml", "content-type");
         aResponse.setHeader(aFormat.format(new NSTimestamp()), "Last-Modified");
-        if (NSLog.debugLoggingAllowedForLevelAndGroups(NSLog.DebugLevelInformational, NSLog.DebugGroupDeployment))
+        if (NSLog.debugLoggingAllowedForLevelAndGroups(NSLog.DebugLevelDetailed, NSLog.DebugGroupDeployment))
             NSLog.debug.appendln("woConfigAction returned: " + xml);
 
         return aResponse;

Applications/wotaskd/Sources/com/webobjects/monitor/wotaskd/LifebeatRequestHandler.java

@@ -196,7 +196,9 @@ private void registerStop(String instanceName, String host, String port) {
     }
 
     private void registerCrash(String instanceName, String host, String port) {
-        // app will stop in a bad way - notify if necessary
+    	NSLog.err.appendln("App '" + instanceName + "' on " + host + ":" + port + " received 'willCrash' notification.");
+
+    	// app will stop in a bad way - notify if necessary
         InetAddress hostAddress = addressForName(host);
 
         theApplication._lock.startReading();

Frameworks/Misc/JavaMonitorFramework/Sources/com/webobjects/monitor/_private/MInstance.java

@@ -761,6 +761,26 @@ public void registerCrash(NSTimestamp registrationDate) {
     }
 
     public void sendDeathNotificationEmail() {
+    	NSTimestamp currentTime = new NSTimestamp();
+        String currentDate = currentTime.toString();
+
+        long cutOffTime = _lastRegistration.getTime() + lifebeatCheckInterval();
+        String assumedToBeDead = "";
+        if (currentTime.getTime() > cutOffTime) {
+        	long secondsDifference = (currentTime.getTime() - cutOffTime) / 1000;
+        	assumedToBeDead = "The app did not respond for " + secondsDifference + "seconds " +
+        			"which is greater than the allowed threshold of " + lifebeatCheckInterval() + " seconds " +
+        			"(Lifebeat Interval * WOAssumeApplicationIsDeadMultiplier) so it is assumed to be dead.\n";
+        }
+    	String message = "Application '" + displayName() + "' on " + _host.name() + ":" + port() +
+	        " stopped running at " + (currentDate) + ".\n" + 
+	        "The app's current state was: " + stateArray[state] + ".\n" +
+	        assumedToBeDead + 
+	        "The last successful communication occurred at: " + _lastRegistration.toString() + ". " + 
+	        "This may be the result of a crash or an intentional shutdown from outside of wotaskd";
+
+    	NSLog.err.appendln(message);
+
         boolean shouldEmail = false;
         Boolean aBool = _application.notificationEmailEnabled();
         if (aBool != null) {
@@ -770,13 +790,10 @@ public void sendDeathNotificationEmail() {
         if (shouldEmail) {
             try {
                 WOMailDelivery mailer = WOMailDelivery.sharedInstance();
-                String currentDate = (new NSTimestamp()).toString();
                 String fromAddress = siteConfig().emailReturnAddr();
                 NSArray toAddress = null;
                 String subject = new String("App stopped running: " + displayName());
-                String bodyText = new String("The application " + displayName() + " listening on port " + port()
-                        + " on host " + _host.name() + " stopped running at " + (currentDate)
-                        + ".  This may be the result of a crash or an intentional shutdown from outside of wotaskd");
+                String bodyText = message;
                 if (fromAddress != null) {
                     fromAddress = new String("root@" + _host.name());
                 }