Update supervisord configuration for process failure expected codes a…

…nd PROCESS_STATE_FATAL handling Signed-off-by: Khushboo Rajput <[email protected]>
opensearch-project · Sep 6, 2023 · f50e3ee · f50e3ee
1 parent 6405acf
commit f50e3ee
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 5 deletions.
diff --git a/config/supervisord.conf b/config/supervisord.conf
@@ -7,7 +7,7 @@ chmod=0770
 [supervisord]
 logfile=/usr/share/supervisor/performance_analyzer/supervisord.log ; (main log file;default $CWD/supervisord.log)
 pidfile=/usr/share/supervisor/performance_analyzer/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
-childlogdir=/usr/share/supervisor/performance_analyzer            ; ('AUTO' child log dir, default $TEMP)
+childlogdir=/usr/share/supervisor/performance_analyzer             ; ('AUTO' child log dir, default $TEMP)
 
 ; the below section must remain in the config file for RPC
 ; (supervisorctl/web interface) to work, additional interfaces may be
@@ -23,10 +23,17 @@ serverurl=/usr/share/supervisord.sock
 ; newlines).  It can also contain wildcards.  The filenames are
 ; interpreted as relative to this file.  Included files *cannot*
 ; include files themselves.
-
 [include]
 files = /etc/supervisor/conf.d/*.conf
 
 [program:performance_analyzer]
 command=/usr/share/opensearch/performance-analyzer-rca/bin/performance-analyzer-agent /usr/share/opensearch
 user=1000
+autostart=true                                                      ; start at supervisord start (default: true)
+autorestart=unexpected                                              ; autorestart if exited after running (def: unexpected)
+exitcodes=1                                                         ; 'expected' exit codes used with autorestart, System.exit(1) called from PerformanceAnalyzerApp.cleanupAndExit() - if Reader Thread crashes.
+
+[eventlistener:stop_supervisord]
+command=bash -c "printf 'READY\n' && while read line; do kill -SIGQUIT $PPID; done < /dev/stdin"
+events=PROCESS_STATE_FATAL
+buffer_size=100
diff --git a/src/main/java/org/opensearch/performanceanalyzer/LocalhostConnectionUtil.java b/src/main/java/org/opensearch/performanceanalyzer/LocalhostConnectionUtil.java
@@ -23,7 +23,7 @@ public class LocalhostConnectionUtil {
     public static void disablePA() throws InterruptedException {
         String PA_CONFIG_PATH = Util.PA_BASE_URL + "/cluster/config";
         String PA_DISABLE_PAYLOAD = "{\"enabled\": false}";
-        int retryCount = 3;
+        int retryCount = 5;
 
         while (retryCount > 0) {
             HttpURLConnection connection = null;
@@ -49,9 +49,9 @@ public static void disablePA() throws InterruptedException {
                 }
             }
             --retryCount;
-            Thread.sleep((int) (5000 * (Math.random() * 2) + 100));
+            Thread.sleep((int) (60000 * (Math.random() * 2) + 100));
         }
-        throw new RuntimeException("Failed to disable PA after 3 attempts");
+        throw new RuntimeException("Failed to disable PA after 5 attempts");
     }
 
     private static HttpURLConnection createHTTPConnection(String path, HttpMethod httpMethod) {