From f50e3ee544e0850046a2f435a9e0ae5ee2a1da4b Mon Sep 17 00:00:00 2001 From: Khushboo Rajput Date: Wed, 6 Sep 2023 16:02:07 -0700 Subject: [PATCH] Update supervisord configuration for process failure expected codes and PROCESS_STATE_FATAL handling Signed-off-by: Khushboo Rajput --- config/supervisord.conf | 11 +++++++++-- .../performanceanalyzer/LocalhostConnectionUtil.java | 6 +++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/config/supervisord.conf b/config/supervisord.conf index 47162f7b4..e4c061cd8 100644 --- a/config/supervisord.conf +++ b/config/supervisord.conf @@ -7,7 +7,7 @@ chmod=0770 [supervisord] logfile=/usr/share/supervisor/performance_analyzer/supervisord.log ; (main log file;default $CWD/supervisord.log) pidfile=/usr/share/supervisor/performance_analyzer/supervisord.pid ; (supervisord pidfile;default supervisord.pid) -childlogdir=/usr/share/supervisor/performance_analyzer ; ('AUTO' child log dir, default $TEMP) +childlogdir=/usr/share/supervisor/performance_analyzer ; ('AUTO' child log dir, default $TEMP) ; the below section must remain in the config file for RPC ; (supervisorctl/web interface) to work, additional interfaces may be @@ -23,10 +23,17 @@ serverurl=/usr/share/supervisord.sock ; newlines). It can also contain wildcards. The filenames are ; interpreted as relative to this file. Included files *cannot* ; include files themselves. - [include] files = /etc/supervisor/conf.d/*.conf [program:performance_analyzer] command=/usr/share/opensearch/performance-analyzer-rca/bin/performance-analyzer-agent /usr/share/opensearch user=1000 +autostart=true ; start at supervisord start (default: true) +autorestart=unexpected ; autorestart if exited after running (def: unexpected) +exitcodes=1 ; 'expected' exit codes used with autorestart, System.exit(1) called from PerformanceAnalyzerApp.cleanupAndExit() - if Reader Thread crashes. + +[eventlistener:stop_supervisord] +command=bash -c "printf 'READY\n' && while read line; do kill -SIGQUIT $PPID; done < /dev/stdin" +events=PROCESS_STATE_FATAL +buffer_size=100 diff --git a/src/main/java/org/opensearch/performanceanalyzer/LocalhostConnectionUtil.java b/src/main/java/org/opensearch/performanceanalyzer/LocalhostConnectionUtil.java index b79514d43..c40fcbea8 100644 --- a/src/main/java/org/opensearch/performanceanalyzer/LocalhostConnectionUtil.java +++ b/src/main/java/org/opensearch/performanceanalyzer/LocalhostConnectionUtil.java @@ -23,7 +23,7 @@ public class LocalhostConnectionUtil { public static void disablePA() throws InterruptedException { String PA_CONFIG_PATH = Util.PA_BASE_URL + "/cluster/config"; String PA_DISABLE_PAYLOAD = "{\"enabled\": false}"; - int retryCount = 3; + int retryCount = 5; while (retryCount > 0) { HttpURLConnection connection = null; @@ -49,9 +49,9 @@ public static void disablePA() throws InterruptedException { } } --retryCount; - Thread.sleep((int) (5000 * (Math.random() * 2) + 100)); + Thread.sleep((int) (60000 * (Math.random() * 2) + 100)); } - throw new RuntimeException("Failed to disable PA after 3 attempts"); + throw new RuntimeException("Failed to disable PA after 5 attempts"); } private static HttpURLConnection createHTTPConnection(String path, HttpMethod httpMethod) {