chaos runner shutdown improvements

Author: scottf

chaos-runner/build.gradle

@@ -13,7 +13,7 @@ plugins {
     id 'signing'
 }
 
-def jarVersion = "0.0.6"
+def jarVersion = "0.0.7"
 group = 'io.synadia'
 
 def isMerge = System.getenv("BUILD_EVENT") == "push"

chaos-runner/src/examples/java/io/synadia/examples/ChaosRunnerShutdown.java

@@ -0,0 +1,94 @@
+// Copyright (c) 2025 Synadia Communications Inc. All Rights Reserved.
+// See LICENSE and NOTICE file for details.
+
+package io.synadia.examples;
+
+import io.synadia.chaos.ChaosArguments;
+import io.synadia.chaos.ChaosRunner;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.URL;
+
+import static io.synadia.chaos.ChaosUtils.out;
+
+public class ChaosRunnerShutdown {
+
+    public static void main(String[] args) throws Exception {
+        ChaosArguments arguments = new ChaosArguments()
+            .servers(3)
+            .workDirectory("C:\\temp\\chaos-runner")
+            .serverNamePrefix("cr-shutdown-server")
+            .clusterName("cr-shutdown-cluster")
+            .delay(30_000)
+            .initialDelay(30_000)
+            .downTime(30_000);
+
+        ChaosRunner runner = ChaosRunner.start(arguments);
+
+        // just give the servers a little time to be ready be first connect
+        Thread.sleep(1000);
+
+        String[] urls = runner.getConnectionUrls();
+        out("Connection Urls");
+        for (String url : urls) {
+            out(" ", url);
+        }
+
+        int[] ports = runner.getConnectionPorts();
+        int[] monitorPorts = runner.getMonitorPorts();
+
+        Thread.sleep(1000);
+        out("H RUNNING", ChaosRunner.isRunning());
+        for (int i = 0; i < monitorPorts.length; i++) {
+            int port = ports[i];
+            int mport = monitorPorts[i];
+            String hz = readHealthz(monitorPorts[i]);
+            out("H", port + "/" + mport, hz);
+        }
+
+        ChaosRunner.shutdown();
+
+        Thread.sleep(1000);
+        out("Z RUNNING", ChaosRunner.isRunning());
+        for (int i = 0; i < monitorPorts.length; i++) {
+            int port = ports[i];
+            int mport = monitorPorts[i];
+            String hz = readHealthz(monitorPorts[i]);
+            out("Z", port + "/" + mport, hz);
+        }
+    }
+
+    private static String readHealthz(int port) {
+        return readEndpoint(port, "healthz");
+    }
+
+    private static String readEndpoint(int port, String endpoint) {
+        String sUrl = "http://localhost:" + port + "/" + endpoint;
+        try {
+            URL url = new URL(sUrl);
+            InputStream inputStream = url.openStream();
+            BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream));
+
+            boolean first = true;
+            String line;
+            StringBuilder content = new StringBuilder();
+            while ((line = reader.readLine()) != null) {
+                if (first) {
+                    first = false;
+                }
+                else {
+                    content.append(System.lineSeparator());
+                }
+                content.append(line);
+            }
+            reader.close();
+            return content.toString().trim();
+        }
+        catch (IOException e) {
+            return e.getMessage();
+        }
+    }
+}

chaos-runner/src/main/java/io/synadia/chaos/ChaosRunner.java

@@ -16,6 +16,7 @@
 import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.ThreadLocalRandom;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantLock;
 import java.util.logging.Level;
 
 import static io.nats.NatsRunnerUtils.*;
@@ -25,6 +26,8 @@ public class ChaosRunner {
 
     private static final String CR_LABEL = "ChaosRunner";
 
+    private static final ReentrantLock INSTANCE_LOCK = new ReentrantLock();
+    private static Thread APP_SHUTDOWN_HOOK_THREAD;
     private static ChaosRunner INSTANCE;
 
     public final ChaosPrinter printer;
@@ -282,35 +285,89 @@ public static ChaosRunner start(ChaosArguments a, ChaosPrinter printer) {
         NatsServerRunner.setDefaultOutputLevel(Level.SEVERE);
         final ChaosPrinter finalPrinter = printer == null ? getDefaultPrinter() : printer;
 
+        INSTANCE_LOCK.lock();
         try {
             INSTANCE = new ChaosRunner(a, finalPrinter);
         }
         catch (IOException e) {
             finalPrinter.err(CR_LABEL, "Failed to start ChaosRunner", e);
             System.exit(-1);
         }
+        finally {
+            INSTANCE_LOCK.unlock();
+        }
 
-        Runtime.getRuntime().addShutdownHook(
-            new Thread("app-shutdown-hook") {
-                @Override
-                public void run() {
-                    INSTANCE.shutdown();
-                    finalPrinter.out(CR_LABEL, "EXIT");
-                }
-            });
+        APP_SHUTDOWN_HOOK_THREAD = new Thread("app-shutdown-hook") {
+            @Override
+            public void run() {
+                shutdownServers();
+                shutdownExecutor();
+                finalPrinter.out(CR_LABEL, "EXIT");
+            }
+        };
+
+        Runtime.getRuntime().addShutdownHook(APP_SHUTDOWN_HOOK_THREAD);
 
         return INSTANCE;
     }
 
-    private void shutdown() {
+    public static boolean isRunning() {
+        INSTANCE_LOCK.lock();
+        try {
+            return INSTANCE != null;
+        }
+        finally {
+            INSTANCE_LOCK.unlock();
+        }
+    }
+
+    public static void shutdown() {
+        INSTANCE_LOCK.lock();
+        try {
+            removeShutdownHook();
+            shutdownExecutor();
+            shutdownServers();
+        }
+        finally {
+            INSTANCE_LOCK.unlock();
+        }
+    }
+
+    public static void shutdownExecutor() {
+        INSTANCE_LOCK.lock();
         try {
-            for (NatsServerRunner runner : natsServerRunners ) {
-                try {
-                    runner.close();
+            INSTANCE.executor.shutdown();
+        }
+        finally {
+            INSTANCE_LOCK.unlock();
+        }
+    }
+
+    private static void removeShutdownHook() {
+        INSTANCE_LOCK.lock();
+        try {
+            if (APP_SHUTDOWN_HOOK_THREAD != null) {
+                Runtime.getRuntime().removeShutdownHook(APP_SHUTDOWN_HOOK_THREAD);
+                APP_SHUTDOWN_HOOK_THREAD = null;
+            }
+        }
+        finally {
+            INSTANCE_LOCK.unlock();
+        }
+    }
+
+    private static void shutdownServers() {
+        INSTANCE_LOCK.lock();
+        try {
+            if (INSTANCE != null) {
+                for (NatsServerRunner runner : INSTANCE.natsServerRunners) {
+                    try { runner.close(); } catch (Exception ignore) {}
                 }
-                catch (Exception ignore) {}
+                INSTANCE = null;
             }
         }
-        catch (Exception ignore) {}
+        finally {
+            INSTANCE_LOCK.unlock();
+        }
     }
 }