7903754: jcstress: Implement fail-on-error run option

jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java

@@ -28,6 +28,7 @@
 import org.openjdk.jcstress.infra.collectors.*;
 import org.openjdk.jcstress.infra.grading.ConsoleReportPrinter;
 import org.openjdk.jcstress.infra.grading.ExceptionReportPrinter;
+import org.openjdk.jcstress.infra.grading.FailFastKiller;
 import org.openjdk.jcstress.infra.grading.TextReportPrinter;
 import org.openjdk.jcstress.infra.grading.HTMLReportPrinter;
 import org.openjdk.jcstress.infra.runners.TestConfig;
@@ -66,11 +67,21 @@ public void run() throws Exception {
 
         ConsoleReportPrinter printer = new ConsoleReportPrinter(opts, new PrintWriter(out, true), config.configs.size(), timeBudget);
         DiskWriteCollector diskCollector = new DiskWriteCollector(opts.getResultFile());
-        TestResultCollector mux = MuxCollector.of(printer, diskCollector);
+        FailFastKiller failFastKiller = null;
+        TestResultCollector mux;
+        if (opts.isFailOnError()) {
+            failFastKiller = new FailFastKiller(opts, new PrintWriter(out, true), config.configs);
+            mux = MuxCollector.of(printer, diskCollector, failFastKiller);
+        } else {
+            mux = MuxCollector.of(printer, diskCollector);
+        }
         SerializedBufferCollector sink = new SerializedBufferCollector(mux);
 
         TestExecutor executor = new TestExecutor(opts.verbosity(), sink, config.scheduler, timeBudget);
         printer.setExecutor(executor);
+        if (failFastKiller != null) {
+            failFastKiller.setExecutor(executor);
+        }
 
         executor.runAll(config.configs);
 

jcstress-core/src/main/java/org/openjdk/jcstress/Options.java

@@ -46,6 +46,9 @@
  * @author Aleksey Shipilev (aleksey.shipilev@oracle.com)
  */
 public class Options {
+
+    public static final String FAIL_ON_ERROR = "foe";
+
     private String resultDir;
     private String testFilter;
     private int strideSize;
@@ -66,6 +69,7 @@ public class Options {
     private AffinityMode affinityMode;
     private boolean pretouchHeap;
     private TimeValue timeBudget;
+    private boolean failOnError;
 
     public Options(String[] args) {
         this.args = args;
@@ -150,6 +154,10 @@ public boolean parse() throws IOException {
                 "Common time suffixes (s/m/h/d) are accepted.")
                 .withRequiredArg().ofType(TimeValue.class).describedAs("time");
 
+        parser.accepts(FAIL_ON_ERROR, "Tells the framework to exit after "
+                + "failure or error and no longer waste HW cycles. Soft errors do not count by default.");
+
+
         parser.accepts("v", "Be verbose.");
         parser.accepts("vv", "Be extra verbose.");
         parser.accepts("vvv", "Be extra extra verbose.");
@@ -260,6 +268,8 @@ public boolean parse() throws IOException {
         this.splitCompilation = orDefault(set.valueOf(optSplitCompilation), true);
         this.affinityMode = orDefault(set.valueOf(optAffinityMode), AffinityMode.LOCAL);
 
+        this.failOnError = set.has(FAIL_ON_ERROR);
+
         return true;
     }
 
@@ -392,6 +402,10 @@ public boolean isPretouchHeap() {
         return pretouchHeap;
     }
 
+    public boolean isFailOnError() {
+        return failOnError;
+    }
+
     public TimeValue timeBudget() { return timeBudget; }
 
 }

jcstress-core/src/main/java/org/openjdk/jcstress/TestExecutor.java

@@ -66,6 +66,7 @@ public class TestExecutor {
     private final ExecutorService supportTasks;
 
     private final TimeBudget timeBudget;
+    private boolean diedFast = false;
 
     public TestExecutor(Verbosity verbosity, TestResultCollector sink, Scheduler scheduler, TimeBudget tb) throws IOException {
         this.verbosity = verbosity;
@@ -142,7 +143,6 @@ public void runAll(List<TestConfig> configs) {
         }
 
         while (!byScl.isEmpty()) {
-
             // Roll over the scheduling classes and try to greedily cram most
             // of the tasks for it. This exits when no scheduling classes can fit
             // the current state of the machine.
@@ -167,8 +167,16 @@ public void runAll(List<TestConfig> configs) {
             while (!processReadyVMs()) {
                 awaitNotification();
             }
+            if (diedFast) {
+                cleanup();
+                return;
+            }
         }
 
+        cleanup();
+    }
+
+    private void cleanup() {
         // Wait until all threads are done, which means everything got processed
         while (!vmByToken.isEmpty()) {
             while (!processReadyVMs()) {
@@ -215,6 +223,14 @@ public int getJVMsFinishing() {
         return jvmsFinishing.get();
     }
 
+    public void setDiedFast() {
+        diedFast=true;
+    }
+
+    public boolean isDiedFast() {
+        return diedFast;
+    }
+
     private class VM {
         private final String host;
         private final int port;

jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ConsoleReportPrinter.java

@@ -29,7 +29,6 @@
 import org.openjdk.jcstress.TimeBudget;
 import org.openjdk.jcstress.Verbosity;
 import org.openjdk.jcstress.infra.collectors.TestResult;
-import org.openjdk.jcstress.infra.collectors.TestResultCollector;
 import org.openjdk.jcstress.vm.VMSupport;
 
 import java.io.PrintWriter;
@@ -41,7 +40,7 @@
  *
  * @author Aleksey Shipilev (aleksey.shipilev@oracle.com)
  */
-public class ConsoleReportPrinter implements TestResultCollector {
+public class ConsoleReportPrinter extends CountingResultCollector {
 
     private static final Integer PRINT_INTERVAL_MS = Integer.getInteger("jcstress.console.printIntervalMs");
 
@@ -62,10 +61,6 @@ public class ConsoleReportPrinter implements TestResultCollector {
     private boolean progressFirstLine;
     private final int[] progressLen;
 
-    private long passed;
-    private long failed;
-    private long softErrors;
-    private long hardErrors;
     private TestExecutor executor;
     private final int totalCpuCount;
 
@@ -103,35 +98,11 @@ public synchronized void add(TestResult r) {
     }
 
     private void printResult(TestResult r) {
-        TestGrading grading = r.grading();
-
-        boolean inHardError = false;
-        switch (r.status()) {
-            case TIMEOUT_ERROR:
-            case CHECK_TEST_ERROR:
-            case TEST_ERROR:
-            case VM_ERROR:
-                hardErrors++;
-                inHardError = true;
-                break;
-            case API_MISMATCH:
-                softErrors++;
-                break;
-            case NORMAL:
-                if (grading.isPassed) {
-                    passed++;
-                } else {
-                    failed++;
-                }
-                break;
-            default:
-                throw new IllegalStateException("Illegal status: " + r.status());
-        }
-
+        boolean inHardError = countResult(r);
         boolean shouldPrintResults =
                 inHardError ||
-                !grading.isPassed ||
-                grading.hasInteresting ||
+                !r.grading().isPassed ||
+                r.grading().hasInteresting ||
                 verbosity.printAllTests();
 
         boolean shouldPrintStatusLine =
@@ -216,6 +187,11 @@ private void clearStatusLine() {
     public void printFinishLine() {
         clearStatusLine();
         printStatusLine();
+        if (executor.isDiedFast()) {
+            output.println("****************************************************");
+            output.println("To much failures occurred. Failing fast as requested");
+            output.println("****************************************************");
+        }
     }
 
     private String computeSpeed() {

jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/CountingResultCollector.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jcstress.infra.grading;
+
+import org.openjdk.jcstress.infra.collectors.TestResult;
+import org.openjdk.jcstress.infra.collectors.TestResultCollector;
+
+
+
+public abstract class  CountingResultCollector implements TestResultCollector {
+
+    protected long total;
+    protected long passed;
+    protected long failed;
+    protected long softErrors;
+    protected long hardErrors;
+
+    protected boolean countResult(TestResult r) {
+        TestGrading grading = r.grading();
+        total ++;
+        boolean inHardError = false;
+        switch (r.status()) {
+            case TIMEOUT_ERROR:
+            case CHECK_TEST_ERROR:
+            case TEST_ERROR:
+            case VM_ERROR:
+                hardErrors++;
+                inHardError = true;
+                break;
+            case API_MISMATCH:
+                softErrors++;
+                break;
+            case NORMAL:
+                if (grading.isPassed) {
+                    passed++;
+                } else {
+                    failed++;
+                }
+                break;
+            default:
+                throw new IllegalStateException("Illegal status: " + r.status());
+        }
+        return inHardError;
+    }
+}

jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/FailFastKiller.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2005, 2014, 2025, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package org.openjdk.jcstress.infra.grading;
+
+import org.openjdk.jcstress.Options;
+import org.openjdk.jcstress.TestExecutor;
+import org.openjdk.jcstress.infra.collectors.TestResult;
+import org.openjdk.jcstress.infra.runners.TestConfig;
+
+import java.io.PrintWriter;
+import java.util.Collections;
+import java.util.List;
+
+
+public class FailFastKiller extends CountingResultCollector {
+
+    private static final String INCLUDE_SOFT_ERRORS = "jcstress.foe.countsoft";
+    private static final String LIMIT = "jcstress.foe.limit";
+
+    private TestExecutor executor;
+    private final double absoluteThreshold;
+    private final double relativeThreshold;
+
+    private final boolean includeSoftErrors;
+
+
+    public FailFastKiller(Options opts, PrintWriter output, List<TestConfig> finalVariants) {
+        output.println("  Fail-on-error enabled as:");
+        String originalValue = System.getProperty(LIMIT, "1");
+        includeSoftErrors = Boolean.parseBoolean(System.getProperty(INCLUDE_SOFT_ERRORS, "false"));
+        double userValue = Double.parseDouble(originalValue.replaceAll("%*", ""));
+        boolean relative = originalValue.endsWith("%");
+        List<TestConfig> variants = Collections.unmodifiableList(finalVariants);
+        output.println("    all tests, " + originalValue);
+        output.println("    include soft errors: " + includeSoftErrors);
+        if (relative) {
+            relativeThreshold = userValue;
+            absoluteThreshold = (relativeThreshold * (double) variants.size()) / 100d;
+        } else {
+            absoluteThreshold = (long) userValue;
+            relativeThreshold = (absoluteThreshold * 100d) / (double) variants.size();
+        }
+        output.println("    The suite will terminate once failure rate reaches " + getRelativeThresholdNice() + "% (" + getAbsoluteThresholdNice() + ") of total tests (" + variants.size() + ")");
+        output.println();
+    }
+
+    @Override
+    public synchronized void add(TestResult r) {
+        addResult(r);
+    }
+
+    private void addResult(TestResult r) {
+        countResult(r);
+        verifyState(r);
+    }
+
+    private void verifyState(TestResult r) {
+        long totalFailed = failed + hardErrors;
+        if (includeSoftErrors) {
+            totalFailed += softErrors;
+        }
+       if (totalFailed >= absoluteThreshold) {
+            executor.setDiedFast();
+        }
+    }
+
+    public void setExecutor(TestExecutor executor) {
+        this.executor = executor;
+    }
+
+    private String getAbsoluteThresholdNice() {
+        return String.format("%.0f", absoluteThreshold);
+    }
+
+    private String getRelativeThresholdNice() {
+        return String.format("%.2f", relativeThreshold);
+    }
+
+}