diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java b/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java index 1a488269..8fb3da7d 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/JCStress.java @@ -30,6 +30,7 @@ import org.openjdk.jcstress.infra.grading.ExceptionReportPrinter; import org.openjdk.jcstress.infra.grading.TextReportPrinter; import org.openjdk.jcstress.infra.grading.HTMLReportPrinter; +import org.openjdk.jcstress.infra.grading.XMLReportPrinter; import org.openjdk.jcstress.infra.runners.TestConfig; import org.openjdk.jcstress.infra.runners.TestList; import org.openjdk.jcstress.os.*; @@ -157,11 +158,18 @@ public void parseResults() throws Exception { drc.dump(); drc.close(); - new TextReportPrinter(opts, collector).work(); - new HTMLReportPrinter(opts, collector, out).work(); + new TextReportPrinter(opts.verbosity(), collector).work(); + new HTMLReportPrinter(opts.getResultDest(), collector, out).work(); + if (XMLReportPrinter.getSparse(out)!=null) { + new XMLReportPrinter(opts.getResultDest(), collector, out, XMLReportPrinter.getSparse(out)).work(); + } else { + new XMLReportPrinter(opts.getResultDest(), collector, out, false).work(); + new XMLReportPrinter(opts.getResultDest(), collector, out, true).work(); + } new ExceptionReportPrinter(collector).work(); } + private SortedSet computeActorCounts(Set tests) { SortedSet counts = new TreeSet<>(); for (String test : tests) { @@ -252,7 +260,7 @@ public int listTests(Options opts) { Set testsToPrint = new TreeSet<>(); for (TestConfig test : configsWithScheduler.configs) { if (opts.verbosity().printAllTests()) { - testsToPrint.add(test.toDetailedTest()); + testsToPrint.add(test.toDetailedTest(true)); } else { testsToPrint.add(test.name); } diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java index 2e06e954..5000efe2 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/HTMLReportPrinter.java @@ -25,7 +25,6 @@ package org.openjdk.jcstress.infra.grading; -import org.openjdk.jcstress.Options; import org.openjdk.jcstress.annotations.Expect; import org.openjdk.jcstress.infra.Status; import org.openjdk.jcstress.infra.TestInfo; @@ -57,19 +56,23 @@ public class HTMLReportPrinter { private final InProcessCollector collector; private int cellStyle = 1; - public HTMLReportPrinter(Options opts, InProcessCollector collector, PrintStream out) { + public HTMLReportPrinter(String resultDir, InProcessCollector collector, PrintStream out) { this.collector = collector; - this.resultDir = opts.getResultDest(); + this.resultDir = resultDir; File dir = new File(resultDir); dir.mkdirs(); - out.println(" HTML report generated at " + dir.getAbsolutePath() + File.separator + "index.html"); + out.println(" HTML report generated at " + dir.getAbsolutePath() + File.separator + getMainFileName()); + } + + private String getMainFileName() { + return "index.html"; } public void work() throws FileNotFoundException { List byName = ReportUtils.mergedByName(collector.getTestResults()); Collections.sort(byName, Comparator.comparing(TestResult::getName)); - PrintWriter output = new PrintWriter(resultDir + "/index.html"); + PrintWriter output = new PrintWriter(resultDir + File.separator + getMainFileName()); printHeader(output); @@ -171,7 +174,7 @@ public void work() throws FileNotFoundException { emitTestReports(ReportUtils.byName(collector.getTestResults())); } - private SortedMap getEnv(List ts) { + static SortedMap getEnv(List ts) { SortedMap env = new TreeMap<>(); for (TestResult result : ts) { if (result != null) { @@ -353,10 +356,7 @@ public void emitTestReport(PrintWriter o, Collection results, TestIn } List sorted = new ArrayList<>(results); - sorted.sort(Comparator - .comparing((TestResult t) -> t.getConfig().getCompileMode()) - .thenComparing((TestResult t) -> t.getConfig().getSchedulingClass().toString()) - .thenComparing((TestResult t) -> StringUtils.join(t.getConfig().jvmArgs, ","))); + resultsOrder(sorted); o.println("

Environment

"); o.println(""); @@ -492,6 +492,13 @@ public void emitTestReport(PrintWriter o, Collection results, TestIn printFooter(o); } + static void resultsOrder(List sorted) { + sorted.sort(Comparator + .comparing((TestResult t) -> t.getConfig().getCompileMode()) + .thenComparing((TestResult t) -> t.getConfig().getSchedulingClass().toString()) + .thenComparing((TestResult t) -> StringUtils.join(t.getConfig().jvmArgs, ","))); + } + private void resultHeader(PrintWriter o, TestResult r) { TestConfig cfg = r.getConfig(); o.println("

"); diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java index b4bc4121..40845779 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/ReportUtils.java @@ -84,6 +84,13 @@ public static Multimap byName(Collection src) { } return result; } + public static Multimap byDetailedName(Collection src) { + Multimap result = new HashMultimap<>(); + for (TestResult r : mergedByConfig(src)) { + result.put(r.getConfig().toDetailedTest(false), r); + } + return result; + } private static TestResult merged(TestConfig config, Collection mergeable) { Counter counter = new Counter<>(); diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TextReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TextReportPrinter.java index ebb76220..815721ef 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TextReportPrinter.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/TextReportPrinter.java @@ -48,10 +48,10 @@ public class TextReportPrinter { private final PrintWriter pw; private final Set emittedTests; - public TextReportPrinter(Options opts, InProcessCollector collector) { + public TextReportPrinter(Verbosity verbosity, InProcessCollector collector) { this.collector = collector; this.pw = new PrintWriter(System.out, true); - this.verbosity = opts.verbosity(); + this.verbosity = verbosity; this.emittedTests = new HashSet<>(); } diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java new file mode 100644 index 00000000..bf2ca6cb --- /dev/null +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/grading/XMLReportPrinter.java @@ -0,0 +1,681 @@ +/* + * Copyright (c) 2005, 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.openjdk.jcstress.infra.grading; + + +import org.openjdk.jcstress.annotations.Expect; +import org.openjdk.jcstress.infra.TestInfo; +import org.openjdk.jcstress.infra.collectors.InProcessCollector; +import org.openjdk.jcstress.infra.collectors.TestResult; +import org.openjdk.jcstress.infra.runners.TestConfig; +import org.openjdk.jcstress.infra.runners.TestList; +import org.openjdk.jcstress.os.SchedulingClass; +import org.openjdk.jcstress.util.Multimap; +import org.openjdk.jcstress.vm.CompileMode; +import org.w3c.dom.Document; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; +import javax.xml.validation.Validator; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.net.InetAddress; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.TreeSet; + + +public class XMLReportPrinter { + + private enum JunitResult { + pass, failure, error, skipped + } + + //how to deal with sofrt errors like api mishmash or similar + public static final String SOFT_ERROR_AS = "jcstress.report.xml.softErrorAs"; //pass/fail/skip/error defaults to skip + //how to deal with hard errors. Those may be timout, but also segfaulting vm + public static final String HARD_ERROR_AS = "jcstress.report.xml.hardErrorAs"; //pass/fail/skip/error defaults to fail + //only for full (non-saprse) output, will wrap each family by its + public static final String USE_TESTSUITES = "jcstress.report.xml.sparse.testsuites"; + //in case of sued testsuiotes, will not replicate the name of suite in test name. + //it is not stripped by default for sake of comparisns + public static final String TESTSUITES_STRIPNAMES = "jcstress.report.xml.sparse.stripNames"; + //will repritn system and jvm info in each test (may significantly waste sapce) + public static final String DUPLICATE_PROPERTIES = "jcstress.report.xml.properties.dupliate"; + //will move stdout/err to failure/error message for failures/errors and omit for passes + //this is for tools,m which do nto show stdout/err properly + //also it is saving a bit of space, but is loosing the granularity + public static final String STDOUTERR_TO_FAILURE = "jcstress.report.xml.souterr2failure"; + //will validate final xmls + public static final String VALIDATE = "jcstress.report.xml.validate"; + //will keep non standart (but widelyused) elements uncomment + public static final String UNCOMMENT_NONSTANDART = "jcstress.report.xml.nonstandart"; + //by default both reports are printed. By setting it to true or false, will include only sparse ot full + public static final String SPARSE = "jcstress.report.xml.sparse"; //true/false/null + + private final String resultDir; + private final InProcessCollector collector; + private final boolean sparse; + private final PrintStream out; + + public XMLReportPrinter(String resultDir, InProcessCollector collector, PrintStream out, boolean sparse) { + //sparse true -ALL_MATCHING + //sparse false - as ALL_MATCHING_COMBINATIONS + //jednou smichat, jednou ne. Varovani kolik jich bude + //-xml true/false, defaults to sparse + this.collector = collector; + this.resultDir = resultDir; + this.sparse = sparse; + File dir = new File(resultDir); + dir.mkdirs(); + out.println(" " + getSparseString() + " XML report generated at " + dir.getAbsolutePath() + File.separator + getMainFileName()); + this.out = out; + } + + private static JunitResult getSoftErrorAs() { + if (System.getProperty(XMLReportPrinter.SOFT_ERROR_AS) == null) { + return JunitResult.skipped; + } + return Enum.valueOf(JunitResult.class, System.getProperty(XMLReportPrinter.SOFT_ERROR_AS)); + } + + private static JunitResult getHardErrorAs() { + if (System.getProperty(XMLReportPrinter.HARD_ERROR_AS) == null) { + return JunitResult.failure; + } + return Enum.valueOf(JunitResult.class, System.getProperty(XMLReportPrinter.HARD_ERROR_AS)); + } + + public static Boolean getSparse(PrintStream out) { + String sparse = System.getProperty(SPARSE); + if (sparse == null) { + return null; + } else { + if ("true".equals(sparse) || "false".equals(sparse)) { + return Boolean.getBoolean(XMLReportPrinter.SPARSE); + } else { + if (out != null) { + out.println("Invalid " + SPARSE + " value of " + sparse + "Should be true/false or missing"); + } + return null; + } + } + } + + + private static boolean isTestsuiteUsed() { + return System.getProperty(XMLReportPrinter.USE_TESTSUITES) != null; + } + + private static boolean isStdoutErrToFailure() { + return System.getProperty(XMLReportPrinter.STDOUTERR_TO_FAILURE) != null; + } + + private static boolean isStripNames() { + return System.getProperty(XMLReportPrinter.TESTSUITES_STRIPNAMES) != null; + } + + private static boolean isValidate() { + return System.getProperty(XMLReportPrinter.VALIDATE) != null; + } + + private static boolean isDuplicateProperties() { + return System.getProperty(XMLReportPrinter.DUPLICATE_PROPERTIES) != null; + } + + private static boolean isXsdLenient() { + return System.getProperty(XMLReportPrinter.UNCOMMENT_NONSTANDART) != null; + } + + private static String printProperty(String key, boolean value) { + return printProperty(key, "" + value); + } + + private static String printProperty(String key, String value) { + return ""; + } + + private static String getBaseProperties(List sorted) { + StringBuilder sb = new StringBuilder(); + for (Map.Entry entry : HTMLReportPrinter.getEnv(sorted).entrySet()) { + sb.append(" " + printProperty(entry.getKey(), entry.getValue())).append("\n"); + } + return sb.toString(); + } + + + private static String getSeed(TestResult r) { + if (r.getConfig().getSeed() != null) { + return (" " + printProperty("seed", getCleanSeed(r)) + "\n"); + } else { + return null; + } + } + + private static String getCleanSeed(TestResult r) { + String[] args = r.getConfig().getSeed().split("="); + if (args.length > 1) { + return args[1]; + } + return args[0]; + } + + private static String getRefs(TestInfo test) { + StringBuilder sb = new StringBuilder(); + for (String ref : test.refs()) { + if (ref != null) { + sb.append(" " + printProperty("bug", ref)).append("\n"); + } + } + return sb.toString(); + } + + private static String getTestDescription(TestInfo test) { + if (test.description() != null && !test.description().equals("null")) { + return " " + printProperty("description", test.description() + "\n"); + } else { + return null; + } + } + + private String getMainFileName() { + return "junit-" + getSparseString() + ".xml"; + } + + private String getSparseString() { + return sparse ? "sparse" : "full"; + } + + public void work() throws FileNotFoundException { + List byName = sparse ? ReportUtils.mergedByName(collector.getTestResults()) : new ArrayList<>(collector.getTestResults()); + Collections.sort(byName, Comparator.comparing(TestResult::getName)); + + String filePath = resultDir + File.separator + getMainFileName(); + PrintWriter output = new PrintWriter(filePath); + + output.println(""); + output.println(""); + if (isMainTestsuite()) { + String header = printTestSuiteHeader(byName, "jcstress"); + output.println(" " + header); + printMainproperties(output, byName); + } + byName = ReportUtils.mergedByName(collector.getTestResults()); + Collections.sort(byName, Comparator.comparing(TestResult::getName)); + emitTestReports(ReportUtils.byName(collector.getTestResults()), output); + if (isMainTestsuite()) { + output.println(" "); + } + output.println(""); + output.flush(); + output.close(); + if (isValidate()) { + validate(filePath); + } + } + + private void printMainproperties(PrintWriter output, List byName) { + output.println(" "); + output.print(getBaseProperties(byName)); + //FIXME print all properties, if 7903889 got ever implemented + printXmlReporterProperties(output); + output.println(" "); + } + + private boolean isMainTestsuite() { + if (sparse) { + return true; + } else { + if (!isTestsuiteUsed()) { + return true; + } else { + return isXsdLenient(); + } + } + } + + private static String printTestSuiteHeader(List results, String name) { + Map> reordered = countJunitResults(results); + String hostname = "localhost"; + try { + hostname = InetAddress.getLocalHost().getHostName(); + } catch (Exception ex) { + //no interest + } + String prefix = ""; + String passed = " passed='" + reordered.get(JunitResult.pass).size() + "'"; + if (!isXsdLenient()) { + prefix = "\n"; + passed = ""; + } + return prefix + ""; + } + + private static Map> countJunitResults(List results) { + List passed = new ArrayList<>(); + List failed = new ArrayList<>(); + List error = new ArrayList<>(); + List skipped = new ArrayList<>(); + Map> sorted = new HashMap<>(); + sorted.put(JunitResult.pass, passed); + sorted.put(JunitResult.error, error); + sorted.put(JunitResult.failure, failed); + sorted.put(JunitResult.skipped, skipped); + for (TestResult result : results) { + switch (testToJunitResult(result)) { + case pass: + passed.add(result); + break; + case failure: + failed.add(result); + break; + case error: + error.add(result); + break; + case skipped: + skipped.add(result); + break; + default: + throw new RuntimeException("Unknown JunitResult: " + result.status() + "/" + testToJunitResult(result)); + } + } + int checksum = passed.size() + failed.size() + error.size() + skipped.size(); + if (checksum != results.size()) { + throw new RuntimeException("Missed tests in summ-up: " + checksum + ", was supposed to be: " + results.size()); + } + return sorted; + } + + private void printXmlReporterProperties(PrintWriter output) { + output.println(" " + printProperty("sparse", sparse)); + output.println(" " + printProperty(USE_TESTSUITES, isTestsuiteUsed())); + output.println(" " + printProperty(TESTSUITES_STRIPNAMES, isStripNames())); + output.println(" " + printProperty(SOFT_ERROR_AS, getSoftErrorAs().toString())); + output.println(" " + printProperty(HARD_ERROR_AS, getHardErrorAs().toString())); + output.println(" " + printProperty(DUPLICATE_PROPERTIES, isDuplicateProperties())); + output.println(" " + printProperty(UNCOMMENT_NONSTANDART, isXsdLenient())); + output.println(" " + printProperty(STDOUTERR_TO_FAILURE, isStdoutErrToFailure())); + output.println(" " + printProperty(SPARSE, Objects.toString(getSparse(null)))); + } + + private void emitTestReports(Multimap multiByName, PrintWriter local) { + multiByName.keys().stream().forEach(name -> { + TestInfo test = TestList.getInfo(name); + emitTestReport(local, multiByName.get(name), test, name); + }); + } + + private void emitTestReport(PrintWriter outw, Collection results, TestInfo test, String suiteCandidate) { + //in sparse mode we print only test.name as test, with result based on cumulative + //otherwise we will be printing only its individual combinations (to mach the summary) + if (sparse) { + List sorted = new ArrayList<>(results); + HTMLReportPrinter.resultsOrder(sorted); + outw.println(" " + getOpenTestcase(results, test.name())); + if (isXsdLenient()) { + printPropertiesPerTest(outw, test, null, sorted); + } + printMainTestBody(outw, sorted, true); + outw.println(""); + } else { + List sorted = new ArrayList<>(results); + HTMLReportPrinter.resultsOrder(sorted); + if (isTestsuiteUsed()) { + String header = printTestSuiteHeader(sorted, suiteCandidate); + outw.println(header); + if (!isMainTestsuite()) { + printMainproperties(outw, sorted); + } + } + for (TestResult r : sorted) { + String testName = r.getConfig().toDetailedTest(false); + if (isTestsuiteUsed() && isStripNames()) { + testName = r.getConfig().getTestVariant(false); + } + outw.println(" " + getOpenTestcase(Arrays.asList(r), testName)); + if (isXsdLenient()) { + printPropertiesPerTest(outw, test, r, sorted); + } + printMainTestBody(outw, Arrays.asList(r), null); + outw.println(""); + } + if (isTestsuiteUsed()) { + outw.println(" "); + } + } + + } + + private static String getOpenTestcase(Collection results, String test) { + return ""; + } + + private static String getTime(Collection results, boolean toNicestring) { + double sum = 0; + for (TestResult resul : results) { + sum += resul.getTotalCount(); + } + if (toNicestring) { + if (sum > 10) { + return "10^" + (long) Math.floor((Math.log10(sum))); + } else { + return String.valueOf(sum); + } + } else { + return "" + sum; + } + } + + private static void printPropertiesPerTest(PrintWriter outw, TestInfo test, TestResult result, List sorted) { + String props = printPropertiesPerTest(test, result, sorted); + if (!props.isEmpty()) { + outw.println(" "); + outw.print(props); + outw.println(" "); + } + } + + private static String printPropertiesPerTest(TestInfo test, TestResult result, List sorted) { + StringBuilder sb = new StringBuilder(); + String description = getTestDescription(test); + if (description != null && !description.isEmpty()) { + sb.append(description); + } + String refs = getRefs(test); + if (refs != null && !refs.isEmpty()) { + sb.append(refs); + } + if (result != null) { + String seed = getSeed(result); + if (seed != null && !seed.isEmpty()) { + sb.append(seed); + } + } + if (result != null) { + if (result.grading().hasInteresting) { + sb.append(" " + printProperty("interesting", true) + "\n"); + } + } else { + boolean anyIntresting = getAnyInteresting(sorted); + if (anyIntresting) { + sb.append(" " + printProperty("interesting", true) + "\n"); + } + } + + if (isDuplicateProperties()) { + String baseProps = getBaseProperties(sorted); + if (baseProps != null && !baseProps.isEmpty()) { + sb.append(baseProps); + } + } + return sb.toString(); + } + + private static boolean getAnyInteresting(List results) { + for (TestResult result : results) { + if (result.grading().hasInteresting) { + return true; + } + } + return false; + } + + private static void printMainTestBody(PrintWriter outw, List results, Boolean header) { + Set keys = new TreeSet<>(); + for (TestResult result : results) { + keys.addAll(result.getStateKeys()); + } + printStatusElement(outw, results, keys, header); + if (!isStdoutErrToFailure()) { + printSystemOutElement(outw, results, header); + printSystemErrElement(outw, results, header); + } + } + + private static void printStatusElement(PrintWriter outw, List results, Set keys, Boolean header) { + JunitResult junitResult = testsToJunitResult(results); + if (junitResult == JunitResult.failure || junitResult == JunitResult.error) { + outw.println("<" + junitResult + ">"); + } + if (junitResult == JunitResult.skipped) { + if (isXsdLenient()) { + outw.println(" "); + } else { + outw.println(" "); + outw.println(" "); + } + } + } + + private static void printSystemErrElement(PrintWriter outw, Collection results, Boolean header) { + outw.println("\n"); + } + + private static void printSystemOutElement(PrintWriter outw, Collection results, Boolean header) { + outw.println(""); + } + + private static void printVmErr(PrintWriter outw, TestResult result, Boolean header) { + printTestLines(result.getVmErr(), outw, result, header); + } + + private static void printVmOut(PrintWriter outw, TestResult result, Boolean header) { + printTestLines(result.getVmOut(), outw, result, header); + } + + private static void printMessages(PrintWriter outw, TestResult result, Boolean header) { + printTestLines(result.getMessages(), outw, result, header); + } + + private static void printTestLines(List lines, PrintWriter outw, TestResult originalResult, Boolean header) { + if (!lines.isEmpty()) { + resultHeader(outw, originalResult, header); + for (String data : lines) { + outw.println(data); + } + outw.println(); + } + } + + private static void printHtmlInfo(TestResult result, PrintWriter out, Set keys) { + String color = ReportUtils.statusToPassed(result) ? "green" : "red"; + String label = ReportUtils.statusToLabel(result); + out.println("html signatue: " + color + " - " + label); + getGradings(result, out, keys); + } + + private static void getGradings(TestResult result, PrintWriter out, Set keys) { + for (String key : keys) { + GradingResult c = result.grading().gradingResults.get(key); + if (c != null) { + out.println(selectHtmlGradingColor(c.expect, c.count == 0) + "/" + c.count + ""); + } else { + out.println(selectHtmlGradingColor(Expect.ACCEPTABLE, true) + "/0"); + } + } + } + + private static void resultHeader(PrintWriter outw, TestResult r, Boolean full) { + if (full != null) { + TestConfig cfg = r.getConfig(); + if (full) { + outw.println(cfg.toDetailedTest(false)); + } else { + outw.println("CompileMode: " + CompileMode.description(cfg.compileMode, cfg.actorNames)); + outw.println("SchedulingClass" + SchedulingClass.description(cfg.shClass, cfg.actorNames)); + outw.println(""); + if (!cfg.jvmArgs.isEmpty()) { + outw.println("jvmargs:" + cfg.jvmArgs); + } + } + } + } + + private static String selectHtmlGradingColor(Expect type, boolean isZero) { + switch (type) { + case ACCEPTABLE: + return isZero ? "LIGHT_GRAY" : "GREEN"; + case FORBIDDEN: + return isZero ? "LIGHT_GRAY" : "RED"; + case ACCEPTABLE_INTERESTING: + return isZero ? "LIGHT_GRAY" : "CYAN"; + case UNKNOWN: + return "RED"; + default: + throw new IllegalStateException(); + } + } + + private void validate(String xml) { + try { + out.println("Checking: " + xml); + wellFormed(xml); + validByXsd(xml); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + private void wellFormed(String xml) throws ParserConfigurationException, SAXException, IOException { + out.println("Well formed?"); + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + factory.setValidating(false); + factory.setNamespaceAware(true); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document document = builder.parse(new InputSource(xml)); + out.println("Well formed!"); + } + + private void validByXsd(String xml) throws ParserConfigurationException, SAXException, IOException, URISyntaxException { + String url = "https://raw.githubusercontent.com/junit-team/junit5/refs/heads/main/platform-tests/src/test/resources/jenkins-junit.xsd"; + out.println("Valid by " + url + " ?"); + SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema schema = factory.newSchema(new URI(url).toURL()); + Validator validator = schema.newValidator(); + validator.validate(new StreamSource(new File(xml))); + out.println("Valid!"); + } + + public static JunitResult testsToJunitResult(Collection results) { + boolean hadError = false; + int countSkipped = 0; + for (TestResult result : results) { + if (testToJunitResult(result) == JunitResult.failure) { + //if there was failure in sub set, return whole group as failure + return JunitResult.failure; + } + if (testToJunitResult(result) == JunitResult.error) { + hadError = true; + } + if (testToJunitResult(result) == JunitResult.skipped) { + countSkipped++; + } + } + //no failure, bute errors presented + if (hadError) { + return JunitResult.error; + } + //no failure, no error, was all skipped? + if (countSkipped == results.size()) { + return JunitResult.skipped; + } + return JunitResult.pass; + } + + public static JunitResult testToJunitResult(TestResult result) { + switch (result.status()) { + case TIMEOUT_ERROR: + return JunitResult.error; + case CHECK_TEST_ERROR: + return JunitResult.error; + case TEST_ERROR: + case VM_ERROR: + return getHardErrorAs(); + case API_MISMATCH: + return getSoftErrorAs(); + case NORMAL: + return JunitResult.pass; + default: + throw new IllegalStateException("Illegal status: " + result.status()); + } + } +} diff --git a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java index 079feabb..d04e8515 100644 --- a/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java +++ b/jcstress-core/src/main/java/org/openjdk/jcstress/infra/runners/TestConfig.java @@ -35,6 +35,7 @@ import java.io.PrintWriter; import java.io.Serializable; +import java.util.ArrayList; import java.util.List; public class TestConfig implements Serializable { @@ -246,13 +247,13 @@ public void generateDirectives(PrintWriter pw, Verbosity verbosity) { } - public String toDetailedTest() { + public String getTestVariant(boolean keepSeed) { //binaryName have correct $ instead of . in name; omitted //generatedRunnerName name with suffix (usually _Test_jcstress) omitted //super.toString() as TestConfig@hash - omitted - StringBuilder verboseOutput = new StringBuilder(name); - verboseOutput.append(" {") - .append(actorNames) + //cpumap - null in listing, no reasonable toString method => omitted + StringBuilder idString = new StringBuilder(); + idString.append(actorNames) .append(", spinLoopStyle: ").append(spinLoopStyle) .append(", threads: ").append(threads) .append(", forkId: ").append(forkId) @@ -261,9 +262,36 @@ public String toDetailedTest() { .append(", shClass: ").append(shClass) .append(", strideSize: ").append(strideSize) .append(", strideCount: ").append(strideCount) - .append(", cpuMap: ").append(cpuMap) - .append(", ").append(jvmArgs) + .append(", ").append(keepSeed ? jvmArgs : maskSeed(jvmArgs)); //this is intentionally last to keep remaining prefix easily searchable + return idString.toString(); + } + + private List maskSeed(List jvmArgs) { + List argsCopy = new ArrayList<>(jvmArgs.size()); + for (String arg : jvmArgs) { + if (arg.startsWith("-XX:StressSeed=")) { + argsCopy.add(arg.replaceAll("[0-9]+", "yyyyyyyy")); + } else { + argsCopy.add(arg); + } + } + return argsCopy; + } + + public String getSeed() { + for (String arg : jvmArgs) { + if (arg.startsWith("-XX:StressSeed=")) { + return arg; + } + } + return null; + } + + public String toDetailedTest(boolean keepSeed) { + StringBuilder verboseOutput = new StringBuilder(name); + verboseOutput.append(" {") + .append(getTestVariant(keepSeed)) .append("}"); return verboseOutput.toString(); } -} +} \ No newline at end of file