From caba51c9a128f5e61ff4f65db936b165d9e733b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Thu, 8 Jan 2026 17:17:35 +0100 Subject: [PATCH 01/51] Minimal reworked impl with IR test passing --- src/hotspot/share/opto/loopnode.cpp | 98 +++++++ .../loopopts/TestReductionReassociation.java | 270 ++++++++++++++++++ 2 files changed, 368 insertions(+) create mode 100644 test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 8dc34af9c198b..61a8f42361c79 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4978,6 +4978,86 @@ bool PhaseIdealLoop::process_expensive_nodes() { return progress; } +static Node* reassociate_chain(Node* node, PhiNode* phi, Node* loop_head, PhaseIdealLoop* phase) { + if (phi == node->in(1)) { + return node->in(2); + } + + if (phi == node->in(2)) { + return node->in(1); + } + + Node* left; + Node* right; + if (node->in(1)->Opcode() == Op_MaxL) { + left = reassociate_chain(node->in(1), phi, loop_head, phase); + right = node->in(2); + } else { + left = node->in(1); + right = reassociate_chain(node->in(2), phi, loop_head, phase); + } + + Node* reassoc = new MaxLNode(phase->C, left, right); + phase->register_new_node(reassoc, loop_head); + return reassoc; +} + +static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLoop* phase) { + Node* chain_head = nullptr; + Node* current = phi; + int chain_length = 0; + while (current != nullptr) { + if (current->outcnt() != 1) { + break; + } + + Node* use = current->find_out_with(Op_MaxL); + if (use != nullptr) { + if (!phase->ctrl_is_member(lpt, use)) { + // Only interested in commutative add nodes that are in use in the loop + return; + } + if (use->in(1)->Opcode() == Op_MaxL && use->in(2)->Opcode() == Op_MaxL) { + // A chain to reassociate cannot be constructed + // when the chain can have multiple paths + return; + } + + chain_length++; + chain_head = use; + } + + current = use; + } + + if (chain_length < 2) { + // Only reassociate long enough chains + return; + } + + // todo remove check + if (!is_power_of_2(chain_length)) { + // If not power of 2 chains are common enough, they could be reassociated + // in a simpler way without using a tree. Moving the Phi value to the front + // would be enough to get a performance boost. + return; + } + + // tty->print("[avoid-cmov] try reassociate; chain length: %d\n", chain_length); + // tty->print("[avoid-cmov] try reassociate:\n"); + // tty->print("[avoid-cmov] phi: "); + // phi->dump(); + // tty->print("[avoid-cmov] try reassociate; chain head:\n"); + // chain_head->dump(); + + Node* loop_head = lpt->head(); + Node* reassociated = reassociate_chain(chain_head, phi, loop_head, phase); + + Node* new_chain_head = new MaxLNode(phase->C, phi, reassociated); + phase->register_new_node(new_chain_head, loop_head); + phase->igvn().replace_node(chain_head, new_chain_head); +} + //============================================================================= //----------------------------build_and_optimize------------------------------- // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to @@ -5353,6 +5433,24 @@ void PhaseIdealLoop::build_and_optimize() { } C->set_major_progress(); } + + if (!C->major_progress()) { + for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { + IdealLoopTree* lpt = iter.current(); + // todo do I need is_counted? + if (lpt->is_innermost() && lpt->is_counted()) { + Node* loop_head = lpt->head(); + + // Look for loop head uses that are Phi + for (DUIterator_Fast imax, i = loop_head->fast_outs(imax); i < imax; i++) { + Node* loop_head_use = loop_head->fast_out(i); + if (loop_head_use->is_Phi()) { + try_reassociate_chain(loop_head_use->as_Phi(), lpt, this); + } + } + } + } + } } #ifndef PRODUCT diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java new file mode 100644 index 0000000000000..0f392706ebc80 --- /dev/null +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -0,0 +1,270 @@ +/* + * Copyright (c) 2026 IBM Corporation. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @bug 8351409 + * @summary Test the IR effects of reduction reassociation + * @library /test/lib / + * @run driver compiler.loopopts.TestReductionReassociation + */ + +package compiler.loopopts; + +import compiler.lib.compile_framework.CompileFramework; +import compiler.lib.generators.Generator; +import compiler.lib.ir_framework.*; +import compiler.lib.template_framework.Template; +import compiler.lib.template_framework.TemplateToken; +import compiler.lib.template_framework.library.CodeGenerationDataNameType; +import compiler.lib.template_framework.library.PrimitiveType; +import compiler.lib.template_framework.library.TestFrameworkClass; +import compiler.lib.verify.Verify; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.IntStream; + +import static compiler.lib.generators.Generators.G; +import static compiler.lib.template_framework.Template.*; +import static compiler.lib.template_framework.Template.let; + +public class TestReductionReassociation { + static final Generator GEN_L = G.longs(); + + static final long[] aL = new long[10_000]; + + final Object[] gold = test(); + + static { + G.fill(GEN_L, aL); + } + + public static void main(String[] args) { + TestFramework.runWithFlags("-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0"); + } + + @Test + @IR(counts = {IRNode.MAX_L, "= 4"}, phase = CompilePhase.AFTER_LOOP_OPTS) + public Object[] test() { + long result = Integer.MIN_VALUE; + long result2 = Integer.MIN_VALUE; + for (int i = 0; i < aL.length; i += 4) { + long v0 = aL[i + 0]; + long v1 = aL[i + 1]; + long v2 = aL[i + 2]; + long v3 = aL[i + 3]; + + // result = max(v3, max(v2, max(v1, max(v0, result)))) + long u0 = Math.max(v0, result); + long u1 = Math.max(v1, u0); + long u2 = Math.max(v2, u1); + long u3 = Math.max(v3, u2); + result = u3; + + // result2 = max(result, max(v3, max(v2, max(v1, v0))) + long t0 = Math.max(v0, v1); + long t1 = Math.max(v2, t0); + long t2 = Math.max(v3, t1); + long t3 = Math.max(result, t2); + result2 = t3; + } + + return new Object[]{result, result2}; + } + + @Check(test = "test") + public void check(Object[] vals) { + Verify.checkEQ(gold[0], vals[0]); + Verify.checkEQ(gold[1], vals[1]); + Verify.checkEQ(vals[0], vals[1]); + } + +// public static void main(String[] args) { +// // Create a new CompileFramework instance. +// CompileFramework comp = new CompileFramework(); +// +// // Add a java source file. +// comp.addJavaSourceCode("compiler.loopopts.templated.ReductionReassociation", generate(comp)); +// +// // Compile the source file. +// comp.compile(); +// +// String[] flags = new String[] {"-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:VerifyIterativeGVN=1000"}; +// comp.invoke("compiler.loopopts.templated.ReductionReassociation", "main", new Object[] {flags}); +// } +// +// public static String generate(CompileFramework comp) { +// List testTemplateTokens = new ArrayList<>(); +// +// final int size = 10_000; +// +// testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, size).generate()); +// +// // Create the test class, which runs all testTemplateTokens. +// return TestFrameworkClass.render( +// // package and class name. +// "compiler.loopopts.templated", "ReductionReassociation", +// // List of imports. +// Set.of("compiler.lib.generators.*", +// "compiler.lib.verify.*"), +// // classpath, so the Test VM has access to the compiled class files. +// comp.getEscapedClassPathOfCompiledClasses(), +// // The list of tests. +// testTemplateTokens); +// } +// +// enum AddOp { +// MAX_L(CodeGenerationDataNameType.longs()); +// +// final PrimitiveType type; +// +// AddOp(PrimitiveType type) { +// this.type = type; +// } +// } +// +// record TestGenerator(AddOp add, int size) { +// public TemplateToken generate() { +// final String id = add.toString(); +// var testTemplate = Template.make(() -> { +// String test = $("test_" + id); +// String input = $("input_" + id); +// String expected = $("expected_" + id); +// String setup = $("setup_" + id); +// String check = $("check_" + id); +// return scope( +// """ +// // --- $test start --- +// +// """, +// generateArrayField(input), +// generateExpectedField(test, expected), +// // generateSetup(setup, input), +// generateTest(input, setup, test), +// generateCheck(test, check, expected), +// """ +// +// // --- $test end --- +// """ +// ); +// }); +// return testTemplate.asToken(); +// } +// +// private TemplateToken generateCheck(String test, String check, String expected) { +// var template = Template.make(() -> scope( +// let("test", test), +// let("check", check), +// let("expected", expected), +// """ +// @Check(test = "#test") +// public void #check(Object[] results) { +// Verify.checkEQ(#expected[0], results[0]); +// Verify.checkEQ(#expected[1], results[1]); +// Verify.checkEQ(results[0], results[1]); +// } +// """ +// )); +// return template.asToken(); +// } +// +// private TemplateToken generateOp(String a, String b) { +// var template = Template.make(() -> scope( +// let("a", a), +// let("b", b), +// switch (add) { +// case MAX_L -> "Math.max(#a, #b)"; +// } +// )); +// return template.asToken(); +// } +// +// private TemplateToken generateTest(String input, String setup, String test) { +// var template = Template.make(() -> scope( +// let("irNodeName", add.name()), +// let("input", input), +// let("setup", setup), +// let("test", test), +// let("type", add.type.name()), +// """ +// @Test +// @IR(counts = {IRNode.#irNodeName, "= 4"}, phase = CompilePhase.AFTER_LOOP_OPTS) +// public Object[] #test() { +// #type result = Integer.MIN_VALUE; +// #type result2 = Integer.MIN_VALUE; +// for (int i = 0; i < #input.length; i += 4) { +// long v0 = #input[i + 0]; +// long v1 = #input[i + 1]; +// long v2 = #input[i + 2]; +// long v3 = #input[i + 3]; +// """, +// "long u0 = ", generateOp("v0", "result"), ";", +// "long u1 = ", generateOp("v1", "u0"), ";", +// "long u2 = ", generateOp("v2", "u1"), ";", +// "long u3 = ", generateOp("v3", "u2"), ";", +// "result = u3;", +// "long t0 = ", generateOp("v0", "v1"), ";", +// "long t1 = ", generateOp("v2", "v3"), ";", +// "long t2 = ", generateOp("t0", "t1"), ";", +// "long t3 = ", generateOp("result", "t2"), ";", +// "result2 = t3;", +// """ +// } +// return new Object[]{result, result2}; +// } +// """ +// )); +// return template.asToken(); +// } +// +// private TemplateToken generateExpectedField(String test, String expected) { +// var template = Template.make(() -> scope( +// let("size", size), +// let("test", test), +// let("expected", expected), +// """ +// private Object[] #expected = #test(); +// """ +// )); +// return template.asToken(); +// } +// +// private TemplateToken generateArrayField(String input) { +// var template = Template.make(() -> scope( +// let("size", size), +// let("input", input), +// let("type", add.type.name()), +// let("gen", add.type.name() + "s"), +// """ +// private static #type[] #input = new #type[#size]; +// static { +// Generators.G.fill(Generators.G.#gen(), #input); +// } +// """ +// )); +// return template.asToken(); +// } +// } +} From f339cdf95994d0d47fb3834381b7a25c2b7b9556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Thu, 8 Jan 2026 17:40:22 +0100 Subject: [PATCH 02/51] Make it a template framework test --- .../loopopts/TestReductionReassociation.java | 356 ++++++++---------- 1 file changed, 153 insertions(+), 203 deletions(-) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 0f392706ebc80..a90466a0ee46b 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -51,220 +51,170 @@ import static compiler.lib.template_framework.Template.let; public class TestReductionReassociation { - static final Generator GEN_L = G.longs(); + public static void main(String[] args) { + // Create a new CompileFramework instance. + CompileFramework comp = new CompileFramework(); - static final long[] aL = new long[10_000]; + // Add a java source file. + comp.addJavaSourceCode("compiler.loopopts.templated.ReductionReassociation", generate(comp)); - final Object[] gold = test(); + // Compile the source file. + comp.compile(); - static { - G.fill(GEN_L, aL); + String[] flags = new String[] {"-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:VerifyIterativeGVN=1000"}; + comp.invoke("compiler.loopopts.templated.ReductionReassociation", "main", new Object[] {flags}); } - public static void main(String[] args) { - TestFramework.runWithFlags("-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0"); + public static String generate(CompileFramework comp) { + List testTemplateTokens = new ArrayList<>(); + + final int size = 10_000; + + testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, size).generate()); + + // Create the test class, which runs all testTemplateTokens. + return TestFrameworkClass.render( + // package and class name. + "compiler.loopopts.templated", "ReductionReassociation", + // List of imports. + Set.of("compiler.lib.generators.*", + "compiler.lib.verify.*"), + // classpath, so the Test VM has access to the compiled class files. + comp.getEscapedClassPathOfCompiledClasses(), + // The list of tests. + testTemplateTokens); } - @Test - @IR(counts = {IRNode.MAX_L, "= 4"}, phase = CompilePhase.AFTER_LOOP_OPTS) - public Object[] test() { - long result = Integer.MIN_VALUE; - long result2 = Integer.MIN_VALUE; - for (int i = 0; i < aL.length; i += 4) { - long v0 = aL[i + 0]; - long v1 = aL[i + 1]; - long v2 = aL[i + 2]; - long v3 = aL[i + 3]; + enum AddOp { + MAX_L(CodeGenerationDataNameType.longs()); - // result = max(v3, max(v2, max(v1, max(v0, result)))) - long u0 = Math.max(v0, result); - long u1 = Math.max(v1, u0); - long u2 = Math.max(v2, u1); - long u3 = Math.max(v3, u2); - result = u3; + final PrimitiveType type; - // result2 = max(result, max(v3, max(v2, max(v1, v0))) - long t0 = Math.max(v0, v1); - long t1 = Math.max(v2, t0); - long t2 = Math.max(v3, t1); - long t3 = Math.max(result, t2); - result2 = t3; + AddOp(PrimitiveType type) { + this.type = type; } - - return new Object[]{result, result2}; } - @Check(test = "test") - public void check(Object[] vals) { - Verify.checkEQ(gold[0], vals[0]); - Verify.checkEQ(gold[1], vals[1]); - Verify.checkEQ(vals[0], vals[1]); - } + record TestGenerator(AddOp add, int size) { + public TemplateToken generate() { + final String id = add.toString(); + var testTemplate = Template.make(() -> { + String test = $("test_" + id); + String input = $("input_" + id); + String expected = $("expected_" + id); + String setup = $("setup_" + id); + String check = $("check_" + id); + return scope( + """ + // --- $test start --- + + """, + generateArrayField(input), + generateExpectedField(test, expected), + // generateSetup(setup, input), + generateTest(input, setup, test), + generateCheck(test, check, expected), + """ + + // --- $test end --- + """ + ); + }); + return testTemplate.asToken(); + } + + private TemplateToken generateCheck(String test, String check, String expected) { + var template = Template.make(() -> scope( + let("test", test), + let("check", check), + let("expected", expected), + """ + @Check(test = "#test") + public void #check(Object[] results) { + Verify.checkEQ(#expected[0], results[0]); + Verify.checkEQ(#expected[1], results[1]); + Verify.checkEQ(results[0], results[1]); + } + """ + )); + return template.asToken(); + } + + private TemplateToken generateOp(String a, String b) { + var template = Template.make(() -> scope( + let("a", a), + let("b", b), + switch (add) { + case MAX_L -> "Math.max(#a, #b)"; + } + )); + return template.asToken(); + } + + private TemplateToken generateTest(String input, String setup, String test) { + var template = Template.make(() -> scope( + let("irNodeName", add.name()), + let("input", input), + let("setup", setup), + let("test", test), + let("type", add.type.name()), + """ + @Test + @IR(counts = {IRNode.#irNodeName, "= 4"}, phase = CompilePhase.AFTER_LOOP_OPTS) + public Object[] #test() { + #type result = Integer.MIN_VALUE; + #type result2 = Integer.MIN_VALUE; + for (int i = 0; i < #input.length; i += 4) { + long v0 = #input[i + 0]; + long v1 = #input[i + 1]; + long v2 = #input[i + 2]; + long v3 = #input[i + 3]; + """, + "long u0 = ", generateOp("v0", "result"), ";", + "long u1 = ", generateOp("v1", "u0"), ";", + "long u2 = ", generateOp("v2", "u1"), ";", + "long u3 = ", generateOp("v3", "u2"), ";", + "result = u3;", + "long t0 = ", generateOp("v0", "v1"), ";", + "long t1 = ", generateOp("v2", "t0"), ";", + "long t2 = ", generateOp("v3", "t1"), ";", + "long t3 = ", generateOp("result", "t2"), ";", + "result2 = t3;", + """ + } + return new Object[]{result, result2}; + } + """ + )); + return template.asToken(); + } -// public static void main(String[] args) { -// // Create a new CompileFramework instance. -// CompileFramework comp = new CompileFramework(); -// -// // Add a java source file. -// comp.addJavaSourceCode("compiler.loopopts.templated.ReductionReassociation", generate(comp)); -// -// // Compile the source file. -// comp.compile(); -// -// String[] flags = new String[] {"-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:VerifyIterativeGVN=1000"}; -// comp.invoke("compiler.loopopts.templated.ReductionReassociation", "main", new Object[] {flags}); -// } -// -// public static String generate(CompileFramework comp) { -// List testTemplateTokens = new ArrayList<>(); -// -// final int size = 10_000; -// -// testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, size).generate()); -// -// // Create the test class, which runs all testTemplateTokens. -// return TestFrameworkClass.render( -// // package and class name. -// "compiler.loopopts.templated", "ReductionReassociation", -// // List of imports. -// Set.of("compiler.lib.generators.*", -// "compiler.lib.verify.*"), -// // classpath, so the Test VM has access to the compiled class files. -// comp.getEscapedClassPathOfCompiledClasses(), -// // The list of tests. -// testTemplateTokens); -// } -// -// enum AddOp { -// MAX_L(CodeGenerationDataNameType.longs()); -// -// final PrimitiveType type; -// -// AddOp(PrimitiveType type) { -// this.type = type; -// } -// } -// -// record TestGenerator(AddOp add, int size) { -// public TemplateToken generate() { -// final String id = add.toString(); -// var testTemplate = Template.make(() -> { -// String test = $("test_" + id); -// String input = $("input_" + id); -// String expected = $("expected_" + id); -// String setup = $("setup_" + id); -// String check = $("check_" + id); -// return scope( -// """ -// // --- $test start --- -// -// """, -// generateArrayField(input), -// generateExpectedField(test, expected), -// // generateSetup(setup, input), -// generateTest(input, setup, test), -// generateCheck(test, check, expected), -// """ -// -// // --- $test end --- -// """ -// ); -// }); -// return testTemplate.asToken(); -// } -// -// private TemplateToken generateCheck(String test, String check, String expected) { -// var template = Template.make(() -> scope( -// let("test", test), -// let("check", check), -// let("expected", expected), -// """ -// @Check(test = "#test") -// public void #check(Object[] results) { -// Verify.checkEQ(#expected[0], results[0]); -// Verify.checkEQ(#expected[1], results[1]); -// Verify.checkEQ(results[0], results[1]); -// } -// """ -// )); -// return template.asToken(); -// } -// -// private TemplateToken generateOp(String a, String b) { -// var template = Template.make(() -> scope( -// let("a", a), -// let("b", b), -// switch (add) { -// case MAX_L -> "Math.max(#a, #b)"; -// } -// )); -// return template.asToken(); -// } -// -// private TemplateToken generateTest(String input, String setup, String test) { -// var template = Template.make(() -> scope( -// let("irNodeName", add.name()), -// let("input", input), -// let("setup", setup), -// let("test", test), -// let("type", add.type.name()), -// """ -// @Test -// @IR(counts = {IRNode.#irNodeName, "= 4"}, phase = CompilePhase.AFTER_LOOP_OPTS) -// public Object[] #test() { -// #type result = Integer.MIN_VALUE; -// #type result2 = Integer.MIN_VALUE; -// for (int i = 0; i < #input.length; i += 4) { -// long v0 = #input[i + 0]; -// long v1 = #input[i + 1]; -// long v2 = #input[i + 2]; -// long v3 = #input[i + 3]; -// """, -// "long u0 = ", generateOp("v0", "result"), ";", -// "long u1 = ", generateOp("v1", "u0"), ";", -// "long u2 = ", generateOp("v2", "u1"), ";", -// "long u3 = ", generateOp("v3", "u2"), ";", -// "result = u3;", -// "long t0 = ", generateOp("v0", "v1"), ";", -// "long t1 = ", generateOp("v2", "v3"), ";", -// "long t2 = ", generateOp("t0", "t1"), ";", -// "long t3 = ", generateOp("result", "t2"), ";", -// "result2 = t3;", -// """ -// } -// return new Object[]{result, result2}; -// } -// """ -// )); -// return template.asToken(); -// } -// -// private TemplateToken generateExpectedField(String test, String expected) { -// var template = Template.make(() -> scope( -// let("size", size), -// let("test", test), -// let("expected", expected), -// """ -// private Object[] #expected = #test(); -// """ -// )); -// return template.asToken(); -// } -// -// private TemplateToken generateArrayField(String input) { -// var template = Template.make(() -> scope( -// let("size", size), -// let("input", input), -// let("type", add.type.name()), -// let("gen", add.type.name() + "s"), -// """ -// private static #type[] #input = new #type[#size]; -// static { -// Generators.G.fill(Generators.G.#gen(), #input); -// } -// """ -// )); -// return template.asToken(); -// } -// } + private TemplateToken generateExpectedField(String test, String expected) { + var template = Template.make(() -> scope( + let("size", size), + let("test", test), + let("expected", expected), + """ + private Object[] #expected = #test(); + """ + )); + return template.asToken(); + } + + private TemplateToken generateArrayField(String input) { + var template = Template.make(() -> scope( + let("size", size), + let("input", input), + let("type", add.type.name()), + let("gen", add.type.name() + "s"), + """ + private static #type[] #input = new #type[#size]; + static { + Generators.G.fill(Generators.G.#gen(), #input); + } + """ + )); + return template.asToken(); + } + } } From e22aacd66d6c7a7ed693d178308e8207f81ec810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Thu, 8 Jan 2026 18:00:48 +0100 Subject: [PATCH 03/51] Remove unnecessary information --- .../compiler/loopopts/TestReductionReassociation.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index a90466a0ee46b..7cfd4d0713da1 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -97,13 +97,12 @@ enum AddOp { record TestGenerator(AddOp add, int size) { public TemplateToken generate() { - final String id = add.toString(); var testTemplate = Template.make(() -> { - String test = $("test_" + id); - String input = $("input_" + id); - String expected = $("expected_" + id); - String setup = $("setup_" + id); - String check = $("check_" + id); + String test = $("test"); + String input = $("input"); + String expected = $("expected"); + String setup = $("setup"); + String check = $("check"); return scope( """ // --- $test start --- From 8e26f00a1b72d11543a0521a8cd6d53756997a0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Fri, 9 Jan 2026 10:00:31 +0100 Subject: [PATCH 04/51] Support non-power-of-2 chains --- src/hotspot/share/opto/loopnode.cpp | 8 ---- .../loopopts/TestReductionReassociation.java | 37 +++++++++++-------- 2 files changed, 21 insertions(+), 24 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 61a8f42361c79..121f4641f97b1 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5035,14 +5035,6 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo return; } - // todo remove check - if (!is_power_of_2(chain_length)) { - // If not power of 2 chains are common enough, they could be reassociated - // in a simpler way without using a tree. Moving the Phi value to the front - // would be enough to get a performance boost. - return; - } - // tty->print("[avoid-cmov] try reassociate; chain length: %d\n", chain_length); // tty->print("[avoid-cmov] try reassociate:\n"); // tty->print("[avoid-cmov] phi: "); diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 7cfd4d0713da1..fd8de3cd961a1 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -69,8 +69,11 @@ public static String generate(CompileFramework comp) { List testTemplateTokens = new ArrayList<>(); final int size = 10_000; + final List batchSizes = List.of(4, 5); - testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, size).generate()); + for (Integer batchSize : batchSizes) { + testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, batchSize, size).generate()); + } // Create the test class, which runs all testTemplateTokens. return TestFrameworkClass.render( @@ -95,7 +98,7 @@ enum AddOp { } } - record TestGenerator(AddOp add, int size) { + record TestGenerator(AddOp add, int batchSize, int size) { public TemplateToken generate() { var testTemplate = Template.make(() -> { String test = $("test"); @@ -159,26 +162,28 @@ private TemplateToken generateTest(String input, String setup, String test) { let("type", add.type.name()), """ @Test - @IR(counts = {IRNode.#irNodeName, "= 4"}, phase = CompilePhase.AFTER_LOOP_OPTS) + """, + "@IR(counts = {IRNode.#irNodeName, \"= ", batchSize, "\"}, phase = CompilePhase.AFTER_LOOP_OPTS)", + """ public Object[] #test() { #type result = Integer.MIN_VALUE; #type result2 = Integer.MIN_VALUE; - for (int i = 0; i < #input.length; i += 4) { - long v0 = #input[i + 0]; - long v1 = #input[i + 1]; - long v2 = #input[i + 2]; - long v3 = #input[i + 3]; """, + "for (int i = 0; i < #input.length; i += ", batchSize, ") {", + IntStream.range(0, batchSize).mapToObj(i -> + List.of("long v", i, " = #input[i + ", i, "];\n") + ).toList(), "long u0 = ", generateOp("v0", "result"), ";", - "long u1 = ", generateOp("v1", "u0"), ";", - "long u2 = ", generateOp("v2", "u1"), ";", - "long u3 = ", generateOp("v3", "u2"), ";", - "result = u3;", + IntStream.range(1, batchSize).mapToObj(i -> + List.of("long u", i, " = ", generateOp("v" + i, "u" + (i - 1)), ";\n") + ).toList(), + "result = u", batchSize - 1,";", "long t0 = ", generateOp("v0", "v1"), ";", - "long t1 = ", generateOp("v2", "t0"), ";", - "long t2 = ", generateOp("v3", "t1"), ";", - "long t3 = ", generateOp("result", "t2"), ";", - "result2 = t3;", + IntStream.range(1, batchSize - 1).mapToObj(i -> + List.of("long t", i, " = ", generateOp("v" + (i + 1), "t" + (i - 1)), ";\n") + ).toList(), + "long t", batchSize - 1, " = ", generateOp("result", "t" + (batchSize - 2)), ";", + "result2 = t", batchSize - 1,";", """ } return new Object[]{result, result2}; From cf1e875a016c081a72b3b2b59da02828df53cc14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Fri, 9 Jan 2026 10:10:36 +0100 Subject: [PATCH 05/51] Remove is_counted() --- src/hotspot/share/opto/loopnode.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 121f4641f97b1..66052a63ff85f 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5429,8 +5429,7 @@ void PhaseIdealLoop::build_and_optimize() { if (!C->major_progress()) { for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { IdealLoopTree* lpt = iter.current(); - // todo do I need is_counted? - if (lpt->is_innermost() && lpt->is_counted()) { + if (lpt->is_innermost()) { Node* loop_head = lpt->head(); // Look for loop head uses that are Phi From d00cadd3580563371f88613027b748c23dbf1e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Fri, 9 Jan 2026 11:39:36 +0100 Subject: [PATCH 06/51] Test intermediate use of value --- .../compiler/loopopts/TestReductionReassociation.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index fd8de3cd961a1..1820a2aaacc00 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -72,9 +72,11 @@ public static String generate(CompileFramework comp) { final List batchSizes = List.of(4, 5); for (Integer batchSize : batchSizes) { - testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, batchSize, size).generate()); + testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, batchSize, false, size).generate()); } + testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, batchSizes.getFirst(), true, size).generate()); + // Create the test class, which runs all testTemplateTokens. return TestFrameworkClass.render( // package and class name. @@ -98,7 +100,7 @@ enum AddOp { } } - record TestGenerator(AddOp add, int batchSize, int size) { + record TestGenerator(AddOp add, int batchSize, boolean useIntermediate, int size) { public TemplateToken generate() { var testTemplate = Template.make(() -> { String test = $("test"); @@ -163,7 +165,9 @@ private TemplateToken generateTest(String input, String setup, String test) { """ @Test """, - "@IR(counts = {IRNode.#irNodeName, \"= ", batchSize, "\"}, phase = CompilePhase.AFTER_LOOP_OPTS)", + "@IR(counts = {IRNode.#irNodeName, \"= ", + useIntermediate ? batchSize * 2 : batchSize, + "\"}, phase = CompilePhase.AFTER_LOOP_OPTS)", """ public Object[] #test() { #type result = Integer.MIN_VALUE; @@ -173,6 +177,7 @@ private TemplateToken generateTest(String input, String setup, String test) { IntStream.range(0, batchSize).mapToObj(i -> List.of("long v", i, " = #input[i + ", i, "];\n") ).toList(), + useIntermediate ? List.of("if (v", batchSize - 1," == #input.hashCode()) { System.out.print(\"\"); }") : "", "long u0 = ", generateOp("v0", "result"), ";", IntStream.range(1, batchSize).mapToObj(i -> List.of("long u", i, " = ", generateOp("v" + i, "u" + (i - 1)), ";\n") From 21a2fc8bbcdedf212ce9ac586345d0426d438771 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Mon, 12 Jan 2026 14:08:22 +0100 Subject: [PATCH 07/51] Extend impl to other AddNodes, MinL for now --- src/hotspot/share/opto/loopnode.cpp | 48 +++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 66052a63ff85f..d1fe2ee90a370 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4978,7 +4978,7 @@ bool PhaseIdealLoop::process_expensive_nodes() { return progress; } -static Node* reassociate_chain(Node* node, PhiNode* phi, Node* loop_head, PhaseIdealLoop* phase) { +static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* loop_head, PhaseIdealLoop* phase) { if (phi == node->in(1)) { return node->in(2); } @@ -4989,15 +4989,25 @@ static Node* reassociate_chain(Node* node, PhiNode* phi, Node* loop_head, PhaseI Node* left; Node* right; - if (node->in(1)->Opcode() == Op_MaxL) { - left = reassociate_chain(node->in(1), phi, loop_head, phase); + if (node->in(1)->is_Add()) { + left = reassociate_chain(add_opcode, node->in(1), phi, loop_head, phase); right = node->in(2); } else { left = node->in(1); - right = reassociate_chain(node->in(2), phi, loop_head, phase); + right = reassociate_chain(add_opcode, node->in(2), phi, loop_head, phase); } - Node* reassoc = new MaxLNode(phase->C, left, right); + Node* reassoc = nullptr; + switch (add_opcode) { + case Op_MinL: + reassoc = new MinLNode(phase->C, left, right); + break; + case Op_MaxL: + reassoc = new MaxLNode(phase->C, left, right); + break; + default: + ShouldNotReachHere(); + } phase->register_new_node(reassoc, loop_head); return reassoc; } @@ -5006,18 +5016,28 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo Node* chain_head = nullptr; Node* current = phi; int chain_length = 0; + int add_opcode = 0; while (current != nullptr) { if (current->outcnt() != 1) { break; } - Node* use = current->find_out_with(Op_MaxL); + Node* use = nullptr; + for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) { + Node* n = current->fast_out(i); + if (n->is_Add()) { + use = n; + add_opcode = use->Opcode(); + break; + } + } + if (use != nullptr) { if (!phase->ctrl_is_member(lpt, use)) { // Only interested in commutative add nodes that are in use in the loop return; } - if (use->in(1)->Opcode() == Op_MaxL && use->in(2)->Opcode() == Op_MaxL) { + if (use->in(1)->Opcode() == add_opcode && use->in(2)->Opcode() == add_opcode) { // A chain to reassociate cannot be constructed // when the chain can have multiple paths return; @@ -5043,9 +5063,19 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo // chain_head->dump(); Node* loop_head = lpt->head(); - Node* reassociated = reassociate_chain(chain_head, phi, loop_head, phase); + Node* reassociated = reassociate_chain(add_opcode, chain_head, phi, loop_head, phase); - Node* new_chain_head = new MaxLNode(phase->C, phi, reassociated); + Node* new_chain_head = nullptr; + switch (add_opcode) { + case Op_MinL: + new_chain_head = new MinLNode(phase->C, phi, reassociated); + break; + case Op_MaxL: + new_chain_head = new MaxLNode(phase->C, phi, reassociated); + break; + default: + ShouldNotReachHere(); + } phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); } From 2080acc01a94abeb118263361af1934f6fcf6100 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Mon, 12 Jan 2026 14:12:14 +0100 Subject: [PATCH 08/51] Test MinL --- .../loopopts/TestReductionReassociation.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 1820a2aaacc00..b7c7fd5e0433c 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -31,6 +31,7 @@ package compiler.loopopts; +import compiler.igvn.TestMinMaxIdeal; import compiler.lib.compile_framework.CompileFramework; import compiler.lib.generators.Generator; import compiler.lib.ir_framework.*; @@ -45,6 +46,7 @@ import java.util.List; import java.util.Set; import java.util.stream.IntStream; +import java.util.stream.Stream; import static compiler.lib.generators.Generators.G; import static compiler.lib.template_framework.Template.*; @@ -69,13 +71,16 @@ public static String generate(CompileFramework comp) { List testTemplateTokens = new ArrayList<>(); final int size = 10_000; - final List batchSizes = List.of(4, 5); + final int batchSize = 4; - for (Integer batchSize : batchSizes) { - testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, batchSize, false, size).generate()); - } + Stream.of(TestReductionReassociation.AddOp.values()) + .map(op -> new TestGenerator(op, batchSize, false, size).generate()) + .forEach(testTemplateTokens::add); - testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, batchSizes.getFirst(), true, size).generate()); + // A single test to test a non-power-of-2 value + testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, 5, false, size).generate()); + // A single test where an intermediate value is used some other way + testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, batchSize, true, size).generate()); // Create the test class, which runs all testTemplateTokens. return TestFrameworkClass.render( @@ -91,6 +96,7 @@ public static String generate(CompileFramework comp) { } enum AddOp { + MIN_L(CodeGenerationDataNameType.longs()), MAX_L(CodeGenerationDataNameType.longs()); final PrimitiveType type; @@ -149,7 +155,8 @@ private TemplateToken generateOp(String a, String b) { let("a", a), let("b", b), switch (add) { - case MAX_L -> "Math.max(#a, #b)"; + case MIN_L -> "Long.min(#a, #b)"; + case MAX_L -> "Long.max(#a, #b)"; } )); return template.asToken(); From 66c77180e4a9ed5b54504efbb1bb41265335a8d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Mon, 12 Jan 2026 14:18:11 +0100 Subject: [PATCH 09/51] Refactor AddNode construction to build_add --- src/hotspot/share/opto/loopnode.cpp | 35 +++++++++++------------------ 1 file changed, 13 insertions(+), 22 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index d1fe2ee90a370..0bf3f04ca60bd 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4978,6 +4978,17 @@ bool PhaseIdealLoop::process_expensive_nodes() { return progress; } +static AddNode* build_add(int opcode, Node* a, Node* b, PhaseIdealLoop* phase) { + switch (opcode) { + case Op_MinL: + return new MinLNode(phase->C, a, b); + case Op_MaxL: + return new MaxLNode(phase->C, a, b); + default: + ShouldNotReachHere(); + } +} + static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* loop_head, PhaseIdealLoop* phase) { if (phi == node->in(1)) { return node->in(2); @@ -4997,17 +5008,7 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l right = reassociate_chain(add_opcode, node->in(2), phi, loop_head, phase); } - Node* reassoc = nullptr; - switch (add_opcode) { - case Op_MinL: - reassoc = new MinLNode(phase->C, left, right); - break; - case Op_MaxL: - reassoc = new MaxLNode(phase->C, left, right); - break; - default: - ShouldNotReachHere(); - } + Node* reassoc = build_add(add_opcode, left, right, phase); phase->register_new_node(reassoc, loop_head); return reassoc; } @@ -5065,17 +5066,7 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo Node* loop_head = lpt->head(); Node* reassociated = reassociate_chain(add_opcode, chain_head, phi, loop_head, phase); - Node* new_chain_head = nullptr; - switch (add_opcode) { - case Op_MinL: - new_chain_head = new MinLNode(phase->C, phi, reassociated); - break; - case Op_MaxL: - new_chain_head = new MaxLNode(phase->C, phi, reassociated); - break; - default: - ShouldNotReachHere(); - } + Node* new_chain_head = build_add(add_opcode, phi, reassociated, phase); phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); } From 6cc392c87943c7dcf03b85a1ed606c83ca841ecb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Mon, 12 Jan 2026 14:39:24 +0100 Subject: [PATCH 10/51] Expand to Min/Max integer --- src/hotspot/share/opto/loopnode.cpp | 4 ++ .../loopopts/TestReductionReassociation.java | 43 +++++++++++++------ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 0bf3f04ca60bd..c745c69e6d630 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4980,6 +4980,10 @@ bool PhaseIdealLoop::process_expensive_nodes() { static AddNode* build_add(int opcode, Node* a, Node* b, PhaseIdealLoop* phase) { switch (opcode) { + case Op_MinI: + return new MinINode(a, b); + case Op_MaxI: + return new MaxINode(a, b); case Op_MinL: return new MinLNode(phase->C, a, b); case Op_MaxL: diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index b7c7fd5e0433c..5f05a91119912 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -96,6 +96,8 @@ public static String generate(CompileFramework comp) { } enum AddOp { + MIN_I(CodeGenerationDataNameType.ints()), + MAX_I(CodeGenerationDataNameType.ints()), MIN_L(CodeGenerationDataNameType.longs()), MAX_L(CodeGenerationDataNameType.longs()); @@ -155,6 +157,8 @@ private TemplateToken generateOp(String a, String b) { let("a", a), let("b", b), switch (add) { + case MIN_I -> "Integer.min(#a, #b)"; + case MAX_I -> "Integer.max(#a, #b)"; case MIN_L -> "Long.min(#a, #b)"; case MAX_L -> "Long.max(#a, #b)"; } @@ -162,6 +166,21 @@ private TemplateToken generateOp(String a, String b) { return template.asToken(); } + private TemplateToken generateResultInit(String resultName) { + var template = Template.make(() -> scope( + let("resultName", resultName), + let("boxedType", add.type.boxedTypeName()), + let("type", add.type.name()), + "#type ", resultName, " = #boxedType.", + switch (add) { + case MIN_I, MIN_L -> "MAX_VALUE"; + case MAX_I, MAX_L -> "MIN_VALUE"; + }, + ";\n" + )); + return template.asToken(); + } + private TemplateToken generateTest(String input, String setup, String test) { var template = Template.make(() -> scope( let("irNodeName", add.name()), @@ -177,25 +196,25 @@ private TemplateToken generateTest(String input, String setup, String test) { "\"}, phase = CompilePhase.AFTER_LOOP_OPTS)", """ public Object[] #test() { - #type result = Integer.MIN_VALUE; - #type result2 = Integer.MIN_VALUE; """, - "for (int i = 0; i < #input.length; i += ", batchSize, ") {", + generateResultInit("result"), + generateResultInit("result2"), + "for (int i = 0; i < #input.length; i += ", batchSize, ") {\n", IntStream.range(0, batchSize).mapToObj(i -> - List.of("long v", i, " = #input[i + ", i, "];\n") + List.of("#type v", i, " = #input[i + ", i, "];\n") ).toList(), - useIntermediate ? List.of("if (v", batchSize - 1," == #input.hashCode()) { System.out.print(\"\"); }") : "", - "long u0 = ", generateOp("v0", "result"), ";", + useIntermediate ? List.of("if (v", batchSize - 1," == #input.hashCode()) { System.out.print(\"\"); }\n") : "", + "#type u0 = ", generateOp("v0", "result"), ";\n", IntStream.range(1, batchSize).mapToObj(i -> - List.of("long u", i, " = ", generateOp("v" + i, "u" + (i - 1)), ";\n") + List.of("#type u", i, " = ", generateOp("v" + i, "u" + (i - 1)), ";\n") ).toList(), - "result = u", batchSize - 1,";", - "long t0 = ", generateOp("v0", "v1"), ";", + "result = u", batchSize - 1,";\n", + "#type t0 = ", generateOp("v0", "v1"), ";\n", IntStream.range(1, batchSize - 1).mapToObj(i -> - List.of("long t", i, " = ", generateOp("v" + (i + 1), "t" + (i - 1)), ";\n") + List.of("#type t", i, " = ", generateOp("v" + (i + 1), "t" + (i - 1)), ";\n") ).toList(), - "long t", batchSize - 1, " = ", generateOp("result", "t" + (batchSize - 2)), ";", - "result2 = t", batchSize - 1,";", + "#type t", batchSize - 1, " = ", generateOp("result", "t" + (batchSize - 2)), ";\n", + "result2 = t", batchSize - 1,";\n", """ } return new Object[]{result, result2}; From 9d0a0b78d9e906a2b533434feea96bf6523e25a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Mon, 12 Jan 2026 17:37:52 +0100 Subject: [PATCH 11/51] Not all AddNodes can reassociate so stick to Min/Max for now --- src/hotspot/share/opto/loopnode.cpp | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index c745c69e6d630..6929ded98a346 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4978,7 +4978,7 @@ bool PhaseIdealLoop::process_expensive_nodes() { return progress; } -static AddNode* build_add(int opcode, Node* a, Node* b, PhaseIdealLoop* phase) { +static MinMaxNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phase) { switch (opcode) { case Op_MinI: return new MinINode(a, b); @@ -5004,7 +5004,7 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l Node* left; Node* right; - if (node->in(1)->is_Add()) { + if (node->in(1)->is_MinMax()) { left = reassociate_chain(add_opcode, node->in(1), phi, loop_head, phase); right = node->in(2); } else { @@ -5012,7 +5012,7 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l right = reassociate_chain(add_opcode, node->in(2), phi, loop_head, phase); } - Node* reassoc = build_add(add_opcode, left, right, phase); + Node* reassoc = build_min_max(add_opcode, left, right, phase); phase->register_new_node(reassoc, loop_head); return reassoc; } @@ -5021,7 +5021,7 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo Node* chain_head = nullptr; Node* current = phi; int chain_length = 0; - int add_opcode = 0; + int opcode = 0; while (current != nullptr) { if (current->outcnt() != 1) { break; @@ -5030,9 +5030,9 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo Node* use = nullptr; for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) { Node* n = current->fast_out(i); - if (n->is_Add()) { + if (n->is_MinMax()) { use = n; - add_opcode = use->Opcode(); + opcode = use->Opcode(); break; } } @@ -5042,7 +5042,7 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo // Only interested in commutative add nodes that are in use in the loop return; } - if (use->in(1)->Opcode() == add_opcode && use->in(2)->Opcode() == add_opcode) { + if (use->in(1)->Opcode() == opcode && use->in(2)->Opcode() == opcode) { // A chain to reassociate cannot be constructed // when the chain can have multiple paths return; @@ -5068,9 +5068,9 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo // chain_head->dump(); Node* loop_head = lpt->head(); - Node* reassociated = reassociate_chain(add_opcode, chain_head, phi, loop_head, phase); + Node* reassociated = reassociate_chain(opcode, chain_head, phi, loop_head, phase); - Node* new_chain_head = build_add(add_opcode, phi, reassociated, phase); + Node* new_chain_head = build_min_max(opcode, phi, reassociated, phase); phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); } From 959fc49ba7e37976ad8b5f7d8bd4180def7fb071 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Mon, 12 Jan 2026 17:41:15 +0100 Subject: [PATCH 12/51] Expand to Min/Max Float --- src/hotspot/share/opto/loopnode.cpp | 4 ++++ .../compiler/loopopts/TestReductionReassociation.java | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 6929ded98a346..ee4eeb1d867f1 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4980,6 +4980,10 @@ bool PhaseIdealLoop::process_expensive_nodes() { static MinMaxNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phase) { switch (opcode) { + case Op_MinF: + return new MinFNode(a, b); + case Op_MaxF: + return new MaxFNode(a, b); case Op_MinI: return new MinINode(a, b); case Op_MaxI: diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 5f05a91119912..3677739384754 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -96,6 +96,8 @@ public static String generate(CompileFramework comp) { } enum AddOp { + MIN_F(CodeGenerationDataNameType.floats()), + MAX_F(CodeGenerationDataNameType.floats()), MIN_I(CodeGenerationDataNameType.ints()), MAX_I(CodeGenerationDataNameType.ints()), MIN_L(CodeGenerationDataNameType.longs()), @@ -157,6 +159,8 @@ private TemplateToken generateOp(String a, String b) { let("a", a), let("b", b), switch (add) { + case MIN_F -> "Float.min(#a, #b)"; + case MAX_F -> "Float.max(#a, #b)"; case MIN_I -> "Integer.min(#a, #b)"; case MAX_I -> "Integer.max(#a, #b)"; case MIN_L -> "Long.min(#a, #b)"; @@ -173,8 +177,8 @@ private TemplateToken generateResultInit(String resultName) { let("type", add.type.name()), "#type ", resultName, " = #boxedType.", switch (add) { - case MIN_I, MIN_L -> "MAX_VALUE"; - case MAX_I, MAX_L -> "MIN_VALUE"; + case MIN_F, MIN_I, MIN_L -> "MAX_VALUE"; + case MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; }, ";\n" )); From fd56d4c712f2bc0c0482818fda0fed4aa5f31144 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Mon, 12 Jan 2026 18:00:45 +0100 Subject: [PATCH 13/51] Expand to Min/Max for doubles --- src/hotspot/share/opto/loopnode.cpp | 4 ++++ .../compiler/loopopts/TestReductionReassociation.java | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index ee4eeb1d867f1..ec7185fa1abbf 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4980,6 +4980,10 @@ bool PhaseIdealLoop::process_expensive_nodes() { static MinMaxNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phase) { switch (opcode) { + case Op_MinD: + return new MinDNode(a, b); + case Op_MaxD: + return new MaxDNode(a, b); case Op_MinF: return new MinFNode(a, b); case Op_MaxF: diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 3677739384754..0e4f6a734f61a 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -96,6 +96,8 @@ public static String generate(CompileFramework comp) { } enum AddOp { + MIN_D(CodeGenerationDataNameType.doubles()), + MAX_D(CodeGenerationDataNameType.doubles()), MIN_F(CodeGenerationDataNameType.floats()), MAX_F(CodeGenerationDataNameType.floats()), MIN_I(CodeGenerationDataNameType.ints()), @@ -159,6 +161,8 @@ private TemplateToken generateOp(String a, String b) { let("a", a), let("b", b), switch (add) { + case MIN_D -> "Double.min(#a, #b)"; + case MAX_D -> "Double.max(#a, #b)"; case MIN_F -> "Float.min(#a, #b)"; case MAX_F -> "Float.max(#a, #b)"; case MIN_I -> "Integer.min(#a, #b)"; @@ -177,8 +181,8 @@ private TemplateToken generateResultInit(String resultName) { let("type", add.type.name()), "#type ", resultName, " = #boxedType.", switch (add) { - case MIN_F, MIN_I, MIN_L -> "MAX_VALUE"; - case MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; + case MIN_D, MIN_F, MIN_I, MIN_L -> "MAX_VALUE"; + case MAX_D, MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; }, ";\n" )); From 538ac8ab2f5851374653aec75397e99437e09816 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Mon, 12 Jan 2026 18:45:25 +0100 Subject: [PATCH 14/51] Expand to Float16 --- src/hotspot/share/opto/loopnode.cpp | 4 ++ .../loopopts/TestReductionReassociation.java | 63 ++++++++++++++----- 2 files changed, 52 insertions(+), 15 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index ec7185fa1abbf..e2617d67479a4 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4988,6 +4988,10 @@ static MinMaxNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* p return new MinFNode(a, b); case Op_MaxF: return new MaxFNode(a, b); + case Op_MinHF: + return new MinHFNode(a, b); + case Op_MaxHF: + return new MaxHFNode(a, b); case Op_MinI: return new MinINode(a, b); case Op_MaxI: diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 0e4f6a734f61a..ded7c89edfc60 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -31,24 +31,20 @@ package compiler.loopopts; -import compiler.igvn.TestMinMaxIdeal; import compiler.lib.compile_framework.CompileFramework; -import compiler.lib.generators.Generator; -import compiler.lib.ir_framework.*; import compiler.lib.template_framework.Template; import compiler.lib.template_framework.TemplateToken; import compiler.lib.template_framework.library.CodeGenerationDataNameType; import compiler.lib.template_framework.library.PrimitiveType; import compiler.lib.template_framework.library.TestFrameworkClass; -import compiler.lib.verify.Verify; import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.Set; import java.util.stream.IntStream; import java.util.stream.Stream; -import static compiler.lib.generators.Generators.G; import static compiler.lib.template_framework.Template.*; import static compiler.lib.template_framework.Template.let; @@ -61,9 +57,12 @@ public static void main(String[] args) { comp.addJavaSourceCode("compiler.loopopts.templated.ReductionReassociation", generate(comp)); // Compile the source file. - comp.compile(); + comp.compile("--add-modules=jdk.incubator.vector"); - String[] flags = new String[] {"-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:VerifyIterativeGVN=1000"}; + String[] flags = new String[] { + "-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:VerifyIterativeGVN=1000", + "--add-modules=jdk.incubator.vector", "--add-opens", "jdk.incubator.vector/jdk.incubator.vector=ALL-UNNAMED" + }; comp.invoke("compiler.loopopts.templated.ReductionReassociation", "main", new Object[] {flags}); } @@ -87,7 +86,8 @@ public static String generate(CompileFramework comp) { // package and class name. "compiler.loopopts.templated", "ReductionReassociation", // List of imports. - Set.of("compiler.lib.generators.*", + Set.of("jdk.incubator.vector.Float16", + "compiler.lib.generators.*", "compiler.lib.verify.*"), // classpath, so the Test VM has access to the compiled class files. comp.getEscapedClassPathOfCompiledClasses(), @@ -98,6 +98,8 @@ public static String generate(CompileFramework comp) { enum AddOp { MIN_D(CodeGenerationDataNameType.doubles()), MAX_D(CodeGenerationDataNameType.doubles()), + MIN_HF(CodeGenerationDataNameType.float16()), + MAX_HF(CodeGenerationDataNameType.float16()), MIN_F(CodeGenerationDataNameType.floats()), MAX_F(CodeGenerationDataNameType.floats()), MIN_I(CodeGenerationDataNameType.ints()), @@ -105,11 +107,15 @@ enum AddOp { MIN_L(CodeGenerationDataNameType.longs()), MAX_L(CodeGenerationDataNameType.longs()); - final PrimitiveType type; + final CodeGenerationDataNameType type; - AddOp(PrimitiveType type) { + AddOp(CodeGenerationDataNameType type) { this.type = type; } + + boolean isFloat16() { + return MIN_HF == this || MAX_HF == this; + } } record TestGenerator(AddOp add, int batchSize, boolean useIntermediate, int size) { @@ -163,6 +169,8 @@ private TemplateToken generateOp(String a, String b) { switch (add) { case MIN_D -> "Double.min(#a, #b)"; case MAX_D -> "Double.max(#a, #b)"; + case MIN_HF -> "Float16.min(#a, #b)"; + case MAX_HF -> "Float16.max(#a, #b)"; case MIN_F -> "Float.min(#a, #b)"; case MAX_F -> "Float.max(#a, #b)"; case MIN_I -> "Integer.min(#a, #b)"; @@ -177,18 +185,26 @@ private TemplateToken generateOp(String a, String b) { private TemplateToken generateResultInit(String resultName) { var template = Template.make(() -> scope( let("resultName", resultName), - let("boxedType", add.type.boxedTypeName()), + let("boxedType", getBoxedTypeName()), let("type", add.type.name()), "#type ", resultName, " = #boxedType.", switch (add) { - case MIN_D, MIN_F, MIN_I, MIN_L -> "MAX_VALUE"; - case MAX_D, MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; + case MIN_HF, MIN_D, MIN_F, MIN_I, MIN_L -> "MAX_VALUE"; + case MAX_HF, MAX_D, MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; }, ";\n" )); return template.asToken(); } + private Object getBoxedTypeName() { + if (add.type instanceof PrimitiveType primitiveType) { + return primitiveType.boxedTypeName(); + } + + return add.type.name(); + } + private TemplateToken generateTest(String input, String setup, String test) { var template = Template.make(() -> scope( let("irNodeName", add.name()), @@ -249,11 +265,28 @@ private TemplateToken generateArrayField(String input) { let("size", size), let("input", input), let("type", add.type.name()), - let("gen", add.type.name() + "s"), + let("gen", add.type.name().toLowerCase(Locale.ROOT) + "s"), """ private static #type[] #input = new #type[#size]; + """, + add.isFloat16() ? + """ + private static final Generator GEN_#input = Generators.G.float16s(); + + static void fill_#input(Float16[] a) { + for (int i = 0; i < a.length; i++) { + a[i] = Float16.shortBitsToFloat16(GEN_#input.next()); + } + } + """ + : "", + """ static { - Generators.G.fill(Generators.G.#gen(), #input); + """, + add.isFloat16() ? + "fill_#input(#input);" + : "Generators.G.fill(Generators.G.#gen(), #input);", + """ } """ )); From f7ca35cb0649f364f9b1e25a44693c269ff11be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 13 Jan 2026 06:32:37 +0100 Subject: [PATCH 15/51] Add IR expectations for Float16 --- .../loopopts/TestReductionReassociation.java | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index ded7c89edfc60..67dc0e0859080 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -72,14 +72,14 @@ public static String generate(CompileFramework comp) { final int size = 10_000; final int batchSize = 4; - Stream.of(TestReductionReassociation.AddOp.values()) + Stream.of(AssociativeAdd.values()) .map(op -> new TestGenerator(op, batchSize, false, size).generate()) .forEach(testTemplateTokens::add); // A single test to test a non-power-of-2 value - testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, 5, false, size).generate()); + testTemplateTokens.add(new TestGenerator(AssociativeAdd.MAX_L, 5, false, size).generate()); // A single test where an intermediate value is used some other way - testTemplateTokens.add(new TestGenerator(AddOp.MAX_L, batchSize, true, size).generate()); + testTemplateTokens.add(new TestGenerator(AssociativeAdd.MAX_L, batchSize, true, size).generate()); // Create the test class, which runs all testTemplateTokens. return TestFrameworkClass.render( @@ -95,7 +95,7 @@ public static String generate(CompileFramework comp) { testTemplateTokens); } - enum AddOp { + enum AssociativeAdd { MIN_D(CodeGenerationDataNameType.doubles()), MAX_D(CodeGenerationDataNameType.doubles()), MIN_HF(CodeGenerationDataNameType.float16()), @@ -109,7 +109,7 @@ enum AddOp { final CodeGenerationDataNameType type; - AddOp(CodeGenerationDataNameType type) { + AssociativeAdd(CodeGenerationDataNameType type) { this.type = type; } @@ -118,7 +118,11 @@ boolean isFloat16() { } } - record TestGenerator(AddOp add, int batchSize, boolean useIntermediate, int size) { + record TestGenerator(int countsIR, AssociativeAdd add, int batchSize, boolean useIntermediate, int size) { + TestGenerator(AssociativeAdd add, int batchSize, boolean useIntermediate, int size) { + this(useIntermediate ? batchSize * 2 : batchSize, add, batchSize, useIntermediate, size); + } + public TemplateToken generate() { var testTemplate = Template.make(() -> { String test = $("test"); @@ -207,6 +211,7 @@ private Object getBoxedTypeName() { private TemplateToken generateTest(String input, String setup, String test) { var template = Template.make(() -> scope( + let("countsIR", countsIR), let("irNodeName", add.name()), let("input", input), let("setup", setup), @@ -215,9 +220,20 @@ private TemplateToken generateTest(String input, String setup, String test) { """ @Test """, - "@IR(counts = {IRNode.#irNodeName, \"= ", - useIntermediate ? batchSize * 2 : batchSize, - "\"}, phase = CompilePhase.AFTER_LOOP_OPTS)", + add.isFloat16() ? + """ + @IR(counts = {IRNode.#irNodeName, "= #countsIR"}, + phase = CompilePhase.AFTER_LOOP_OPTS, + applyIfCPUFeatureOr = {"avx512_fp16", "true", "zfh", "true"}) + @IR(counts = {IRNode.#irNodeName, "= #countsIR"}, + phase = CompilePhase.AFTER_LOOP_OPTS, + applyIfCPUFeatureAnd = {"fphp", "true", "asimdhp", "true"}) + """ + : + """ + @IR(counts = {IRNode.#irNodeName, "= #countsIR"}, + phase = CompilePhase.AFTER_LOOP_OPTS) + """, """ public Object[] #test() { """, From 2c085deb194142bd3804d37fbbacfdccb9726c44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 13 Jan 2026 15:08:24 +0100 Subject: [PATCH 16/51] Comment/revert Float16 changes, has a different IR shape --- src/hotspot/share/opto/loopnode.cpp | 8 +-- .../loopopts/TestReductionReassociation.java | 52 ++++--------------- 2 files changed, 14 insertions(+), 46 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index e2617d67479a4..1d8002e452593 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4988,10 +4988,10 @@ static MinMaxNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* p return new MinFNode(a, b); case Op_MaxF: return new MaxFNode(a, b); - case Op_MinHF: - return new MinHFNode(a, b); - case Op_MaxHF: - return new MaxHFNode(a, b); + // case Op_MinHF: + // return new MinHFNode(a, b); + // case Op_MaxHF: + // return new MaxHFNode(a, b); case Op_MinI: return new MinINode(a, b); case Op_MaxI: diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 67dc0e0859080..9e04af7078a50 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -98,8 +98,8 @@ public static String generate(CompileFramework comp) { enum AssociativeAdd { MIN_D(CodeGenerationDataNameType.doubles()), MAX_D(CodeGenerationDataNameType.doubles()), - MIN_HF(CodeGenerationDataNameType.float16()), - MAX_HF(CodeGenerationDataNameType.float16()), +// MIN_HF(CodeGenerationDataNameType.float16()), +// MAX_HF(CodeGenerationDataNameType.float16()), MIN_F(CodeGenerationDataNameType.floats()), MAX_F(CodeGenerationDataNameType.floats()), MIN_I(CodeGenerationDataNameType.ints()), @@ -113,9 +113,9 @@ enum AssociativeAdd { this.type = type; } - boolean isFloat16() { - return MIN_HF == this || MAX_HF == this; - } +// boolean isFloat16() { +// return MIN_HF == this || MAX_HF == this; +// } } record TestGenerator(int countsIR, AssociativeAdd add, int batchSize, boolean useIntermediate, int size) { @@ -173,8 +173,6 @@ private TemplateToken generateOp(String a, String b) { switch (add) { case MIN_D -> "Double.min(#a, #b)"; case MAX_D -> "Double.max(#a, #b)"; - case MIN_HF -> "Float16.min(#a, #b)"; - case MAX_HF -> "Float16.max(#a, #b)"; case MIN_F -> "Float.min(#a, #b)"; case MAX_F -> "Float.max(#a, #b)"; case MIN_I -> "Integer.min(#a, #b)"; @@ -193,8 +191,8 @@ private TemplateToken generateResultInit(String resultName) { let("type", add.type.name()), "#type ", resultName, " = #boxedType.", switch (add) { - case MIN_HF, MIN_D, MIN_F, MIN_I, MIN_L -> "MAX_VALUE"; - case MAX_HF, MAX_D, MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; + case MIN_D, MIN_F, MIN_I, MIN_L -> "MAX_VALUE"; + case MAX_D, MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; }, ";\n" )); @@ -219,22 +217,8 @@ private TemplateToken generateTest(String input, String setup, String test) { let("type", add.type.name()), """ @Test - """, - add.isFloat16() ? - """ - @IR(counts = {IRNode.#irNodeName, "= #countsIR"}, - phase = CompilePhase.AFTER_LOOP_OPTS, - applyIfCPUFeatureOr = {"avx512_fp16", "true", "zfh", "true"}) - @IR(counts = {IRNode.#irNodeName, "= #countsIR"}, - phase = CompilePhase.AFTER_LOOP_OPTS, - applyIfCPUFeatureAnd = {"fphp", "true", "asimdhp", "true"}) - """ - : - """ - @IR(counts = {IRNode.#irNodeName, "= #countsIR"}, - phase = CompilePhase.AFTER_LOOP_OPTS) - """, - """ + @IR(counts = {IRNode.#irNodeName, "= #countsIR"}, + phase = CompilePhase.AFTER_LOOP_OPTS) public Object[] #test() { """, generateResultInit("result"), @@ -284,25 +268,9 @@ private TemplateToken generateArrayField(String input) { let("gen", add.type.name().toLowerCase(Locale.ROOT) + "s"), """ private static #type[] #input = new #type[#size]; - """, - add.isFloat16() ? - """ - private static final Generator GEN_#input = Generators.G.float16s(); - static void fill_#input(Float16[] a) { - for (int i = 0; i < a.length; i++) { - a[i] = Float16.shortBitsToFloat16(GEN_#input.next()); - } - } - """ - : "", - """ static { - """, - add.isFloat16() ? - "fill_#input(#input);" - : "Generators.G.fill(Generators.G.#gen(), #input);", - """ + Generators.G.fill(Generators.G.#gen(), #input); } """ )); From 26f9f819653fb8f345e081e07d377fc9d1b5f08a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Thu, 15 Jan 2026 09:08:55 +0100 Subject: [PATCH 17/51] Add support for AddI/AddL --- src/hotspot/share/opto/loopnode.cpp | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 1d8002e452593..9189ac6d120b8 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4978,8 +4978,12 @@ bool PhaseIdealLoop::process_expensive_nodes() { return progress; } -static MinMaxNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phase) { +static AddNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phase) { switch (opcode) { + case Op_AddI: + return new AddINode(a, b); + case Op_AddL: + return new AddLNode(a, b); case Op_MinD: return new MinDNode(a, b); case Op_MaxD: @@ -5005,6 +5009,10 @@ static MinMaxNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* p } } +static bool is_associative(Node* node) { + return node->is_MinMax() || node->Opcode() == Op_AddI || node->Opcode() == Op_AddL; +} + static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* loop_head, PhaseIdealLoop* phase) { if (phi == node->in(1)) { return node->in(2); @@ -5016,7 +5024,7 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l Node* left; Node* right; - if (node->in(1)->is_MinMax()) { + if (is_associative(node->in(1))) { left = reassociate_chain(add_opcode, node->in(1), phi, loop_head, phase); right = node->in(2); } else { @@ -5042,7 +5050,7 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo Node* use = nullptr; for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) { Node* n = current->fast_out(i); - if (n->is_MinMax()) { + if (is_associative(n)) { use = n; opcode = use->Opcode(); break; @@ -5072,12 +5080,12 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo return; } - // tty->print("[avoid-cmov] try reassociate; chain length: %d\n", chain_length); - // tty->print("[avoid-cmov] try reassociate:\n"); - // tty->print("[avoid-cmov] phi: "); - // phi->dump(); - // tty->print("[avoid-cmov] try reassociate; chain head:\n"); - // chain_head->dump(); + tty->print("[avoid-cmov] try reassociate; chain length: %d\n", chain_length); + tty->print("[avoid-cmov] try reassociate:\n"); + tty->print("[avoid-cmov] phi: "); + phi->dump(); + tty->print("[avoid-cmov] try reassociate; chain head:\n"); + chain_head->dump(); Node* loop_head = lpt->head(); Node* reassociated = reassociate_chain(opcode, chain_head, phi, loop_head, phase); @@ -5085,6 +5093,9 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo Node* new_chain_head = build_min_max(opcode, phi, reassociated, phase); phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); + + tty->print("[avoid-cmov] after reassociate; new chain head:\n"); + new_chain_head->dump(); } //============================================================================= From f0118c5c71778521696e5d5baa3441f822231512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Thu, 15 Jan 2026 09:09:07 +0100 Subject: [PATCH 18/51] Test with AddL, commented test for AddI --- .../compiler/loopopts/TestReductionReassociation.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 9e04af7078a50..8d1cb953d2217 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -96,6 +96,8 @@ public static String generate(CompileFramework comp) { } enum AssociativeAdd { +// ADD_I(CodeGenerationDataNameType.ints()), + ADD_L(CodeGenerationDataNameType.longs()), MIN_D(CodeGenerationDataNameType.doubles()), MAX_D(CodeGenerationDataNameType.doubles()), // MIN_HF(CodeGenerationDataNameType.float16()), @@ -159,7 +161,7 @@ private TemplateToken generateCheck(String test, String check, String expected) public void #check(Object[] results) { Verify.checkEQ(#expected[0], results[0]); Verify.checkEQ(#expected[1], results[1]); - Verify.checkEQ(results[0], results[1]); + // Verify.checkEQ(results[0], results[1]); } """ )); @@ -171,6 +173,7 @@ private TemplateToken generateOp(String a, String b) { let("a", a), let("b", b), switch (add) { + case ADD_L -> "#a + #b"; case MIN_D -> "Double.min(#a, #b)"; case MAX_D -> "Double.max(#a, #b)"; case MIN_F -> "Float.min(#a, #b)"; @@ -192,7 +195,7 @@ private TemplateToken generateResultInit(String resultName) { "#type ", resultName, " = #boxedType.", switch (add) { case MIN_D, MIN_F, MIN_I, MIN_L -> "MAX_VALUE"; - case MAX_D, MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; + case ADD_L, MAX_D, MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; }, ";\n" )); From b2b8ae2053eab0398a1f6db93bcfc9a38141d0bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Mon, 19 Jan 2026 11:00:11 +0100 Subject: [PATCH 19/51] Phi with more than one output not enough, new approach needed --- src/hotspot/share/opto/loopnode.cpp | 22 ++++- .../loopopts/TestReductionReassociation.java | 2 +- .../loopopts/TestReductionReassociation2.java | 87 +++++++++++++++++ .../loopopts/TestReductionReassociation3.java | 94 +++++++++++++++++++ 4 files changed, 201 insertions(+), 4 deletions(-) create mode 100644 test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation2.java create mode 100644 test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation3.java diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 9189ac6d120b8..b4b241e020684 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5033,15 +5033,32 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l } Node* reassoc = build_min_max(add_opcode, left, right, phase); + // tty->print("[avoid-cmov] reassociate_chain: new node:"); + // reassoc->dump(); + phase->register_new_node(reassoc, loop_head); return reassoc; } static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLoop* phase) { Node* chain_head = nullptr; - Node* current = phi; - int chain_length = 0; + Node* current = nullptr; int opcode = 0; + for (DUIterator_Fast imax, i = phi->fast_outs(imax); i < imax; i++) { + Node* n = phi->fast_out(i); + if (is_associative(n)) { + if (current == nullptr) { + current = n; + opcode = current->Opcode(); + } else { + // Reassociation only supported when the Phi has a single + // associative operation as output that it can search the chain through + return; + } + } + } + + int chain_length = 1; while (current != nullptr) { if (current->outcnt() != 1) { break; @@ -5052,7 +5069,6 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo Node* n = current->fast_out(i); if (is_associative(n)) { use = n; - opcode = use->Opcode(); break; } } diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 8d1cb953d2217..77578021d84b1 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -161,7 +161,7 @@ private TemplateToken generateCheck(String test, String check, String expected) public void #check(Object[] results) { Verify.checkEQ(#expected[0], results[0]); Verify.checkEQ(#expected[1], results[1]); - // Verify.checkEQ(results[0], results[1]); + Verify.checkEQ(results[0], results[1]); } """ )); diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation2.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation2.java new file mode 100644 index 0000000000000..a6f2920d478b3 --- /dev/null +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation2.java @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2025 IBM Corporation. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @bug 8351409 + * @summary Test the IR effects of reduction reassociation + * @library /test/lib / + * @run driver compiler.loopopts.TestReductionReassociation2 + */ + +package compiler.loopopts; + +import compiler.lib.generators.Generators; +import compiler.lib.ir_framework.*; +import compiler.lib.verify.Verify; + +public class TestReductionReassociation2 { + private static long[] input_2 = new long[10000]; + + static { + Generators.G.fill(Generators.G.longs(), input_2); + } + + private Object[] expected_2 = test_2(); + + public static void main(String[] args) { + TestFramework.runWithFlags("-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:CompileCommand=dontinline,*::*dontinline*", "-XX:VerifyIterativeGVN=1000"); + } + + @Test + @IR( + counts = {IRNode.ADD_L, "= 4"}, + phase = CompilePhase.AFTER_LOOP_OPTS) + public Object[] test_2() { + long result = Long.MIN_VALUE; + long result2 = Long.MIN_VALUE; + for (int i = 0; i < input_2.length; i += 4) { + long v0 = getArrayLong_dontinline(i + 0); + long v1 = getArrayLong_dontinline(i + 1); + long v2 = getArrayLong_dontinline(i + 2); + long v3 = getArrayLong_dontinline(i + 3); + long u0 = v0 + result; + long u1 = v1 + u0; + long u2 = v2 + u1; + long u3 = v3 + u2; + long t0 = v0 + v1; + long t1 = v2 + t0; + long t2 = v3 + t1; + long t3 = result + t2; + result = u3; + result2 = t3; + } + return new Object[] {result, result2}; + } + + private static long getArrayLong_dontinline(int i) { + return input_2[i]; + } + + @Check(test = "test_2") + public void check_2(Object[] results) { + Verify.checkEQ(expected_2[0], results[0]); + Verify.checkEQ(expected_2[1], results[1]); + Verify.checkEQ(results[0], results[1]); + } +} \ No newline at end of file diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation3.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation3.java new file mode 100644 index 0000000000000..ff3339d97e322 --- /dev/null +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation3.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2025 IBM Corporation. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @bug 8351409 + * @summary Test the IR effects of reduction reassociation + * @library /test/lib / + * @run driver compiler.loopopts.TestReductionReassociation3 + */ + +package compiler.loopopts; + +import compiler.lib.generators.Generators; +import compiler.lib.ir_framework.Check; +import compiler.lib.ir_framework.CompilePhase; +import compiler.lib.ir_framework.IR; +import compiler.lib.ir_framework.IRNode; +import compiler.lib.ir_framework.Test; +import compiler.lib.ir_framework.TestFramework; +import compiler.lib.verify.Verify; + +public class TestReductionReassociation3 { + private static long[] input_122 = new long[10000]; + + static { + Generators.G.fill(Generators.G.longs(), input_122); + } + + private Object[] expected_122 = test_122(); + + public static void main(String[] args) { + TestFramework.runWithFlags("-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:CompileCommand=dontinline,*::*dontinline*", "-XX:VerifyIterativeGVN=1000"); + } + + @Test + @IR( + counts = {IRNode.MAX_L, "= 4"}, + phase = CompilePhase.AFTER_LOOP_OPTS) + public Object[] test_122() { + // i = max(a, max(b, max(c, max(d, i)))) + // i' = max(i, max(a, max(b, max(c, d)))) + + // result = max(v3, max(v2, max(v1, max(v0, result)))) + // result2 = max(result, max(v3, max(v2, max(v1, v0)))) + + long result = Long.MIN_VALUE; + long result2 = Long.MIN_VALUE; + for (int i = 0; i < input_122.length; i += 4) { + long v0 = input_122[i + 0]; + long v1 = input_122[i + 1]; + long v2 = input_122[i + 2]; + long v3 = input_122[i + 3]; + long u0 = Long.max(v0, result); + long u1 = Long.max(v1, u0); + long u2 = Long.max(v2, u1); + long u3 = Long.max(v3, u2); + long t0 = Long.max(v0, v1); + long t1 = Long.max(v2, t0); + long t2 = Long.max(v3, t1); + long t3 = Long.max(result, t2); + result = u3; + result2 = t3; + } + return new Object[] {result, result2}; + } + + @Check(test = "test_122") + public void check_122(Object[] results) { + Verify.checkEQ(expected_122[0], results[0]); + Verify.checkEQ(expected_122[1], results[1]); + Verify.checkEQ(results[0], results[1]); + } +} \ No newline at end of file From 7b336c24b54d63d74df9b5c53ae224b3abeeb009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 20 Jan 2026 09:31:39 +0100 Subject: [PATCH 20/51] Deal with Phi with more than 1 chain, try each --- src/hotspot/share/opto/loopnode.cpp | 49 ++++--- .../loopopts/TestReductionReassociation.java | 128 ++++++++++++++++-- 2 files changed, 137 insertions(+), 40 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index b4b241e020684..353d0acf8abd7 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5040,23 +5040,10 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l return reassoc; } -static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLoop* phase) { +static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLoop* phase) { Node* chain_head = nullptr; - Node* current = nullptr; - int opcode = 0; - for (DUIterator_Fast imax, i = phi->fast_outs(imax); i < imax; i++) { - Node* n = phi->fast_out(i); - if (is_associative(n)) { - if (current == nullptr) { - current = n; - opcode = current->Opcode(); - } else { - // Reassociation only supported when the Phi has a single - // associative operation as output that it can search the chain through - return; - } - } - } + Node* current = n; + int opcode = current->Opcode(); int chain_length = 1; while (current != nullptr) { @@ -5096,12 +5083,12 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo return; } - tty->print("[avoid-cmov] try reassociate; chain length: %d\n", chain_length); - tty->print("[avoid-cmov] try reassociate:\n"); - tty->print("[avoid-cmov] phi: "); - phi->dump(); - tty->print("[avoid-cmov] try reassociate; chain head:\n"); - chain_head->dump(); + // tty->print("[avoid-cmov] try reassociate; chain length: %d\n", chain_length); + // tty->print("[avoid-cmov] try reassociate:\n"); + // tty->print("[avoid-cmov] phi: "); + // phi->dump(); + // tty->print("[avoid-cmov] try reassociate; chain head:\n"); + // chain_head->dump(); Node* loop_head = lpt->head(); Node* reassociated = reassociate_chain(opcode, chain_head, phi, loop_head, phase); @@ -5110,8 +5097,8 @@ static void try_reassociate_chain(PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLo phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); - tty->print("[avoid-cmov] after reassociate; new chain head:\n"); - new_chain_head->dump(); + // tty->print("[avoid-cmov] after reassociate; new chain head:\n"); + // new_chain_head->dump(); } //============================================================================= @@ -5500,7 +5487,19 @@ void PhaseIdealLoop::build_and_optimize() { for (DUIterator_Fast imax, i = loop_head->fast_outs(imax); i < imax; i++) { Node* loop_head_use = loop_head->fast_out(i); if (loop_head_use->is_Phi()) { - try_reassociate_chain(loop_head_use->as_Phi(), lpt, this); + PhiNode* phi = loop_head_use->as_Phi(); + Unique_Node_List wq; + for (DUIterator_Fast jmax, j = phi->fast_outs(jmax); j < jmax; j++) { + Node* n = phi->fast_out(j); + if (is_associative(n)) { + wq.push(n); + } + } + // Reassociating changes Phi nodes, so iteration and reassociation have to be separated + for (uint next = 0; next < wq.size(); ++next) { + Node* m = wq.at(next); + try_reassociate_chain(m, phi, lpt, this); + } } } } diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 77578021d84b1..1ac5eb2c1cd43 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -61,6 +61,7 @@ public static void main(String[] args) { String[] flags = new String[] { "-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:VerifyIterativeGVN=1000", + "-XX:CompileCommand=dontinline,*::*dontinline*", "--add-modules=jdk.incubator.vector", "--add-opens", "jdk.incubator.vector/jdk.incubator.vector=ALL-UNNAMED" }; comp.invoke("compiler.loopopts.templated.ReductionReassociation", "main", new Object[] {flags}); @@ -73,13 +74,13 @@ public static String generate(CompileFramework comp) { final int batchSize = 4; Stream.of(AssociativeAdd.values()) - .map(op -> new TestGenerator(op, batchSize, false, size).generate()) + .map(op -> new TestGenerator(op, batchSize, size).generate()) .forEach(testTemplateTokens::add); // A single test to test a non-power-of-2 value - testTemplateTokens.add(new TestGenerator(AssociativeAdd.MAX_L, 5, false, size).generate()); + testTemplateTokens.add(new TestGenerator(AssociativeAdd.MAX_L, 5, size).generate()); // A single test where an intermediate value is used some other way - testTemplateTokens.add(new TestGenerator(AssociativeAdd.MAX_L, batchSize, true, size).generate()); + testTemplateTokens.add(new TestGenerator(AssociativeAdd.MAX_L, batchSize, size).generateUseIntermediate()); // Create the test class, which runs all testTemplateTokens. return TestFrameworkClass.render( @@ -120,11 +121,7 @@ enum AssociativeAdd { // } } - record TestGenerator(int countsIR, AssociativeAdd add, int batchSize, boolean useIntermediate, int size) { - TestGenerator(AssociativeAdd add, int batchSize, boolean useIntermediate, int size) { - this(useIntermediate ? batchSize * 2 : batchSize, add, batchSize, useIntermediate, size); - } - + record TestGenerator(AssociativeAdd add, int batchSize, int size) { public TemplateToken generate() { var testTemplate = Template.make(() -> { String test = $("test"); @@ -139,9 +136,35 @@ public TemplateToken generate() { """, generateArrayField(input), generateExpectedField(test, expected), - // generateSetup(setup, input), generateTest(input, setup, test), generateCheck(test, check, expected), + add == AssociativeAdd.ADD_L ? generateArrayLoad(input) : "", + """ + + // --- $test end --- + """ + ); + }); + return testTemplate.asToken(); + } + + public TemplateToken generateUseIntermediate() { + var testTemplate = Template.make(() -> { + String test = $("test"); + String input = $("input"); + String expected = $("expected"); + String setup = $("setup"); + String check = $("check"); + return scope( + """ + // --- $test start --- + + """, + generateArrayField(input), + generateExpectedField(test, expected), + generateUseIntermediateTest(input, setup, test), + generateCheck(test, check, expected), + add == AssociativeAdd.ADD_L ? generateArrayLoad(input) : "", """ // --- $test end --- @@ -151,6 +174,75 @@ public TemplateToken generate() { return testTemplate.asToken(); } + private TemplateToken generateUseIntermediateTest(String input, String setup, String test) { + var template = Template.make(() -> scope( + let("irNodeName", add.name()), + let("input", input), + let("setup", setup), + let("test", test), + let("type", add.type.name()), + """ + @Test + @IR(counts = {IRNode.#irNodeName, "= 8"}, + phase = CompilePhase.AFTER_LOOP_OPTS) + public Object[] #test() { + long result = Long.MIN_VALUE; + long result2 = Long.MIN_VALUE; + for (int i = 0; i < #input.length; i += 8) { + var v0 = #input[i + 0]; + var v1 = #input[i + 1]; + var v2 = #input[i + 2]; + var v3 = #input[i + 3]; + var v4 = #input[i + 4]; + var v5 = #input[i + 5]; + var v6 = #input[i + 6]; + var v7 = #input[i + 7]; + var u0 = Math.max(v0, result); + var u1 = Math.max(v1, u0); + var u2 = Math.max(v2, u1); + var u3 = Math.max(v3, u2); + if (u3 == #input.hashCode()) { + System.out.print(""); + } + var u4 = Math.max(v4, u3); + var u5 = Math.max(v5, u4); + var u6 = Math.max(v6, u5); + var u7 = Math.max(v7, u6); + + long t0 = Long.max(v0, v1); + long t1 = Long.max(v2, t0); + long t2 = Long.max(v3, t1); + long t3 = Long.max(result, t2); + if (t3 == #input.hashCode()) { + System.out.print(""); + } + long t4 = Long.max(v4, t3); + long t5 = Long.max(v5, t4); + long t6 = Long.max(v6, t5); + long t7 = Long.max(v7, t6); + result = u7; + result2 = t7; + } + return new Object[]{result, result2}; + } + """ + )); + return template.asToken(); + } + + private TemplateToken generateArrayLoad(String input) { + var template = Template.make(() -> scope( + let("input", input), + let("type", add.type.name()), + """ + static #type getArray_dontinline_#input(int i) { + return #input[i]; + } + """ + )); + return template.asToken(); + } + private TemplateToken generateCheck(String test, String check, String expected) { var template = Template.make(() -> scope( let("test", test), @@ -212,7 +304,7 @@ private Object getBoxedTypeName() { private TemplateToken generateTest(String input, String setup, String test) { var template = Template.make(() -> scope( - let("countsIR", countsIR), + let("countsIR", batchSize), let("irNodeName", add.name()), let("input", input), let("setup", setup), @@ -227,20 +319,17 @@ private TemplateToken generateTest(String input, String setup, String test) { generateResultInit("result"), generateResultInit("result2"), "for (int i = 0; i < #input.length; i += ", batchSize, ") {\n", - IntStream.range(0, batchSize).mapToObj(i -> - List.of("#type v", i, " = #input[i + ", i, "];\n") - ).toList(), - useIntermediate ? List.of("if (v", batchSize - 1," == #input.hashCode()) { System.out.print(\"\"); }\n") : "", + IntStream.range(0, batchSize).mapToObj(this::callArrayLoad).toList(), "#type u0 = ", generateOp("v0", "result"), ";\n", IntStream.range(1, batchSize).mapToObj(i -> List.of("#type u", i, " = ", generateOp("v" + i, "u" + (i - 1)), ";\n") ).toList(), - "result = u", batchSize - 1,";\n", "#type t0 = ", generateOp("v0", "v1"), ";\n", IntStream.range(1, batchSize - 1).mapToObj(i -> List.of("#type t", i, " = ", generateOp("v" + (i + 1), "t" + (i - 1)), ";\n") ).toList(), "#type t", batchSize - 1, " = ", generateOp("result", "t" + (batchSize - 2)), ";\n", + "result = u", batchSize - 1,";\n", "result2 = t", batchSize - 1,";\n", """ } @@ -251,6 +340,15 @@ private TemplateToken generateTest(String input, String setup, String test) { return template.asToken(); } + private Object callArrayLoad(int i) { + final List result = new ArrayList<>(List.of("#type v", i)); + switch (add) { + case ADD_L -> result.addAll(List.of(" = getArray_dontinline_#input(i + ", i, ");\n")); + default -> result.addAll(List.of(" = #input[i + ", i, "];\n")); + }; + return result; + } + private TemplateToken generateExpectedField(String test, String expected) { var template = Template.make(() -> scope( let("size", size), From a02478ca2412b26c42fad0339427deeb56ee2f88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 20 Jan 2026 11:08:11 +0100 Subject: [PATCH 21/51] Use auxiliary methods throughout that don't expose Add nodes --- .../loopopts/TestReductionReassociation.java | 85 ++++++++++--------- 1 file changed, 46 insertions(+), 39 deletions(-) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 1ac5eb2c1cd43..37bd49c0f2365 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -138,7 +138,7 @@ public TemplateToken generate() { generateExpectedField(test, expected), generateTest(input, setup, test), generateCheck(test, check, expected), - add == AssociativeAdd.ADD_L ? generateArrayLoad(input) : "", + generateAuxMethods(input), """ // --- $test end --- @@ -164,7 +164,7 @@ public TemplateToken generateUseIntermediate() { generateExpectedField(test, expected), generateUseIntermediateTest(input, setup, test), generateCheck(test, check, expected), - add == AssociativeAdd.ADD_L ? generateArrayLoad(input) : "", + generateAuxMethods(input), """ // --- $test end --- @@ -188,15 +188,16 @@ private TemplateToken generateUseIntermediateTest(String input, String setup, St public Object[] #test() { long result = Long.MIN_VALUE; long result2 = Long.MIN_VALUE; - for (int i = 0; i < #input.length; i += 8) { - var v0 = #input[i + 0]; - var v1 = #input[i + 1]; - var v2 = #input[i + 2]; - var v3 = #input[i + 3]; - var v4 = #input[i + 4]; - var v5 = #input[i + 5]; - var v6 = #input[i + 6]; - var v7 = #input[i + 7]; + int i = 0; + while (i < #input.length) { + var v0 = getArray_dontinline_#input(0, i); + var v1 = getArray_dontinline_#input(1, i); + var v2 = getArray_dontinline_#input(2, i); + var v3 = getArray_dontinline_#input(3, i); + var v4 = getArray_dontinline_#input(4, i); + var v5 = getArray_dontinline_#input(5, i); + var v6 = getArray_dontinline_#input(6, i); + var v7 = getArray_dontinline_#input(7, i); var u0 = Math.max(v0, result); var u1 = Math.max(v1, u0); var u2 = Math.max(v2, u1); @@ -222,21 +223,30 @@ private TemplateToken generateUseIntermediateTest(String input, String setup, St long t7 = Long.max(v7, t6); result = u7; result2 = t7; + i = sum_dontinline_#input(i, 8); } - return new Object[]{result, result2}; + return asArray_dontinline_#input(result, result2); } """ )); return template.asToken(); } - private TemplateToken generateArrayLoad(String input) { + private TemplateToken generateAuxMethods(String input) { var template = Template.make(() -> scope( let("input", input), let("type", add.type.name()), """ - static #type getArray_dontinline_#input(int i) { - return #input[i]; + static #type getArray_dontinline_#input(int pos, int base) { + return #input[pos + base]; + } + + static Object[] asArray_dontinline_#input(#type result, #type result2) { + return new Object[]{result, result2}; + } + + static int sum_dontinline_#input(int a, int b) { + return a + b; } """ )); @@ -318,37 +328,34 @@ private TemplateToken generateTest(String input, String setup, String test) { """, generateResultInit("result"), generateResultInit("result2"), - "for (int i = 0; i < #input.length; i += ", batchSize, ") {\n", - IntStream.range(0, batchSize).mapToObj(this::callArrayLoad).toList(), - "#type u0 = ", generateOp("v0", "result"), ";\n", - IntStream.range(1, batchSize).mapToObj(i -> - List.of("#type u", i, " = ", generateOp("v" + i, "u" + (i - 1)), ";\n") - ).toList(), - "#type t0 = ", generateOp("v0", "v1"), ";\n", - IntStream.range(1, batchSize - 1).mapToObj(i -> - List.of("#type t", i, " = ", generateOp("v" + (i + 1), "t" + (i - 1)), ";\n") - ).toList(), - "#type t", batchSize - 1, " = ", generateOp("result", "t" + (batchSize - 2)), ";\n", - "result = u", batchSize - 1,";\n", - "result2 = t", batchSize - 1,";\n", """ - } - return new Object[]{result, result2}; + int i = 0; + while (i < #input.length) { + """, + IntStream.range(0, batchSize).mapToObj(i -> + List.of("#type v", i, " = getArray_dontinline_#input(", i, ", i);\n")).toList(), + "#type u0 = ", generateOp("v0", "result"), ";\n", + IntStream.range(1, batchSize).mapToObj(i -> + List.of("#type u", i, " = ", generateOp("v" + i, "u" + (i - 1)), ";\n") + ).toList(), + "#type t0 = ", generateOp("v0", "v1"), ";\n", + IntStream.range(1, batchSize - 1).mapToObj(i -> + List.of("#type t", i, " = ", generateOp("v" + (i + 1), "t" + (i - 1)), ";\n") + ).toList(), + "#type t", batchSize - 1, " = ", generateOp("result", "t" + (batchSize - 2)), ";\n", + "result = u", batchSize - 1,";\n", + "result2 = t", batchSize - 1,";\n", + "", + """ + i = sum_dontinline_#input(i, #countsIR); + } + return asArray_dontinline_#input(result, result2); } """ )); return template.asToken(); } - private Object callArrayLoad(int i) { - final List result = new ArrayList<>(List.of("#type v", i)); - switch (add) { - case ADD_L -> result.addAll(List.of(" = getArray_dontinline_#input(i + ", i, ");\n")); - default -> result.addAll(List.of(" = #input[i + ", i, "];\n")); - }; - return result; - } - private TemplateToken generateExpectedField(String test, String expected) { var template = Template.make(() -> scope( let("size", size), From 5bed557a7b2d5c95b51350ca7323f7f002b6a73a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 20 Jan 2026 11:09:32 +0100 Subject: [PATCH 22/51] Test passing for AddI --- .../jtreg/compiler/loopopts/TestReductionReassociation.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 37bd49c0f2365..ccaa48d9286d5 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -97,7 +97,7 @@ public static String generate(CompileFramework comp) { } enum AssociativeAdd { -// ADD_I(CodeGenerationDataNameType.ints()), + ADD_I(CodeGenerationDataNameType.ints()), ADD_L(CodeGenerationDataNameType.longs()), MIN_D(CodeGenerationDataNameType.doubles()), MAX_D(CodeGenerationDataNameType.doubles()), @@ -275,7 +275,7 @@ private TemplateToken generateOp(String a, String b) { let("a", a), let("b", b), switch (add) { - case ADD_L -> "#a + #b"; + case ADD_I, ADD_L -> "#a + #b"; case MIN_D -> "Double.min(#a, #b)"; case MAX_D -> "Double.max(#a, #b)"; case MIN_F -> "Float.min(#a, #b)"; @@ -297,7 +297,7 @@ private TemplateToken generateResultInit(String resultName) { "#type ", resultName, " = #boxedType.", switch (add) { case MIN_D, MIN_F, MIN_I, MIN_L -> "MAX_VALUE"; - case ADD_L, MAX_D, MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; + case ADD_I, ADD_L, MAX_D, MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; }, ";\n" )); From 510223ecc39386e94c31dbdee972ff7356c352b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 20 Jan 2026 11:09:40 +0100 Subject: [PATCH 23/51] Auxiliary test, to be removed in the end --- .../loopopts/TestReductionReassociation4.java | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation4.java diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation4.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation4.java new file mode 100644 index 0000000000000..60ffb4649b0e5 --- /dev/null +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation4.java @@ -0,0 +1,102 @@ +/* + * Copyright (c) 2025 IBM Corporation. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @bug 8351409 + * @summary Test the IR effects of reduction reassociation + * @library /test/lib / + * @run driver compiler.loopopts.TestReductionReassociation4 + */ + +package compiler.loopopts; + +import compiler.lib.generators.Generators; +import compiler.lib.ir_framework.Check; +import compiler.lib.ir_framework.CompilePhase; +import compiler.lib.ir_framework.IR; +import compiler.lib.ir_framework.IRNode; +import compiler.lib.ir_framework.Test; +import compiler.lib.ir_framework.TestFramework; +import compiler.lib.verify.Verify; + +public class TestReductionReassociation4 { + private static int[] input_2 = new int[10000]; + + static { + Generators.G.fill(Generators.G.ints(), input_2); + } + + private Object[] expected_2 = test_2(); + + public static void main(String[] args) { + TestFramework.runWithFlags("-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:CompileCommand=dontinline,*::*dontinline*", "-XX:VerifyIterativeGVN=1000"); + } + + @Test + @IR( + counts = {IRNode.ADD_I, "= 4"}, + phase = CompilePhase.AFTER_LOOP_OPTS) + public Object[] test_2() { + int result = Integer.MIN_VALUE; + int result2 = Integer.MIN_VALUE; + int i = 0; + while (i < input_2.length) { + int v0 = getArrayInt_dontinline(0, i); + int v1 = getArrayInt_dontinline(1, i); + int v2 = getArrayInt_dontinline(2, i); + int v3 = getArrayInt_dontinline(3, i); + int u0 = v0 + result; + int u1 = v1 + u0; + int u2 = v2 + u1; + int u3 = v3 + u2; + int t0 = v0 + v1; + int t1 = v2 + t0; + int t2 = v3 + t1; + int t3 = result + t2; + result = u3; + result2 = t3; + i = sum_dontinline(i, 4); + } + return asArray_dontinline(result, result2); + } + + private static Object[] asArray_dontinline(int result, int result2) { + return new Object[]{result, result2}; + } + + private static int sum_dontinline(int a, int b) { + return a + b; + } + + private static int getArrayInt_dontinline(int pos, int base) { + return input_2[pos + base]; + } + + @Check(test = "test_2") + public void check_2(Object[] results) { + Verify.checkEQ(expected_2[0], results[0]); + Verify.checkEQ(expected_2[1], results[1]); + Verify.checkEQ(results[0], results[1]); + } +} \ No newline at end of file From 69ff53780991d57f52a00d0d158d472d48f5d3aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 20 Jan 2026 15:31:39 +0100 Subject: [PATCH 24/51] Add support for OrL reassociation --- src/hotspot/share/opto/loopnode.cpp | 4 +++- .../compiler/loopopts/TestReductionReassociation.java | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 353d0acf8abd7..e8bad153aa8d6 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5004,13 +5004,15 @@ static AddNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phas return new MinLNode(phase->C, a, b); case Op_MaxL: return new MaxLNode(phase->C, a, b); + case Op_OrL: + return new OrLNode(a, b); default: ShouldNotReachHere(); } } static bool is_associative(Node* node) { - return node->is_MinMax() || node->Opcode() == Op_AddI || node->Opcode() == Op_AddL; + return node->is_MinMax() || node->Opcode() == Op_AddI || node->Opcode() == Op_AddL || node->Opcode() == Op_OrL; } static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* loop_head, PhaseIdealLoop* phase) { diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index ccaa48d9286d5..c24efa685729f 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -108,7 +108,8 @@ enum AssociativeAdd { MIN_I(CodeGenerationDataNameType.ints()), MAX_I(CodeGenerationDataNameType.ints()), MIN_L(CodeGenerationDataNameType.longs()), - MAX_L(CodeGenerationDataNameType.longs()); + MAX_L(CodeGenerationDataNameType.longs()), + OR_L(CodeGenerationDataNameType.longs()); final CodeGenerationDataNameType type; @@ -284,6 +285,7 @@ private TemplateToken generateOp(String a, String b) { case MAX_I -> "Integer.max(#a, #b)"; case MIN_L -> "Long.min(#a, #b)"; case MAX_L -> "Long.max(#a, #b)"; + case OR_L -> "#a | #b"; } )); return template.asToken(); @@ -294,10 +296,11 @@ private TemplateToken generateResultInit(String resultName) { let("resultName", resultName), let("boxedType", getBoxedTypeName()), let("type", add.type.name()), - "#type ", resultName, " = #boxedType.", + "#type ", resultName, " = ", switch (add) { - case MIN_D, MIN_F, MIN_I, MIN_L -> "MAX_VALUE"; - case ADD_I, ADD_L, MAX_D, MAX_F, MAX_I, MAX_L -> "MIN_VALUE"; + case MIN_D, MIN_F, MIN_I, MIN_L -> "#boxedType.MAX_VALUE"; + case ADD_I, ADD_L, MAX_D, MAX_F, MAX_I, MAX_L -> "#boxedType.MIN_VALUE"; + case OR_L -> "0"; }, ";\n" )); From f873057eba7de609e98a70e9192d570999ff1f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 20 Jan 2026 15:51:12 +0100 Subject: [PATCH 25/51] Add support for OrI reassociation --- src/hotspot/share/opto/loopnode.cpp | 6 +++++- .../jtreg/compiler/loopopts/TestReductionReassociation.java | 5 +++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index e8bad153aa8d6..20f7bee06c651 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5004,6 +5004,8 @@ static AddNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phas return new MinLNode(phase->C, a, b); case Op_MaxL: return new MaxLNode(phase->C, a, b); + case Op_OrI: + return new OrINode(a, b); case Op_OrL: return new OrLNode(a, b); default: @@ -5012,7 +5014,9 @@ static AddNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phas } static bool is_associative(Node* node) { - return node->is_MinMax() || node->Opcode() == Op_AddI || node->Opcode() == Op_AddL || node->Opcode() == Op_OrL; + return node->is_MinMax() || + node->Opcode() == Op_AddI || node->Opcode() == Op_AddL || + node->Opcode() == Op_OrI || node->Opcode() == Op_OrL; } static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* loop_head, PhaseIdealLoop* phase) { diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index c24efa685729f..9921511078766 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -109,6 +109,7 @@ enum AssociativeAdd { MAX_I(CodeGenerationDataNameType.ints()), MIN_L(CodeGenerationDataNameType.longs()), MAX_L(CodeGenerationDataNameType.longs()), + OR_I(CodeGenerationDataNameType.ints()), OR_L(CodeGenerationDataNameType.longs()); final CodeGenerationDataNameType type; @@ -285,7 +286,7 @@ private TemplateToken generateOp(String a, String b) { case MAX_I -> "Integer.max(#a, #b)"; case MIN_L -> "Long.min(#a, #b)"; case MAX_L -> "Long.max(#a, #b)"; - case OR_L -> "#a | #b"; + case OR_I, OR_L -> "#a | #b"; } )); return template.asToken(); @@ -300,7 +301,7 @@ private TemplateToken generateResultInit(String resultName) { switch (add) { case MIN_D, MIN_F, MIN_I, MIN_L -> "#boxedType.MAX_VALUE"; case ADD_I, ADD_L, MAX_D, MAX_F, MAX_I, MAX_L -> "#boxedType.MIN_VALUE"; - case OR_L -> "0"; + case OR_I, OR_L -> "0"; }, ";\n" )); From c70e5b45684bceedbe2b869c619dbf8e5fa73bb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 20 Jan 2026 15:55:25 +0100 Subject: [PATCH 26/51] Add support for XOrI and XOrL --- src/hotspot/share/opto/loopnode.cpp | 7 ++++++- .../compiler/loopopts/TestReductionReassociation.java | 7 +++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 20f7bee06c651..f106e4ca1c3a8 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5008,6 +5008,10 @@ static AddNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phas return new OrINode(a, b); case Op_OrL: return new OrLNode(a, b); + case Op_XorI: + return new XorINode(a, b); + case Op_XorL: + return new XorLNode(a, b); default: ShouldNotReachHere(); } @@ -5016,7 +5020,8 @@ static AddNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phas static bool is_associative(Node* node) { return node->is_MinMax() || node->Opcode() == Op_AddI || node->Opcode() == Op_AddL || - node->Opcode() == Op_OrI || node->Opcode() == Op_OrL; + node->Opcode() == Op_OrI || node->Opcode() == Op_OrL || + node->Opcode() == Op_XorI || node->Opcode() == Op_XorL; } static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* loop_head, PhaseIdealLoop* phase) { diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 9921511078766..3ea5a749591ed 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -110,7 +110,9 @@ enum AssociativeAdd { MIN_L(CodeGenerationDataNameType.longs()), MAX_L(CodeGenerationDataNameType.longs()), OR_I(CodeGenerationDataNameType.ints()), - OR_L(CodeGenerationDataNameType.longs()); + OR_L(CodeGenerationDataNameType.longs()), + XOR_I(CodeGenerationDataNameType.ints()), + XOR_L(CodeGenerationDataNameType.longs()); final CodeGenerationDataNameType type; @@ -287,6 +289,7 @@ private TemplateToken generateOp(String a, String b) { case MIN_L -> "Long.min(#a, #b)"; case MAX_L -> "Long.max(#a, #b)"; case OR_I, OR_L -> "#a | #b"; + case XOR_I, XOR_L -> "#a ^ #b"; } )); return template.asToken(); @@ -301,7 +304,7 @@ private TemplateToken generateResultInit(String resultName) { switch (add) { case MIN_D, MIN_F, MIN_I, MIN_L -> "#boxedType.MAX_VALUE"; case ADD_I, ADD_L, MAX_D, MAX_F, MAX_I, MAX_L -> "#boxedType.MIN_VALUE"; - case OR_I, OR_L -> "0"; + case OR_I, OR_L, XOR_I, XOR_L -> "0"; }, ";\n" )); From 7ec2b93e52d01c966fce9ad78d735a0a2e6d4cd9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 20 Jan 2026 15:56:34 +0100 Subject: [PATCH 27/51] Remove auxiliary tests --- .../loopopts/TestReductionReassociation2.java | 87 --------------- .../loopopts/TestReductionReassociation3.java | 94 ---------------- .../loopopts/TestReductionReassociation4.java | 102 ------------------ 3 files changed, 283 deletions(-) delete mode 100644 test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation2.java delete mode 100644 test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation3.java delete mode 100644 test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation4.java diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation2.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation2.java deleted file mode 100644 index a6f2920d478b3..0000000000000 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation2.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright (c) 2025 IBM Corporation. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -/** - * @test - * @bug 8351409 - * @summary Test the IR effects of reduction reassociation - * @library /test/lib / - * @run driver compiler.loopopts.TestReductionReassociation2 - */ - -package compiler.loopopts; - -import compiler.lib.generators.Generators; -import compiler.lib.ir_framework.*; -import compiler.lib.verify.Verify; - -public class TestReductionReassociation2 { - private static long[] input_2 = new long[10000]; - - static { - Generators.G.fill(Generators.G.longs(), input_2); - } - - private Object[] expected_2 = test_2(); - - public static void main(String[] args) { - TestFramework.runWithFlags("-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:CompileCommand=dontinline,*::*dontinline*", "-XX:VerifyIterativeGVN=1000"); - } - - @Test - @IR( - counts = {IRNode.ADD_L, "= 4"}, - phase = CompilePhase.AFTER_LOOP_OPTS) - public Object[] test_2() { - long result = Long.MIN_VALUE; - long result2 = Long.MIN_VALUE; - for (int i = 0; i < input_2.length; i += 4) { - long v0 = getArrayLong_dontinline(i + 0); - long v1 = getArrayLong_dontinline(i + 1); - long v2 = getArrayLong_dontinline(i + 2); - long v3 = getArrayLong_dontinline(i + 3); - long u0 = v0 + result; - long u1 = v1 + u0; - long u2 = v2 + u1; - long u3 = v3 + u2; - long t0 = v0 + v1; - long t1 = v2 + t0; - long t2 = v3 + t1; - long t3 = result + t2; - result = u3; - result2 = t3; - } - return new Object[] {result, result2}; - } - - private static long getArrayLong_dontinline(int i) { - return input_2[i]; - } - - @Check(test = "test_2") - public void check_2(Object[] results) { - Verify.checkEQ(expected_2[0], results[0]); - Verify.checkEQ(expected_2[1], results[1]); - Verify.checkEQ(results[0], results[1]); - } -} \ No newline at end of file diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation3.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation3.java deleted file mode 100644 index ff3339d97e322..0000000000000 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation3.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Copyright (c) 2025 IBM Corporation. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -/** - * @test - * @bug 8351409 - * @summary Test the IR effects of reduction reassociation - * @library /test/lib / - * @run driver compiler.loopopts.TestReductionReassociation3 - */ - -package compiler.loopopts; - -import compiler.lib.generators.Generators; -import compiler.lib.ir_framework.Check; -import compiler.lib.ir_framework.CompilePhase; -import compiler.lib.ir_framework.IR; -import compiler.lib.ir_framework.IRNode; -import compiler.lib.ir_framework.Test; -import compiler.lib.ir_framework.TestFramework; -import compiler.lib.verify.Verify; - -public class TestReductionReassociation3 { - private static long[] input_122 = new long[10000]; - - static { - Generators.G.fill(Generators.G.longs(), input_122); - } - - private Object[] expected_122 = test_122(); - - public static void main(String[] args) { - TestFramework.runWithFlags("-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:CompileCommand=dontinline,*::*dontinline*", "-XX:VerifyIterativeGVN=1000"); - } - - @Test - @IR( - counts = {IRNode.MAX_L, "= 4"}, - phase = CompilePhase.AFTER_LOOP_OPTS) - public Object[] test_122() { - // i = max(a, max(b, max(c, max(d, i)))) - // i' = max(i, max(a, max(b, max(c, d)))) - - // result = max(v3, max(v2, max(v1, max(v0, result)))) - // result2 = max(result, max(v3, max(v2, max(v1, v0)))) - - long result = Long.MIN_VALUE; - long result2 = Long.MIN_VALUE; - for (int i = 0; i < input_122.length; i += 4) { - long v0 = input_122[i + 0]; - long v1 = input_122[i + 1]; - long v2 = input_122[i + 2]; - long v3 = input_122[i + 3]; - long u0 = Long.max(v0, result); - long u1 = Long.max(v1, u0); - long u2 = Long.max(v2, u1); - long u3 = Long.max(v3, u2); - long t0 = Long.max(v0, v1); - long t1 = Long.max(v2, t0); - long t2 = Long.max(v3, t1); - long t3 = Long.max(result, t2); - result = u3; - result2 = t3; - } - return new Object[] {result, result2}; - } - - @Check(test = "test_122") - public void check_122(Object[] results) { - Verify.checkEQ(expected_122[0], results[0]); - Verify.checkEQ(expected_122[1], results[1]); - Verify.checkEQ(results[0], results[1]); - } -} \ No newline at end of file diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation4.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation4.java deleted file mode 100644 index 60ffb4649b0e5..0000000000000 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation4.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright (c) 2025 IBM Corporation. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -/** - * @test - * @bug 8351409 - * @summary Test the IR effects of reduction reassociation - * @library /test/lib / - * @run driver compiler.loopopts.TestReductionReassociation4 - */ - -package compiler.loopopts; - -import compiler.lib.generators.Generators; -import compiler.lib.ir_framework.Check; -import compiler.lib.ir_framework.CompilePhase; -import compiler.lib.ir_framework.IR; -import compiler.lib.ir_framework.IRNode; -import compiler.lib.ir_framework.Test; -import compiler.lib.ir_framework.TestFramework; -import compiler.lib.verify.Verify; - -public class TestReductionReassociation4 { - private static int[] input_2 = new int[10000]; - - static { - Generators.G.fill(Generators.G.ints(), input_2); - } - - private Object[] expected_2 = test_2(); - - public static void main(String[] args) { - TestFramework.runWithFlags("-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:CompileCommand=dontinline,*::*dontinline*", "-XX:VerifyIterativeGVN=1000"); - } - - @Test - @IR( - counts = {IRNode.ADD_I, "= 4"}, - phase = CompilePhase.AFTER_LOOP_OPTS) - public Object[] test_2() { - int result = Integer.MIN_VALUE; - int result2 = Integer.MIN_VALUE; - int i = 0; - while (i < input_2.length) { - int v0 = getArrayInt_dontinline(0, i); - int v1 = getArrayInt_dontinline(1, i); - int v2 = getArrayInt_dontinline(2, i); - int v3 = getArrayInt_dontinline(3, i); - int u0 = v0 + result; - int u1 = v1 + u0; - int u2 = v2 + u1; - int u3 = v3 + u2; - int t0 = v0 + v1; - int t1 = v2 + t0; - int t2 = v3 + t1; - int t3 = result + t2; - result = u3; - result2 = t3; - i = sum_dontinline(i, 4); - } - return asArray_dontinline(result, result2); - } - - private static Object[] asArray_dontinline(int result, int result2) { - return new Object[]{result, result2}; - } - - private static int sum_dontinline(int a, int b) { - return a + b; - } - - private static int getArrayInt_dontinline(int pos, int base) { - return input_2[pos + base]; - } - - @Check(test = "test_2") - public void check_2(Object[] results) { - Verify.checkEQ(expected_2[0], results[0]); - Verify.checkEQ(expected_2[1], results[1]); - Verify.checkEQ(results[0], results[1]); - } -} \ No newline at end of file From 7e77b03dc3f640ca299e7b109fbb02386b123054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Wed, 21 Jan 2026 10:42:36 +0100 Subject: [PATCH 28/51] Wrap new functionality in UseNewCode and rename test --- src/hotspot/share/opto/loopnode.cpp | 4 +++- ...TestReductionReassociationForAssociativeAdds.java} | 11 ++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) rename test/hotspot/jtreg/compiler/loopopts/{TestReductionReassociation.java => TestReductionReassociationForAssociativeAdds.java} (98%) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index f106e4ca1c3a8..c5458f9337265 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5509,7 +5509,9 @@ void PhaseIdealLoop::build_and_optimize() { // Reassociating changes Phi nodes, so iteration and reassociation have to be separated for (uint next = 0; next < wq.size(); ++next) { Node* m = wq.at(next); - try_reassociate_chain(m, phi, lpt, this); + if (UseNewCode) { + try_reassociate_chain(m, phi, lpt, this); + } } } } diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java similarity index 98% rename from test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java rename to test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java index 3ea5a749591ed..280df9b54325c 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java @@ -26,7 +26,7 @@ * @bug 8351409 * @summary Test the IR effects of reduction reassociation * @library /test/lib / - * @run driver compiler.loopopts.TestReductionReassociation + * @run driver ${test.main.class} */ package compiler.loopopts; @@ -48,23 +48,24 @@ import static compiler.lib.template_framework.Template.*; import static compiler.lib.template_framework.Template.let; -public class TestReductionReassociation { +public class TestReductionReassociationForAssociativeAdds { public static void main(String[] args) { // Create a new CompileFramework instance. CompileFramework comp = new CompileFramework(); // Add a java source file. - comp.addJavaSourceCode("compiler.loopopts.templated.ReductionReassociation", generate(comp)); + comp.addJavaSourceCode("compiler.loopopts.templated.ReductionReassociationForAssociativeAdds", generate(comp)); // Compile the source file. comp.compile("--add-modules=jdk.incubator.vector"); String[] flags = new String[] { "-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:VerifyIterativeGVN=1000", + "-XX:+UseNewCode", "-XX:CompileCommand=dontinline,*::*dontinline*", "--add-modules=jdk.incubator.vector", "--add-opens", "jdk.incubator.vector/jdk.incubator.vector=ALL-UNNAMED" }; - comp.invoke("compiler.loopopts.templated.ReductionReassociation", "main", new Object[] {flags}); + comp.invoke("compiler.loopopts.templated.ReductionReassociationForAssociativeAdds", "main", new Object[] {flags}); } public static String generate(CompileFramework comp) { @@ -85,7 +86,7 @@ public static String generate(CompileFramework comp) { // Create the test class, which runs all testTemplateTokens. return TestFrameworkClass.render( // package and class name. - "compiler.loopopts.templated", "ReductionReassociation", + "compiler.loopopts.templated", "ReductionReassociationForAssociativeAdds", // List of imports. Set.of("jdk.incubator.vector.Float16", "compiler.lib.generators.*", From 09b1cebcc7343178122857a8a581c68d9bd5d1f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Wed, 21 Jan 2026 10:44:55 +0100 Subject: [PATCH 29/51] Adjust test description --- .../loopopts/TestReductionReassociationForAssociativeAdds.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java index 280df9b54325c..73995f9d5ef65 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java @@ -24,7 +24,7 @@ /** * @test * @bug 8351409 - * @summary Test the IR effects of reduction reassociation + * @summary Test the IR effects of reduction reassociation on all associative add operations * @library /test/lib / * @run driver ${test.main.class} */ From 7d40ff3996f524866b621ac8fe904eb99f9f4409 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Wed, 21 Jan 2026 11:02:25 +0100 Subject: [PATCH 30/51] Separate edge case scenarios that apply to all to a separate test --- .../loopopts/TestReductionReassociation.java | 145 ++++++++++++++++++ ...uctionReassociationForAssociativeAdds.java | 89 ----------- 2 files changed, 145 insertions(+), 89 deletions(-) create mode 100644 test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java new file mode 100644 index 0000000000000..28a2b15e996c0 --- /dev/null +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -0,0 +1,145 @@ +/* + * Copyright (c) 2026 IBM Corporation. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @bug 8351409 + * @summary Test the IR effects of reduction reassociation + * @library /test/lib / + * @run driver ${test.main.class} + */ + +package compiler.loopopts; + +import compiler.lib.generators.Generators; +import compiler.lib.ir_framework.*; +import compiler.lib.verify.Verify; + +public class TestReductionReassociation { + private static long[] input = new long[10000]; + + static { + Generators.G.fill(Generators.G.longs(), input); + } + + private Object[] expectedNonPowerOfTwoBatch = testNonPowerOfTwoBatch(); + + public static void main(String[] args) { + TestFramework.runWithFlags( + "-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", + "-XX:+UseNewCode", + "-XX:CompileCommand=dontinline,*::*dontinline*", "-XX:VerifyIterativeGVN=1000" + ); + } + + @Test + @IR(counts = {IRNode.MAX_L, "= 5"}, + phase = CompilePhase.AFTER_LOOP_OPTS) + public Object[] testNonPowerOfTwoBatch() { + long result = Long.MIN_VALUE; + long result2 = Long.MIN_VALUE; + for (int i = 0; i < input.length; i += 5) { + long v0 = input[i + 0]; + long v1 = input[i + 1]; + long v2 = input[i + 2]; + long v3 = input[i + 3]; + long v4 = input[i + 4]; + long u0 = Long.max(v0, result); + long u1 = Long.max(v1, u0); + long u2 = Long.max(v2, u1); + long u3 = Long.max(v3, u2); + long u4 = Long.max(v4, u3); + long t0 = Long.max(v0, v1); + long t1 = Long.max(v2, t0); + long t2 = Long.max(v3, t1); + long t3 = Long.max(v4, t2); + long t4 = Long.max(result, t3); + result = u4; + result2 = t4; + } + return new Object[]{result, result2}; + } + + @Check(test = "testNonPowerOfTwoBatch") + public void checkNonPowerOfTwoBatch(Object[] results) { + Verify.checkEQ(expectedNonPowerOfTwoBatch[0], results[0]); + Verify.checkEQ(expectedNonPowerOfTwoBatch[1], results[1]); + Verify.checkEQ(results[0], results[1]); + } + + private Object[] expectedUseIntermediate = testUseIntermediate(); + + @Test + @IR(counts = {IRNode.MAX_L, "= 8"}, + phase = CompilePhase.AFTER_LOOP_OPTS) + public Object[] testUseIntermediate() { + long result = Long.MIN_VALUE; + long result2 = Long.MIN_VALUE; + for (int i = 0; i < input.length; i += 8) { + long v0 = input[i + 0]; + long v1 = input[i + 1]; + long v2 = input[i + 2]; + long v3 = input[i + 3]; + long v4 = input[i + 4]; + long v5 = input[i + 5]; + long v6 = input[i + 6]; + long v7 = input[i + 7]; + + var u0 = Math.max(v0, result); + var u1 = Math.max(v1, u0); + var u2 = Math.max(v2, u1); + var u3 = Math.max(v3, u2); + if (u3 == input.hashCode()) { + System.out.print(""); + } + var u4 = Math.max(v4, u3); + var u5 = Math.max(v5, u4); + var u6 = Math.max(v6, u5); + var u7 = Math.max(v7, u6); + + long t0 = Long.max(v0, v1); + long t1 = Long.max(v2, t0); + long t2 = Long.max(v3, t1); + long t3 = Long.max(result, t2); + if (t3 == input.hashCode()) { + System.out.print(""); + } + long t4 = Long.max(v4, t3); + long t5 = Long.max(v5, t4); + long t6 = Long.max(v6, t5); + long t7 = Long.max(v7, t6); + + result = u7; + result2 = t7; + } + return new Object[]{result, result2}; + } + + @Check(test = "testUseIntermediate") + public void checkUseIntermediate(Object[] results) { + Verify.checkEQ(expectedUseIntermediate[0], results[0]); + Verify.checkEQ(expectedUseIntermediate[1], results[1]); + Verify.checkEQ(results[0], results[1]); + } + +} \ No newline at end of file diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java index 73995f9d5ef65..a06efef04994a 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java @@ -78,11 +78,6 @@ public static String generate(CompileFramework comp) { .map(op -> new TestGenerator(op, batchSize, size).generate()) .forEach(testTemplateTokens::add); - // A single test to test a non-power-of-2 value - testTemplateTokens.add(new TestGenerator(AssociativeAdd.MAX_L, 5, size).generate()); - // A single test where an intermediate value is used some other way - testTemplateTokens.add(new TestGenerator(AssociativeAdd.MAX_L, batchSize, size).generateUseIntermediate()); - // Create the test class, which runs all testTemplateTokens. return TestFrameworkClass.render( // package and class name. @@ -153,90 +148,6 @@ public TemplateToken generate() { return testTemplate.asToken(); } - public TemplateToken generateUseIntermediate() { - var testTemplate = Template.make(() -> { - String test = $("test"); - String input = $("input"); - String expected = $("expected"); - String setup = $("setup"); - String check = $("check"); - return scope( - """ - // --- $test start --- - - """, - generateArrayField(input), - generateExpectedField(test, expected), - generateUseIntermediateTest(input, setup, test), - generateCheck(test, check, expected), - generateAuxMethods(input), - """ - - // --- $test end --- - """ - ); - }); - return testTemplate.asToken(); - } - - private TemplateToken generateUseIntermediateTest(String input, String setup, String test) { - var template = Template.make(() -> scope( - let("irNodeName", add.name()), - let("input", input), - let("setup", setup), - let("test", test), - let("type", add.type.name()), - """ - @Test - @IR(counts = {IRNode.#irNodeName, "= 8"}, - phase = CompilePhase.AFTER_LOOP_OPTS) - public Object[] #test() { - long result = Long.MIN_VALUE; - long result2 = Long.MIN_VALUE; - int i = 0; - while (i < #input.length) { - var v0 = getArray_dontinline_#input(0, i); - var v1 = getArray_dontinline_#input(1, i); - var v2 = getArray_dontinline_#input(2, i); - var v3 = getArray_dontinline_#input(3, i); - var v4 = getArray_dontinline_#input(4, i); - var v5 = getArray_dontinline_#input(5, i); - var v6 = getArray_dontinline_#input(6, i); - var v7 = getArray_dontinline_#input(7, i); - var u0 = Math.max(v0, result); - var u1 = Math.max(v1, u0); - var u2 = Math.max(v2, u1); - var u3 = Math.max(v3, u2); - if (u3 == #input.hashCode()) { - System.out.print(""); - } - var u4 = Math.max(v4, u3); - var u5 = Math.max(v5, u4); - var u6 = Math.max(v6, u5); - var u7 = Math.max(v7, u6); - - long t0 = Long.max(v0, v1); - long t1 = Long.max(v2, t0); - long t2 = Long.max(v3, t1); - long t3 = Long.max(result, t2); - if (t3 == #input.hashCode()) { - System.out.print(""); - } - long t4 = Long.max(v4, t3); - long t5 = Long.max(v5, t4); - long t6 = Long.max(v6, t5); - long t7 = Long.max(v7, t6); - result = u7; - result2 = t7; - i = sum_dontinline_#input(i, 8); - } - return asArray_dontinline_#input(result, result2); - } - """ - )); - return template.asToken(); - } - private TemplateToken generateAuxMethods(String input) { var template = Template.make(() -> scope( let("input", input), From 3f04d072e740d1b06b7dbf00e5a5caab881b0375 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Wed, 21 Jan 2026 11:04:00 +0100 Subject: [PATCH 31/51] Update test description --- .../jtreg/compiler/loopopts/TestReductionReassociation.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 28a2b15e996c0..c7888bf1b504b 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -24,7 +24,7 @@ /** * @test * @bug 8351409 - * @summary Test the IR effects of reduction reassociation + * @summary Test the IR effects of reduction reassociation for edge cases that apply to any of the associative adds * @library /test/lib / * @run driver ${test.main.class} */ From 040fe47b676661518b92bde5908e7ced0787c48e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Wed, 21 Jan 2026 12:08:00 +0100 Subject: [PATCH 32/51] Chain has to be of same original opcode, fixes sum + max mixed --- src/hotspot/share/opto/loopnode.cpp | 3 +- .../loopopts/TestReductionReassociation.java | 39 ++++++++++++++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index c5458f9337265..7c4a2dcbcd387 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5035,6 +5035,7 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l Node* left; Node* right; + // todo check == add_opcode as opposed to is_associative if (is_associative(node->in(1))) { left = reassociate_chain(add_opcode, node->in(1), phi, loop_head, phase); right = node->in(2); @@ -5065,7 +5066,7 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha Node* use = nullptr; for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) { Node* n = current->fast_out(i); - if (is_associative(n)) { + if (n->Opcode() == opcode) { use = n; break; } diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index c7888bf1b504b..4c02320b936d9 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -142,4 +142,41 @@ public void checkUseIntermediate(Object[] results) { Verify.checkEQ(results[0], results[1]); } -} \ No newline at end of file + private Object[] expectedSumMax = testSumMax(); + + @Test + @IR(counts = {IRNode.MAX_L, "= 4", IRNode.ADD_L, "= 2"}, + phase = CompilePhase.AFTER_LOOP_OPTS) + public Object[] testSumMax() { + long result = Long.MIN_VALUE; + long result2 = Long.MIN_VALUE; + for (int i = 0; i < input.length; i += 4) { + long v0 = getArray_dontinline(i + 0); + long v1 = getArray_dontinline(i + 1); + long v2 = getArray_dontinline(i + 2); + long v3 = getArray_dontinline(i + 3); + long u0 = Long.max(v0, result); + long u1 = Long.max(v1, u0); + long u2 = Long.max(v2, u1); + long u3 = Long.max(v3, u2); + long t0 = Long.max(v0, v1); + long t1 = Long.max(v2, t0); + long t2 = Long.max(v3, t1); + long t3 = Long.max(result, t2); + result += u3; + result2 += t3; + } + return new Object[]{result, result2}; + } + + @Check(test = "testSumMax") + public void checkSumMax(Object[] results) { + Verify.checkEQ(expectedSumMax[1], results[1]); + Verify.checkEQ(expectedSumMax[0], results[0]); + Verify.checkEQ(results[0], results[1]); + } + + static long getArray_dontinline(int i) { + return input[i]; + } +} From 648f1b6a2bdcc5014938beb0e3f512c1bad3948a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Wed, 21 Jan 2026 13:16:26 +0100 Subject: [PATCH 33/51] Check against given opcode when reassociating --- src/hotspot/share/opto/loopnode.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 7c4a2dcbcd387..2b5a9063e9d35 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5035,8 +5035,7 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l Node* left; Node* right; - // todo check == add_opcode as opposed to is_associative - if (is_associative(node->in(1))) { + if (node->in(1)->Opcode() == add_opcode) { left = reassociate_chain(add_opcode, node->in(1), phi, loop_head, phase); right = node->in(2); } else { From 3da547839e48486f3df4c662d9ca41b564b758a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Wed, 21 Jan 2026 13:26:46 +0100 Subject: [PATCH 34/51] Remove UseNewCode protection --- src/hotspot/share/opto/loopnode.cpp | 4 +--- .../jtreg/compiler/loopopts/TestReductionReassociation.java | 1 - .../TestReductionReassociationForAssociativeAdds.java | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 2b5a9063e9d35..0c092d32121bf 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5509,9 +5509,7 @@ void PhaseIdealLoop::build_and_optimize() { // Reassociating changes Phi nodes, so iteration and reassociation have to be separated for (uint next = 0; next < wq.size(); ++next) { Node* m = wq.at(next); - if (UseNewCode) { - try_reassociate_chain(m, phi, lpt, this); - } + try_reassociate_chain(m, phi, lpt, this); } } } diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 4c02320b936d9..7c226f98e7cbf 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -47,7 +47,6 @@ public class TestReductionReassociation { public static void main(String[] args) { TestFramework.runWithFlags( "-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", - "-XX:+UseNewCode", "-XX:CompileCommand=dontinline,*::*dontinline*", "-XX:VerifyIterativeGVN=1000" ); } diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java index a06efef04994a..a46dec21e193b 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java @@ -61,7 +61,6 @@ public static void main(String[] args) { String[] flags = new String[] { "-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:VerifyIterativeGVN=1000", - "-XX:+UseNewCode", "-XX:CompileCommand=dontinline,*::*dontinline*", "--add-modules=jdk.incubator.vector", "--add-opens", "jdk.incubator.vector/jdk.incubator.vector=ALL-UNNAMED" }; From 56395e6f935c1fc37a421eacc366cd8c87d93f58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Thu, 22 Jan 2026 06:46:51 +0100 Subject: [PATCH 35/51] Add missing Verify.java @compile --- .../loopopts/TestReductionReassociationForAssociativeAdds.java | 1 + 1 file changed, 1 insertion(+) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java index a46dec21e193b..d5711282f5f60 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java @@ -26,6 +26,7 @@ * @bug 8351409 * @summary Test the IR effects of reduction reassociation on all associative add operations * @library /test/lib / + * @compile ../lib/verify/Verify.java * @run driver ${test.main.class} */ From 91a6967b35a0433a94407eb2946e314346e14f51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Thu, 29 Jan 2026 18:19:37 +0100 Subject: [PATCH 36/51] Limit to Long Min/Max and add some documentation --- src/hotspot/share/opto/loopnode.cpp | 46 +------------------ ...uctionReassociationForAssociativeAdds.java | 46 ++++++------------- 2 files changed, 15 insertions(+), 77 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 0c092d32121bf..0e6e7296ab140 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4980,48 +4980,17 @@ bool PhaseIdealLoop::process_expensive_nodes() { static AddNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phase) { switch (opcode) { - case Op_AddI: - return new AddINode(a, b); - case Op_AddL: - return new AddLNode(a, b); - case Op_MinD: - return new MinDNode(a, b); - case Op_MaxD: - return new MaxDNode(a, b); - case Op_MinF: - return new MinFNode(a, b); - case Op_MaxF: - return new MaxFNode(a, b); - // case Op_MinHF: - // return new MinHFNode(a, b); - // case Op_MaxHF: - // return new MaxHFNode(a, b); - case Op_MinI: - return new MinINode(a, b); - case Op_MaxI: - return new MaxINode(a, b); case Op_MinL: return new MinLNode(phase->C, a, b); case Op_MaxL: return new MaxLNode(phase->C, a, b); - case Op_OrI: - return new OrINode(a, b); - case Op_OrL: - return new OrLNode(a, b); - case Op_XorI: - return new XorINode(a, b); - case Op_XorL: - return new XorLNode(a, b); default: ShouldNotReachHere(); } } static bool is_associative(Node* node) { - return node->is_MinMax() || - node->Opcode() == Op_AddI || node->Opcode() == Op_AddL || - node->Opcode() == Op_OrI || node->Opcode() == Op_OrL || - node->Opcode() == Op_XorI || node->Opcode() == Op_XorL; + return node->Opcode() == Op_MinL || node->Opcode() == Op_MaxL; } static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* loop_head, PhaseIdealLoop* phase) { @@ -5044,9 +5013,6 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l } Node* reassoc = build_min_max(add_opcode, left, right, phase); - // tty->print("[avoid-cmov] reassociate_chain: new node:"); - // reassoc->dump(); - phase->register_new_node(reassoc, loop_head); return reassoc; } @@ -5094,22 +5060,12 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha return; } - // tty->print("[avoid-cmov] try reassociate; chain length: %d\n", chain_length); - // tty->print("[avoid-cmov] try reassociate:\n"); - // tty->print("[avoid-cmov] phi: "); - // phi->dump(); - // tty->print("[avoid-cmov] try reassociate; chain head:\n"); - // chain_head->dump(); - Node* loop_head = lpt->head(); Node* reassociated = reassociate_chain(opcode, chain_head, phi, loop_head, phase); Node* new_chain_head = build_min_max(opcode, phi, reassociated, phase); phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); - - // tty->print("[avoid-cmov] after reassociate; new chain head:\n"); - // new_chain_head->dump(); } //============================================================================= diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java index d5711282f5f60..be78307d5f67a 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java @@ -24,7 +24,7 @@ /** * @test * @bug 8351409 - * @summary Test the IR effects of reduction reassociation on all associative add operations + * @summary Test the IR effects of reduction reassociation on long min/max loops * @library /test/lib / * @compile ../lib/verify/Verify.java * @run driver ${test.main.class} @@ -49,6 +49,13 @@ import static compiler.lib.template_framework.Template.*; import static compiler.lib.template_framework.Template.let; +/** + * For the time being this test only covers Min/Max for Long values, + * but it has been designed to easily extend it to other associative AddNode + * implementations. For example, if implementing it for AddI or AddL nodes, + * a typical loop would generate additional IR nodes. Hence, the test uses + * a custom while loop with non-inlined helper methods. + */ public class TestReductionReassociationForAssociativeAdds { public static void main(String[] args) { // Create a new CompileFramework instance. @@ -93,32 +100,14 @@ public static String generate(CompileFramework comp) { } enum AssociativeAdd { - ADD_I(CodeGenerationDataNameType.ints()), - ADD_L(CodeGenerationDataNameType.longs()), - MIN_D(CodeGenerationDataNameType.doubles()), - MAX_D(CodeGenerationDataNameType.doubles()), -// MIN_HF(CodeGenerationDataNameType.float16()), -// MAX_HF(CodeGenerationDataNameType.float16()), - MIN_F(CodeGenerationDataNameType.floats()), - MAX_F(CodeGenerationDataNameType.floats()), - MIN_I(CodeGenerationDataNameType.ints()), - MAX_I(CodeGenerationDataNameType.ints()), MIN_L(CodeGenerationDataNameType.longs()), - MAX_L(CodeGenerationDataNameType.longs()), - OR_I(CodeGenerationDataNameType.ints()), - OR_L(CodeGenerationDataNameType.longs()), - XOR_I(CodeGenerationDataNameType.ints()), - XOR_L(CodeGenerationDataNameType.longs()); + MAX_L(CodeGenerationDataNameType.longs()); final CodeGenerationDataNameType type; AssociativeAdd(CodeGenerationDataNameType type) { this.type = type; } - -// boolean isFloat16() { -// return MIN_HF == this || MAX_HF == this; -// } } record TestGenerator(AssociativeAdd add, int batchSize, int size) { @@ -191,17 +180,8 @@ private TemplateToken generateOp(String a, String b) { let("a", a), let("b", b), switch (add) { - case ADD_I, ADD_L -> "#a + #b"; - case MIN_D -> "Double.min(#a, #b)"; - case MAX_D -> "Double.max(#a, #b)"; - case MIN_F -> "Float.min(#a, #b)"; - case MAX_F -> "Float.max(#a, #b)"; - case MIN_I -> "Integer.min(#a, #b)"; - case MAX_I -> "Integer.max(#a, #b)"; case MIN_L -> "Long.min(#a, #b)"; case MAX_L -> "Long.max(#a, #b)"; - case OR_I, OR_L -> "#a | #b"; - case XOR_I, XOR_L -> "#a ^ #b"; } )); return template.asToken(); @@ -214,9 +194,8 @@ private TemplateToken generateResultInit(String resultName) { let("type", add.type.name()), "#type ", resultName, " = ", switch (add) { - case MIN_D, MIN_F, MIN_I, MIN_L -> "#boxedType.MAX_VALUE"; - case ADD_I, ADD_L, MAX_D, MAX_F, MAX_I, MAX_L -> "#boxedType.MIN_VALUE"; - case OR_I, OR_L, XOR_I, XOR_L -> "0"; + case MIN_L -> "#boxedType.MAX_VALUE"; + case MAX_L -> "#boxedType.MIN_VALUE"; }, ";\n" )); @@ -247,6 +226,9 @@ private TemplateToken generateTest(String input, String setup, String test) { """, generateResultInit("result"), generateResultInit("result2"), + // A manually constructed loop that uses auxiliary methods + // that doesn't produce associative Add nodes that could get + // in the way of commoning. """ int i = 0; while (i < #input.length) { From 7f4dbe3837d3166b5d7fb838fe483642af64f27b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Fri, 6 Feb 2026 11:13:15 +0100 Subject: [PATCH 37/51] Use MinMaxNode::build_min_max_long instead of roll own --- src/hotspot/share/opto/loopnode.cpp | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 0e6e7296ab140..c54e1b15b7f2a 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4978,17 +4978,6 @@ bool PhaseIdealLoop::process_expensive_nodes() { return progress; } -static AddNode* build_min_max(int opcode, Node* a, Node* b, PhaseIdealLoop* phase) { - switch (opcode) { - case Op_MinL: - return new MinLNode(phase->C, a, b); - case Op_MaxL: - return new MaxLNode(phase->C, a, b); - default: - ShouldNotReachHere(); - } -} - static bool is_associative(Node* node) { return node->Opcode() == Op_MinL || node->Opcode() == Op_MaxL; } @@ -5012,7 +5001,7 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l right = reassociate_chain(add_opcode, node->in(2), phi, loop_head, phase); } - Node* reassoc = build_min_max(add_opcode, left, right, phase); + Node* reassoc = MinMaxNode::build_min_max_long(&phase->igvn(), left, right, add_opcode == Op_MaxL); phase->register_new_node(reassoc, loop_head); return reassoc; } @@ -5063,7 +5052,7 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha Node* loop_head = lpt->head(); Node* reassociated = reassociate_chain(opcode, chain_head, phi, loop_head, phase); - Node* new_chain_head = build_min_max(opcode, phi, reassociated, phase); + Node* new_chain_head = MinMaxNode::build_min_max_long(&phase->igvn(), phi, reassociated, opcode == Op_MaxL); phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); } From 3b6d371584649bb5559679a1b6344ef583dceac7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Fri, 6 Feb 2026 11:18:39 +0100 Subject: [PATCH 38/51] Use unique_out instead of hand rolled loop --- src/hotspot/share/opto/loopnode.cpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index c54e1b15b7f2a..b47d7cfd5cc93 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5017,15 +5017,7 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha break; } - Node* use = nullptr; - for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) { - Node* n = current->fast_out(i); - if (n->Opcode() == opcode) { - use = n; - break; - } - } - + Node* use = current->unique_out(); if (use != nullptr) { if (!phase->ctrl_is_member(lpt, use)) { // Only interested in commutative add nodes that are in use in the loop From d7cf51fc332f1c5d2ffc66200d7863363f47dae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Fri, 6 Feb 2026 13:34:57 +0100 Subject: [PATCH 39/51] Avoid work list by using an iterator that allows deletes --- src/hotspot/share/opto/loopnode.cpp | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index b47d7cfd5cc93..f1f8d9a217571 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5006,7 +5006,7 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l return reassoc; } -static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLoop* phase) { +static bool try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLoop* phase) { Node* chain_head = nullptr; Node* current = n; int opcode = current->Opcode(); @@ -5021,12 +5021,12 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha if (use != nullptr) { if (!phase->ctrl_is_member(lpt, use)) { // Only interested in commutative add nodes that are in use in the loop - return; + return false; } if (use->in(1)->Opcode() == opcode && use->in(2)->Opcode() == opcode) { // A chain to reassociate cannot be constructed // when the chain can have multiple paths - return; + return false; } chain_length++; @@ -5038,7 +5038,7 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha if (chain_length < 2) { // Only reassociate long enough chains - return; + return false; } Node* loop_head = lpt->head(); @@ -5047,6 +5047,7 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha Node* new_chain_head = MinMaxNode::build_min_max_long(&phase->igvn(), phi, reassociated, opcode == Op_MaxL); phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); + return true; } //============================================================================= @@ -5436,18 +5437,14 @@ void PhaseIdealLoop::build_and_optimize() { Node* loop_head_use = loop_head->fast_out(i); if (loop_head_use->is_Phi()) { PhiNode* phi = loop_head_use->as_Phi(); - Unique_Node_List wq; - for (DUIterator_Fast jmax, j = phi->fast_outs(jmax); j < jmax; j++) { - Node* n = phi->fast_out(j); + for (DUIterator j = phi->outs(); phi->has_out(j); j++) { + Node* n = phi->out(j); if (is_associative(n)) { - wq.push(n); + if (try_reassociate_chain(n, phi, lpt, this)) { + --j; + } } } - // Reassociating changes Phi nodes, so iteration and reassociation have to be separated - for (uint next = 0; next < wq.size(); ++next) { - Node* m = wq.at(next); - try_reassociate_chain(m, phi, lpt, this); - } } } } From 66f3cd7f3ca9201b7b294b56274c104a52b31d71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Fri, 6 Feb 2026 13:41:57 +0100 Subject: [PATCH 40/51] Copy node notes to new nodes --- src/hotspot/share/opto/loopnode.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index f1f8d9a217571..c78bcf7faf816 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5003,6 +5003,7 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l Node* reassoc = MinMaxNode::build_min_max_long(&phase->igvn(), left, right, add_opcode == Op_MaxL); phase->register_new_node(reassoc, loop_head); + phase->C->copy_node_notes_to(reassoc, node); return reassoc; } @@ -5047,6 +5048,7 @@ static bool try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha Node* new_chain_head = MinMaxNode::build_min_max_long(&phase->igvn(), phi, reassociated, opcode == Op_MaxL); phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); + phase->C->copy_node_notes_to(new_chain_head, chain_head); return true; } From 6ce4dfc82062333aea3eac302bbc4116a2f35a5c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 3 Mar 2026 09:24:00 +0100 Subject: [PATCH 41/51] Revert "Copy node notes to new nodes" This reverts commit 66f3cd7f3ca9201b7b294b56274c104a52b31d71. --- src/hotspot/share/opto/loopnode.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index c78bcf7faf816..f1f8d9a217571 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5003,7 +5003,6 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l Node* reassoc = MinMaxNode::build_min_max_long(&phase->igvn(), left, right, add_opcode == Op_MaxL); phase->register_new_node(reassoc, loop_head); - phase->C->copy_node_notes_to(reassoc, node); return reassoc; } @@ -5048,7 +5047,6 @@ static bool try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha Node* new_chain_head = MinMaxNode::build_min_max_long(&phase->igvn(), phi, reassociated, opcode == Op_MaxL); phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); - phase->C->copy_node_notes_to(new_chain_head, chain_head); return true; } From 4f38d49c16e1c683e2eb4d5b2c665e5da0e58290 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 3 Mar 2026 09:24:06 +0100 Subject: [PATCH 42/51] Revert "Avoid work list by using an iterator that allows deletes" This reverts commit d7cf51fc332f1c5d2ffc66200d7863363f47dae3. --- src/hotspot/share/opto/loopnode.cpp | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index f1f8d9a217571..b47d7cfd5cc93 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5006,7 +5006,7 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l return reassoc; } -static bool try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLoop* phase) { +static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLoop* phase) { Node* chain_head = nullptr; Node* current = n; int opcode = current->Opcode(); @@ -5021,12 +5021,12 @@ static bool try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha if (use != nullptr) { if (!phase->ctrl_is_member(lpt, use)) { // Only interested in commutative add nodes that are in use in the loop - return false; + return; } if (use->in(1)->Opcode() == opcode && use->in(2)->Opcode() == opcode) { // A chain to reassociate cannot be constructed // when the chain can have multiple paths - return false; + return; } chain_length++; @@ -5038,7 +5038,7 @@ static bool try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha if (chain_length < 2) { // Only reassociate long enough chains - return false; + return; } Node* loop_head = lpt->head(); @@ -5047,7 +5047,6 @@ static bool try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha Node* new_chain_head = MinMaxNode::build_min_max_long(&phase->igvn(), phi, reassociated, opcode == Op_MaxL); phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); - return true; } //============================================================================= @@ -5437,14 +5436,18 @@ void PhaseIdealLoop::build_and_optimize() { Node* loop_head_use = loop_head->fast_out(i); if (loop_head_use->is_Phi()) { PhiNode* phi = loop_head_use->as_Phi(); - for (DUIterator j = phi->outs(); phi->has_out(j); j++) { - Node* n = phi->out(j); + Unique_Node_List wq; + for (DUIterator_Fast jmax, j = phi->fast_outs(jmax); j < jmax; j++) { + Node* n = phi->fast_out(j); if (is_associative(n)) { - if (try_reassociate_chain(n, phi, lpt, this)) { - --j; - } + wq.push(n); } } + // Reassociating changes Phi nodes, so iteration and reassociation have to be separated + for (uint next = 0; next < wq.size(); ++next) { + Node* m = wq.at(next); + try_reassociate_chain(m, phi, lpt, this); + } } } } From 6f6aedacbd644704068c2d4489ed39ce86ffc903 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 3 Mar 2026 09:24:07 +0100 Subject: [PATCH 43/51] Revert "Use unique_out instead of hand rolled loop" This reverts commit 3b6d371584649bb5559679a1b6344ef583dceac7. --- src/hotspot/share/opto/loopnode.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index b47d7cfd5cc93..c54e1b15b7f2a 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5017,7 +5017,15 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha break; } - Node* use = current->unique_out(); + Node* use = nullptr; + for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) { + Node* n = current->fast_out(i); + if (n->Opcode() == opcode) { + use = n; + break; + } + } + if (use != nullptr) { if (!phase->ctrl_is_member(lpt, use)) { // Only interested in commutative add nodes that are in use in the loop From f4413ef936396fb24f7df823b19025e8bf665fe2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 3 Mar 2026 09:29:52 +0100 Subject: [PATCH 44/51] Improved: Use unique_out instead of hand rolled loop --- src/hotspot/share/opto/loopnode.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index c54e1b15b7f2a..3a2ef2bcb5bc3 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -5018,12 +5018,9 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha } Node* use = nullptr; - for (DUIterator_Fast imax, i = current->fast_outs(imax); i < imax; i++) { - Node* n = current->fast_out(i); - if (n->Opcode() == opcode) { - use = n; - break; - } + Node* out = current->unique_out(); + if (out->Opcode() == opcode) { + use = out; } if (use != nullptr) { From f2975ba754a148d1d77d16fc74da3b06aa0c0847 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 3 Mar 2026 11:19:07 +0100 Subject: [PATCH 45/51] Improved avoid work list with iterator allowing deletes --- src/hotspot/share/opto/loopnode.cpp | 33 +++++++++++++---------------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 3a2ef2bcb5bc3..ffcbccdacdc16 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4978,10 +4978,6 @@ bool PhaseIdealLoop::process_expensive_nodes() { return progress; } -static bool is_associative(Node* node) { - return node->Opcode() == Op_MinL || node->Opcode() == Op_MaxL; -} - static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* loop_head, PhaseIdealLoop* phase) { if (phi == node->in(1)) { return node->in(2); @@ -5006,7 +5002,12 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l return reassoc; } -static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLoop* phase) { +static bool try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLoop* phase) { + bool is_associative = n->Opcode() == Op_MinL || n->Opcode() == Op_MaxL; + if (!is_associative) { + return false; + } + Node* chain_head = nullptr; Node* current = n; int opcode = current->Opcode(); @@ -5026,12 +5027,12 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha if (use != nullptr) { if (!phase->ctrl_is_member(lpt, use)) { // Only interested in commutative add nodes that are in use in the loop - return; + return false; } if (use->in(1)->Opcode() == opcode && use->in(2)->Opcode() == opcode) { // A chain to reassociate cannot be constructed // when the chain can have multiple paths - return; + return false; } chain_length++; @@ -5043,7 +5044,7 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha if (chain_length < 2) { // Only reassociate long enough chains - return; + return false; } Node* loop_head = lpt->head(); @@ -5052,6 +5053,8 @@ static void try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha Node* new_chain_head = MinMaxNode::build_min_max_long(&phase->igvn(), phi, reassociated, opcode == Op_MaxL); phase->register_new_node(new_chain_head, loop_head); phase->igvn().replace_node(chain_head, new_chain_head); + + return true; } //============================================================================= @@ -5441,18 +5444,12 @@ void PhaseIdealLoop::build_and_optimize() { Node* loop_head_use = loop_head->fast_out(i); if (loop_head_use->is_Phi()) { PhiNode* phi = loop_head_use->as_Phi(); - Unique_Node_List wq; - for (DUIterator_Fast jmax, j = phi->fast_outs(jmax); j < jmax; j++) { - Node* n = phi->fast_out(j); - if (is_associative(n)) { - wq.push(n); + for (DUIterator j = phi->outs(); phi->has_out(j); j++) { + Node* n = phi->out(j); + if (try_reassociate_chain(n, phi, lpt, this)) { + --j; } } - // Reassociating changes Phi nodes, so iteration and reassociation have to be separated - for (uint next = 0; next < wq.size(); ++next) { - Node* m = wq.at(next); - try_reassociate_chain(m, phi, lpt, this); - } } } } From bf4c27e9999048a547eeec739898415832963604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 3 Mar 2026 11:26:32 +0100 Subject: [PATCH 46/51] Copy nodes to new nodes --- src/hotspot/share/opto/loopnode.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index ffcbccdacdc16..3f1a3308bb9cf 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4999,6 +4999,7 @@ static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* l Node* reassoc = MinMaxNode::build_min_max_long(&phase->igvn(), left, right, add_opcode == Op_MaxL); phase->register_new_node(reassoc, loop_head); + phase->C->copy_node_notes_to(reassoc, node); return reassoc; } @@ -5052,6 +5053,7 @@ static bool try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, Pha Node* new_chain_head = MinMaxNode::build_min_max_long(&phase->igvn(), phi, reassociated, opcode == Op_MaxL); phase->register_new_node(new_chain_head, loop_head); + phase->C->copy_node_notes_to(new_chain_head, chain_head); phase->igvn().replace_node(chain_head, new_chain_head); return true; From 82678b5aee5c8aeb63b4a929118775a8d40bd5c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 3 Mar 2026 11:40:03 +0100 Subject: [PATCH 47/51] Combine templated and IR tests into a single class --- .../loopopts/TestReductionReassociation.java | 294 +++++++++++++++++- ...uctionReassociationForAssociativeAdds.java | 289 ----------------- 2 files changed, 283 insertions(+), 300 deletions(-) delete mode 100644 test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java index 7c226f98e7cbf..cd54ecb222fab 100644 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java +++ b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociation.java @@ -22,20 +22,299 @@ */ /** - * @test + * @test id=main * @bug 8351409 - * @summary Test the IR effects of reduction reassociation for edge cases that apply to any of the associative adds + * @summary Test the IR effects of reduction reassociation on long min/max loops * @library /test/lib / - * @run driver ${test.main.class} + * @compile ../lib/verify/Verify.java + * @run driver ${test.main.class} main + */ + +/** + * @test id=edge-cases + * @bug 8351409 + * @summary Test the IR effects of reduction reassociation for edge cases that apply to long min/max loops + * @library /test/lib / + * @run driver ${test.main.class} edge-cases */ package compiler.loopopts; +import compiler.lib.compile_framework.CompileFramework; import compiler.lib.generators.Generators; -import compiler.lib.ir_framework.*; +import compiler.lib.ir_framework.Check; +import compiler.lib.ir_framework.CompilePhase; +import compiler.lib.ir_framework.IR; +import compiler.lib.ir_framework.IRNode; +import compiler.lib.ir_framework.Test; +import compiler.lib.ir_framework.TestFramework; +import compiler.lib.template_framework.Template; +import compiler.lib.template_framework.TemplateToken; +import compiler.lib.template_framework.library.CodeGenerationDataNameType; +import compiler.lib.template_framework.library.PrimitiveType; +import compiler.lib.template_framework.library.TestFrameworkClass; import compiler.lib.verify.Verify; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Set; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static compiler.lib.template_framework.Template.*; +import static compiler.lib.template_framework.Template.let; + +/** + * For the time being this test only covers Min/Max for Long values, + * but it has been designed to easily extend it to other associative AddNode + * implementations. For example, if implementing it for AddI or AddL nodes, + * a typical loop would generate additional IR nodes. Hence, the test uses + * a custom while loop with non-inlined helper methods. + */ public class TestReductionReassociation { + public static void main(String[] args) { + switch (args[0]) { + case "main" -> { + // Create a new CompileFramework instance. + CompileFramework comp = new CompileFramework(); + + // Add a java source file. + comp.addJavaSourceCode("compiler.loopopts.templated.ReductionReassociation", generate(comp)); + + // Compile the source file. + comp.compile("--add-modules=jdk.incubator.vector"); + + String[] flags = new String[] { + "-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:VerifyIterativeGVN=1000", + "-XX:CompileCommand=dontinline,*::*dontinline*", + "--add-modules=jdk.incubator.vector", "--add-opens", "jdk.incubator.vector/jdk.incubator.vector=ALL-UNNAMED" + }; + comp.invoke("compiler.loopopts.templated.ReductionReassociation", "main", new Object[] {flags}); + } + case "edge-cases" -> { + TestFramework.runWithFlags( + "-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", + "-XX:CompileCommand=dontinline,*::*dontinline*", "-XX:VerifyIterativeGVN=1000" + ); + } + default -> throw new RuntimeException("unknown run id=" + args[0]); + } + + } + + public static String generate(CompileFramework comp) { + List testTemplateTokens = new ArrayList<>(); + + final int size = 10_000; + final int batchSize = 4; + + Stream.of(AssociativeAdd.values()) + .map(op -> new TestGenerator(op, batchSize, size).generate()) + .forEach(testTemplateTokens::add); + + // Create the test class, which runs all testTemplateTokens. + return TestFrameworkClass.render( + // package and class name. + "compiler.loopopts.templated", "ReductionReassociation", + // List of imports. + Set.of("jdk.incubator.vector.Float16", + "compiler.lib.generators.*", + "compiler.lib.verify.*"), + // classpath, so the Test VM has access to the compiled class files. + comp.getEscapedClassPathOfCompiledClasses(), + // The list of tests. + testTemplateTokens); + } + + enum AssociativeAdd { + MIN_L(CodeGenerationDataNameType.longs()), + MAX_L(CodeGenerationDataNameType.longs()); + + final CodeGenerationDataNameType type; + + AssociativeAdd(CodeGenerationDataNameType type) { + this.type = type; + } + } + + record TestGenerator(AssociativeAdd add, int batchSize, int size) { + public TemplateToken generate() { + var testTemplate = Template.make(() -> { + String test = $("test"); + String input = $("input"); + String expected = $("expected"); + String setup = $("setup"); + String check = $("check"); + return scope( + """ + // --- $test start --- + + """, + generateArrayField(input), + generateExpectedField(test, expected), + generateTest(input, setup, test), + generateCheck(test, check, expected), + generateAuxMethods(input), + """ + + // --- $test end --- + """ + ); + }); + return testTemplate.asToken(); + } + + private TemplateToken generateAuxMethods(String input) { + var template = Template.make(() -> scope( + let("input", input), + let("type", add.type.name()), + """ + static #type getArray_dontinline_#input(int pos, int base) { + return #input[pos + base]; + } + + static Object[] asArray_dontinline_#input(#type result, #type result2) { + return new Object[]{result, result2}; + } + + static int sum_dontinline_#input(int a, int b) { + return a + b; + } + """ + )); + return template.asToken(); + } + + private TemplateToken generateCheck(String test, String check, String expected) { + var template = Template.make(() -> scope( + let("test", test), + let("check", check), + let("expected", expected), + """ + @Check(test = "#test") + public void #check(Object[] results) { + Verify.checkEQ(#expected[0], results[0]); + Verify.checkEQ(#expected[1], results[1]); + Verify.checkEQ(results[0], results[1]); + } + """ + )); + return template.asToken(); + } + + private TemplateToken generateOp(String a, String b) { + var template = Template.make(() -> scope( + let("a", a), + let("b", b), + switch (add) { + case MIN_L -> "Long.min(#a, #b)"; + case MAX_L -> "Long.max(#a, #b)"; + } + )); + return template.asToken(); + } + + private TemplateToken generateResultInit(String resultName) { + var template = Template.make(() -> scope( + let("resultName", resultName), + let("boxedType", getBoxedTypeName()), + let("type", add.type.name()), + "#type ", resultName, " = ", + switch (add) { + case MIN_L -> "#boxedType.MAX_VALUE"; + case MAX_L -> "#boxedType.MIN_VALUE"; + }, + ";\n" + )); + return template.asToken(); + } + + private Object getBoxedTypeName() { + if (add.type instanceof PrimitiveType primitiveType) { + return primitiveType.boxedTypeName(); + } + + return add.type.name(); + } + + private TemplateToken generateTest(String input, String setup, String test) { + var template = Template.make(() -> scope( + let("countsIR", batchSize), + let("irNodeName", add.name()), + let("input", input), + let("setup", setup), + let("test", test), + let("type", add.type.name()), + """ + @Test + @IR(counts = {IRNode.#irNodeName, "= #countsIR"}, + phase = CompilePhase.AFTER_LOOP_OPTS) + public Object[] #test() { + """, + generateResultInit("result"), + generateResultInit("result2"), + // A manually constructed loop that uses auxiliary methods + // that doesn't produce associative Add nodes that could get + // in the way of commoning. + """ + int i = 0; + while (i < #input.length) { + """, + IntStream.range(0, batchSize).mapToObj(i -> + List.of("#type v", i, " = getArray_dontinline_#input(", i, ", i);\n")).toList(), + "#type u0 = ", generateOp("v0", "result"), ";\n", + IntStream.range(1, batchSize).mapToObj(i -> + List.of("#type u", i, " = ", generateOp("v" + i, "u" + (i - 1)), ";\n") + ).toList(), + "#type t0 = ", generateOp("v0", "v1"), ";\n", + IntStream.range(1, batchSize - 1).mapToObj(i -> + List.of("#type t", i, " = ", generateOp("v" + (i + 1), "t" + (i - 1)), ";\n") + ).toList(), + "#type t", batchSize - 1, " = ", generateOp("result", "t" + (batchSize - 2)), ";\n", + "result = u", batchSize - 1,";\n", + "result2 = t", batchSize - 1,";\n", + "", + """ + i = sum_dontinline_#input(i, #countsIR); + } + return asArray_dontinline_#input(result, result2); + } + """ + )); + return template.asToken(); + } + + private TemplateToken generateExpectedField(String test, String expected) { + var template = Template.make(() -> scope( + let("size", size), + let("test", test), + let("expected", expected), + """ + private Object[] #expected = #test(); + """ + )); + return template.asToken(); + } + + private TemplateToken generateArrayField(String input) { + var template = Template.make(() -> scope( + let("size", size), + let("input", input), + let("type", add.type.name()), + let("gen", add.type.name().toLowerCase(Locale.ROOT) + "s"), + """ + private static #type[] #input = new #type[#size]; + + static { + Generators.G.fill(Generators.G.#gen(), #input); + } + """ + )); + return template.asToken(); + } + } + private static long[] input = new long[10000]; static { @@ -44,13 +323,6 @@ public class TestReductionReassociation { private Object[] expectedNonPowerOfTwoBatch = testNonPowerOfTwoBatch(); - public static void main(String[] args) { - TestFramework.runWithFlags( - "-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", - "-XX:CompileCommand=dontinline,*::*dontinline*", "-XX:VerifyIterativeGVN=1000" - ); - } - @Test @IR(counts = {IRNode.MAX_L, "= 5"}, phase = CompilePhase.AFTER_LOOP_OPTS) diff --git a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java b/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java deleted file mode 100644 index be78307d5f67a..0000000000000 --- a/test/hotspot/jtreg/compiler/loopopts/TestReductionReassociationForAssociativeAdds.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Copyright (c) 2026 IBM Corporation. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -/** - * @test - * @bug 8351409 - * @summary Test the IR effects of reduction reassociation on long min/max loops - * @library /test/lib / - * @compile ../lib/verify/Verify.java - * @run driver ${test.main.class} - */ - -package compiler.loopopts; - -import compiler.lib.compile_framework.CompileFramework; -import compiler.lib.template_framework.Template; -import compiler.lib.template_framework.TemplateToken; -import compiler.lib.template_framework.library.CodeGenerationDataNameType; -import compiler.lib.template_framework.library.PrimitiveType; -import compiler.lib.template_framework.library.TestFrameworkClass; - -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.Set; -import java.util.stream.IntStream; -import java.util.stream.Stream; - -import static compiler.lib.template_framework.Template.*; -import static compiler.lib.template_framework.Template.let; - -/** - * For the time being this test only covers Min/Max for Long values, - * but it has been designed to easily extend it to other associative AddNode - * implementations. For example, if implementing it for AddI or AddL nodes, - * a typical loop would generate additional IR nodes. Hence, the test uses - * a custom while loop with non-inlined helper methods. - */ -public class TestReductionReassociationForAssociativeAdds { - public static void main(String[] args) { - // Create a new CompileFramework instance. - CompileFramework comp = new CompileFramework(); - - // Add a java source file. - comp.addJavaSourceCode("compiler.loopopts.templated.ReductionReassociationForAssociativeAdds", generate(comp)); - - // Compile the source file. - comp.compile("--add-modules=jdk.incubator.vector"); - - String[] flags = new String[] { - "-XX:-UseSuperWord", "-XX:LoopMaxUnroll=0", "-XX:VerifyIterativeGVN=1000", - "-XX:CompileCommand=dontinline,*::*dontinline*", - "--add-modules=jdk.incubator.vector", "--add-opens", "jdk.incubator.vector/jdk.incubator.vector=ALL-UNNAMED" - }; - comp.invoke("compiler.loopopts.templated.ReductionReassociationForAssociativeAdds", "main", new Object[] {flags}); - } - - public static String generate(CompileFramework comp) { - List testTemplateTokens = new ArrayList<>(); - - final int size = 10_000; - final int batchSize = 4; - - Stream.of(AssociativeAdd.values()) - .map(op -> new TestGenerator(op, batchSize, size).generate()) - .forEach(testTemplateTokens::add); - - // Create the test class, which runs all testTemplateTokens. - return TestFrameworkClass.render( - // package and class name. - "compiler.loopopts.templated", "ReductionReassociationForAssociativeAdds", - // List of imports. - Set.of("jdk.incubator.vector.Float16", - "compiler.lib.generators.*", - "compiler.lib.verify.*"), - // classpath, so the Test VM has access to the compiled class files. - comp.getEscapedClassPathOfCompiledClasses(), - // The list of tests. - testTemplateTokens); - } - - enum AssociativeAdd { - MIN_L(CodeGenerationDataNameType.longs()), - MAX_L(CodeGenerationDataNameType.longs()); - - final CodeGenerationDataNameType type; - - AssociativeAdd(CodeGenerationDataNameType type) { - this.type = type; - } - } - - record TestGenerator(AssociativeAdd add, int batchSize, int size) { - public TemplateToken generate() { - var testTemplate = Template.make(() -> { - String test = $("test"); - String input = $("input"); - String expected = $("expected"); - String setup = $("setup"); - String check = $("check"); - return scope( - """ - // --- $test start --- - - """, - generateArrayField(input), - generateExpectedField(test, expected), - generateTest(input, setup, test), - generateCheck(test, check, expected), - generateAuxMethods(input), - """ - - // --- $test end --- - """ - ); - }); - return testTemplate.asToken(); - } - - private TemplateToken generateAuxMethods(String input) { - var template = Template.make(() -> scope( - let("input", input), - let("type", add.type.name()), - """ - static #type getArray_dontinline_#input(int pos, int base) { - return #input[pos + base]; - } - - static Object[] asArray_dontinline_#input(#type result, #type result2) { - return new Object[]{result, result2}; - } - - static int sum_dontinline_#input(int a, int b) { - return a + b; - } - """ - )); - return template.asToken(); - } - - private TemplateToken generateCheck(String test, String check, String expected) { - var template = Template.make(() -> scope( - let("test", test), - let("check", check), - let("expected", expected), - """ - @Check(test = "#test") - public void #check(Object[] results) { - Verify.checkEQ(#expected[0], results[0]); - Verify.checkEQ(#expected[1], results[1]); - Verify.checkEQ(results[0], results[1]); - } - """ - )); - return template.asToken(); - } - - private TemplateToken generateOp(String a, String b) { - var template = Template.make(() -> scope( - let("a", a), - let("b", b), - switch (add) { - case MIN_L -> "Long.min(#a, #b)"; - case MAX_L -> "Long.max(#a, #b)"; - } - )); - return template.asToken(); - } - - private TemplateToken generateResultInit(String resultName) { - var template = Template.make(() -> scope( - let("resultName", resultName), - let("boxedType", getBoxedTypeName()), - let("type", add.type.name()), - "#type ", resultName, " = ", - switch (add) { - case MIN_L -> "#boxedType.MAX_VALUE"; - case MAX_L -> "#boxedType.MIN_VALUE"; - }, - ";\n" - )); - return template.asToken(); - } - - private Object getBoxedTypeName() { - if (add.type instanceof PrimitiveType primitiveType) { - return primitiveType.boxedTypeName(); - } - - return add.type.name(); - } - - private TemplateToken generateTest(String input, String setup, String test) { - var template = Template.make(() -> scope( - let("countsIR", batchSize), - let("irNodeName", add.name()), - let("input", input), - let("setup", setup), - let("test", test), - let("type", add.type.name()), - """ - @Test - @IR(counts = {IRNode.#irNodeName, "= #countsIR"}, - phase = CompilePhase.AFTER_LOOP_OPTS) - public Object[] #test() { - """, - generateResultInit("result"), - generateResultInit("result2"), - // A manually constructed loop that uses auxiliary methods - // that doesn't produce associative Add nodes that could get - // in the way of commoning. - """ - int i = 0; - while (i < #input.length) { - """, - IntStream.range(0, batchSize).mapToObj(i -> - List.of("#type v", i, " = getArray_dontinline_#input(", i, ", i);\n")).toList(), - "#type u0 = ", generateOp("v0", "result"), ";\n", - IntStream.range(1, batchSize).mapToObj(i -> - List.of("#type u", i, " = ", generateOp("v" + i, "u" + (i - 1)), ";\n") - ).toList(), - "#type t0 = ", generateOp("v0", "v1"), ";\n", - IntStream.range(1, batchSize - 1).mapToObj(i -> - List.of("#type t", i, " = ", generateOp("v" + (i + 1), "t" + (i - 1)), ";\n") - ).toList(), - "#type t", batchSize - 1, " = ", generateOp("result", "t" + (batchSize - 2)), ";\n", - "result = u", batchSize - 1,";\n", - "result2 = t", batchSize - 1,";\n", - "", - """ - i = sum_dontinline_#input(i, #countsIR); - } - return asArray_dontinline_#input(result, result2); - } - """ - )); - return template.asToken(); - } - - private TemplateToken generateExpectedField(String test, String expected) { - var template = Template.make(() -> scope( - let("size", size), - let("test", test), - let("expected", expected), - """ - private Object[] #expected = #test(); - """ - )); - return template.asToken(); - } - - private TemplateToken generateArrayField(String input) { - var template = Template.make(() -> scope( - let("size", size), - let("input", input), - let("type", add.type.name()), - let("gen", add.type.name().toLowerCase(Locale.ROOT) + "s"), - """ - private static #type[] #input = new #type[#size]; - - static { - Generators.G.fill(Generators.G.#gen(), #input); - } - """ - )); - return template.asToken(); - } - } -} From 78df7c48dd105d3552a75def9e7cd9042f7a7498 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 3 Mar 2026 17:07:47 +0100 Subject: [PATCH 48/51] Refactor reassociation to loopopts and encapsulate --- src/hotspot/share/opto/loopnode.cpp | 102 +------------------------ src/hotspot/share/opto/loopnode.hpp | 3 + src/hotspot/share/opto/loopopts.cpp | 113 ++++++++++++++++++++++++++++ 3 files changed, 117 insertions(+), 101 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.cpp b/src/hotspot/share/opto/loopnode.cpp index 3f1a3308bb9cf..fb7986b78d6e3 100644 --- a/src/hotspot/share/opto/loopnode.cpp +++ b/src/hotspot/share/opto/loopnode.cpp @@ -4978,87 +4978,6 @@ bool PhaseIdealLoop::process_expensive_nodes() { return progress; } -static Node* reassociate_chain(int add_opcode, Node* node, PhiNode* phi, Node* loop_head, PhaseIdealLoop* phase) { - if (phi == node->in(1)) { - return node->in(2); - } - - if (phi == node->in(2)) { - return node->in(1); - } - - Node* left; - Node* right; - if (node->in(1)->Opcode() == add_opcode) { - left = reassociate_chain(add_opcode, node->in(1), phi, loop_head, phase); - right = node->in(2); - } else { - left = node->in(1); - right = reassociate_chain(add_opcode, node->in(2), phi, loop_head, phase); - } - - Node* reassoc = MinMaxNode::build_min_max_long(&phase->igvn(), left, right, add_opcode == Op_MaxL); - phase->register_new_node(reassoc, loop_head); - phase->C->copy_node_notes_to(reassoc, node); - return reassoc; -} - -static bool try_reassociate_chain(Node* n, PhiNode* phi, IdealLoopTree* lpt, PhaseIdealLoop* phase) { - bool is_associative = n->Opcode() == Op_MinL || n->Opcode() == Op_MaxL; - if (!is_associative) { - return false; - } - - Node* chain_head = nullptr; - Node* current = n; - int opcode = current->Opcode(); - - int chain_length = 1; - while (current != nullptr) { - if (current->outcnt() != 1) { - break; - } - - Node* use = nullptr; - Node* out = current->unique_out(); - if (out->Opcode() == opcode) { - use = out; - } - - if (use != nullptr) { - if (!phase->ctrl_is_member(lpt, use)) { - // Only interested in commutative add nodes that are in use in the loop - return false; - } - if (use->in(1)->Opcode() == opcode && use->in(2)->Opcode() == opcode) { - // A chain to reassociate cannot be constructed - // when the chain can have multiple paths - return false; - } - - chain_length++; - chain_head = use; - } - - current = use; - } - - if (chain_length < 2) { - // Only reassociate long enough chains - return false; - } - - Node* loop_head = lpt->head(); - Node* reassociated = reassociate_chain(opcode, chain_head, phi, loop_head, phase); - - Node* new_chain_head = MinMaxNode::build_min_max_long(&phase->igvn(), phi, reassociated, opcode == Op_MaxL); - phase->register_new_node(new_chain_head, loop_head); - phase->C->copy_node_notes_to(new_chain_head, chain_head); - phase->igvn().replace_node(chain_head, new_chain_head); - - return true; -} - //============================================================================= //----------------------------build_and_optimize------------------------------- // Create a PhaseLoop. Build the ideal Loop tree. Map each Ideal Node to @@ -5436,26 +5355,7 @@ void PhaseIdealLoop::build_and_optimize() { } if (!C->major_progress()) { - for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { - IdealLoopTree* lpt = iter.current(); - if (lpt->is_innermost()) { - Node* loop_head = lpt->head(); - - // Look for loop head uses that are Phi - for (DUIterator_Fast imax, i = loop_head->fast_outs(imax); i < imax; i++) { - Node* loop_head_use = loop_head->fast_out(i); - if (loop_head_use->is_Phi()) { - PhiNode* phi = loop_head_use->as_Phi(); - for (DUIterator j = phi->outs(); phi->has_out(j); j++) { - Node* n = phi->out(j); - if (try_reassociate_chain(n, phi, lpt, this)) { - --j; - } - } - } - } - } - } + reassociate_reduction_chains(); } } diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index ffc283ac94196..ebcac2c416122 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -888,6 +888,7 @@ class PhaseIdealLoop : public PhaseTransform { friend class SuperWord; friend class ShenandoahBarrierC2Support; friend class AutoNodeBudget; + friend class ReassociateReductionChain; // Map loop membership for CFG nodes, and ctrl for non-CFG nodes. // @@ -1544,6 +1545,8 @@ class PhaseIdealLoop : public PhaseTransform { void eliminate_useless_zero_trip_guard(); void eliminate_useless_multiversion_if(); + void reassociate_reduction_chains(); + public: // Change the control input of expensive nodes to allow commoning by // IGVN when it is guaranteed to not result in a more frequent diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index 1855263539b1d..9895787ed2707 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -4508,6 +4508,119 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old return true; } +class ReassociateReductionChain : public StackObj { +public: + ReassociateReductionChain(IdealLoopTree* loop, PhaseIdealLoop* phase) : _loop(loop), _phase(phase) { + } + + bool transform(Node* n, PhiNode* phi) { + bool is_associative = n->Opcode() == Op_MinL || n->Opcode() == Op_MaxL; + if (!is_associative) { + return false; + } + + Node* chain_head = nullptr; + Node* current = n; + int opcode = current->Opcode(); + + int chain_length = 1; + while (current != nullptr) { + if (current->outcnt() != 1) { + break; + } + + Node* use = nullptr; + Node* out = current->unique_out(); + if (out->Opcode() == opcode) { + use = out; + } + + if (use != nullptr) { + if (!_phase->ctrl_is_member(_loop, use)) { + // Only interested in commutative add nodes that are in use in the loop + return false; + } + if (use->in(1)->Opcode() == opcode && use->in(2)->Opcode() == opcode) { + // A chain to reassociate cannot be constructed + // when the chain can have multiple paths + return false; + } + + chain_length++; + chain_head = use; + } + + current = use; + } + + if (chain_length < 2) { + // Only reassociate long enough chains + return false; + } + + Node* reassociated = reassociate_chain(chain_head, opcode, phi); + + Node* new_chain_head = MinMaxNode::build_min_max_long(&_phase->igvn(), phi, reassociated, opcode == Op_MaxL); + _phase->register_new_node(new_chain_head, _loop->head()); + _phase->C->copy_node_notes_to(new_chain_head, chain_head); + _phase->igvn().replace_node(chain_head, new_chain_head); + + return true; + } + +private: + IdealLoopTree* _loop; + PhaseIdealLoop* _phase; + + Node* reassociate_chain(Node* node, int opcode, PhiNode* phi) { + if (phi == node->in(1)) { + return node->in(2); + } + + if (phi == node->in(2)) { + return node->in(1); + } + + Node* left; + Node* right; + if (node->in(1)->Opcode() == opcode) { + left = reassociate_chain(node->in(1), opcode, phi); + right = node->in(2); + } else { + left = node->in(1); + right = reassociate_chain(node->in(2), opcode, phi); + } + + Node* reassoc = MinMaxNode::build_min_max_long(&_phase->igvn(), left, right, opcode == Op_MaxL); + _phase->register_new_node(reassoc, _loop->head()); + _phase->C->copy_node_notes_to(reassoc, node); + return reassoc; + } +}; + +void PhaseIdealLoop::reassociate_reduction_chains() { + for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { + IdealLoopTree* loop = iter.current(); + if (loop->is_innermost()) { + Node* loop_head = loop->head(); + ReassociateReductionChain rr(loop, this); + + for (DUIterator_Fast imax, i = loop_head->fast_outs(imax); i < imax; i++) { + Node* loop_head_use = loop_head->fast_out(i); + if (loop_head_use->is_Phi()) { + PhiNode* phi = loop_head_use->as_Phi(); + for (DUIterator j = phi->outs(); phi->has_out(j); j++) { + Node* n = phi->out(j); + if (rr.transform(n, phi)) { + --j; + } + } + } + } + } + } +} + // AutoVectorize the loop: replace scalar ops with vector ops. PhaseIdealLoop::AutoVectorizeStatus PhaseIdealLoop::auto_vectorize(IdealLoopTree* lpt, VSharedData &vshared) { From 1dfc62b8f364ea9f3e1b0796d35255a9c880e902 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 3 Mar 2026 17:13:35 +0100 Subject: [PATCH 49/51] Minor adjustments --- src/hotspot/share/opto/loopnode.hpp | 1 - src/hotspot/share/opto/loopopts.cpp | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/hotspot/share/opto/loopnode.hpp b/src/hotspot/share/opto/loopnode.hpp index ebcac2c416122..9b7745d4beb1c 100644 --- a/src/hotspot/share/opto/loopnode.hpp +++ b/src/hotspot/share/opto/loopnode.hpp @@ -888,7 +888,6 @@ class PhaseIdealLoop : public PhaseTransform { friend class SuperWord; friend class ShenandoahBarrierC2Support; friend class AutoNodeBudget; - friend class ReassociateReductionChain; // Map loop membership for CFG nodes, and ctrl for non-CFG nodes. // diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index 9895787ed2707..8dcbcf946d65c 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -4508,9 +4508,9 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old return true; } -class ReassociateReductionChain : public StackObj { +class ReassociateReductionChains : public StackObj { public: - ReassociateReductionChain(IdealLoopTree* loop, PhaseIdealLoop* phase) : _loop(loop), _phase(phase) { + ReassociateReductionChains(IdealLoopTree* loop, PhaseIdealLoop* phase) : _loop(loop), _phase(phase) { } bool transform(Node* n, PhiNode* phi) { @@ -4603,7 +4603,7 @@ void PhaseIdealLoop::reassociate_reduction_chains() { IdealLoopTree* loop = iter.current(); if (loop->is_innermost()) { Node* loop_head = loop->head(); - ReassociateReductionChain rr(loop, this); + ReassociateReductionChains rrc(loop, this); for (DUIterator_Fast imax, i = loop_head->fast_outs(imax); i < imax; i++) { Node* loop_head_use = loop_head->fast_out(i); @@ -4611,7 +4611,7 @@ void PhaseIdealLoop::reassociate_reduction_chains() { PhiNode* phi = loop_head_use->as_Phi(); for (DUIterator j = phi->outs(); phi->has_out(j); j++) { Node* n = phi->out(j); - if (rr.transform(n, phi)) { + if (rrc.transform(n, phi)) { --j; } } From 6eaa04051021aa545e79e71a0df610791eea9b82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Tue, 3 Mar 2026 17:46:47 +0100 Subject: [PATCH 50/51] Small refactorings after PR review --- src/hotspot/share/opto/loopopts.cpp | 56 ++++++++++++++++------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index 8dcbcf946d65c..1e7633055e9fe 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -4513,9 +4513,32 @@ class ReassociateReductionChains : public StackObj { ReassociateReductionChains(IdealLoopTree* loop, PhaseIdealLoop* phase) : _loop(loop), _phase(phase) { } - bool transform(Node* n, PhiNode* phi) { - bool is_associative = n->Opcode() == Op_MinL || n->Opcode() == Op_MaxL; - if (!is_associative) { + void reassociate_chains() { + Node* loop_head = _loop->head(); + for (DUIterator_Fast imax, i = loop_head->fast_outs(imax); i < imax; i++) { + Node* loop_head_use = loop_head->fast_out(i); + if (loop_head_use->is_Phi()) { + PhiNode* phi = loop_head_use->as_Phi(); + for (DUIterator j = phi->outs(); phi->has_out(j); j++) { + Node* n = phi->out(j); + if (try_reassociate_chain(n, phi)) { + --j; + } + } + } + } + } + +private: + IdealLoopTree* _loop; + PhaseIdealLoop* _phase; + + static bool is_associative(Node* n) { + return n->Opcode() == Op_MinL || n->Opcode() == Op_MaxL; + } + + bool try_reassociate_chain(Node* n, PhiNode* phi) { + if (!is_associative(n)) { return false; } @@ -4558,7 +4581,7 @@ class ReassociateReductionChains : public StackObj { return false; } - Node* reassociated = reassociate_chain(chain_head, opcode, phi); + Node* reassociated = do_reassociate_chain(chain_head, opcode, phi); Node* new_chain_head = MinMaxNode::build_min_max_long(&_phase->igvn(), phi, reassociated, opcode == Op_MaxL); _phase->register_new_node(new_chain_head, _loop->head()); @@ -4568,11 +4591,7 @@ class ReassociateReductionChains : public StackObj { return true; } -private: - IdealLoopTree* _loop; - PhaseIdealLoop* _phase; - - Node* reassociate_chain(Node* node, int opcode, PhiNode* phi) { + Node* do_reassociate_chain(Node* node, int opcode, PhiNode* phi) { if (phi == node->in(1)) { return node->in(2); } @@ -4584,11 +4603,11 @@ class ReassociateReductionChains : public StackObj { Node* left; Node* right; if (node->in(1)->Opcode() == opcode) { - left = reassociate_chain(node->in(1), opcode, phi); + left = do_reassociate_chain(node->in(1), opcode, phi); right = node->in(2); } else { left = node->in(1); - right = reassociate_chain(node->in(2), opcode, phi); + right = do_reassociate_chain(node->in(2), opcode, phi); } Node* reassoc = MinMaxNode::build_min_max_long(&_phase->igvn(), left, right, opcode == Op_MaxL); @@ -4602,21 +4621,8 @@ void PhaseIdealLoop::reassociate_reduction_chains() { for (LoopTreeIterator iter(_ltree_root); !iter.done(); iter.next()) { IdealLoopTree* loop = iter.current(); if (loop->is_innermost()) { - Node* loop_head = loop->head(); ReassociateReductionChains rrc(loop, this); - - for (DUIterator_Fast imax, i = loop_head->fast_outs(imax); i < imax; i++) { - Node* loop_head_use = loop_head->fast_out(i); - if (loop_head_use->is_Phi()) { - PhiNode* phi = loop_head_use->as_Phi(); - for (DUIterator j = phi->outs(); phi->has_out(j); j++) { - Node* n = phi->out(j); - if (rrc.transform(n, phi)) { - --j; - } - } - } - } + rrc.reassociate_chains(); } } } From 23329732ceebfe05ff46f7d24b8ccab4055253f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Galder=20Zamarre=C3=B1o?= Date: Wed, 4 Mar 2026 06:47:16 +0100 Subject: [PATCH 51/51] Add some documentation --- src/hotspot/share/opto/loopopts.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/hotspot/share/opto/loopopts.cpp b/src/hotspot/share/opto/loopopts.cpp index 1e7633055e9fe..cc66100f16085 100644 --- a/src/hotspot/share/opto/loopopts.cpp +++ b/src/hotspot/share/opto/loopopts.cpp @@ -4508,6 +4508,15 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old return true; } +// Reassociates latency-bound reduction loop chains for long Min/Max that have a shape like this: +// OP(A, OP(B, OP(C, Phi))) +// To become the following by shifting the Phi node to the front and shifting the rest of inputs: +// OP(Phi, OP(A, OP(B, C))) +// This transformation reduces latency thanks to an increase CPU-level parallel processing. +// This increased parallelism can produce register pressure as a side effect. +// This is why the optimization currently only applies to specific AddNode subclasses +// that can particularly suffer in certain scenarios, e.g. long Min/Max. +// Any attempt to expand this to other AddNode types should take this into consideration. class ReassociateReductionChains : public StackObj { public: ReassociateReductionChains(IdealLoopTree* loop, PhaseIdealLoop* phase) : _loop(loop), _phase(phase) {