8353290: C2: Refactor PhaseIdealLoop::is_counted_loop()

Reviewed-by: chagedorn, roland

src/hotspot/share/opto/loopnode.hpp

@@ -273,11 +273,6 @@ public:
   CountedLoopEndNode* loopexit() const { return (CountedLoopEndNode*) BaseCountedLoopNode::loopexit(); }
   int   stride_con() const;
 
-  // Match increment with optional truncation
-  static Node*
-  match_incr_with_optional_truncation(Node* expr, Node** trunc1, Node** trunc2, const TypeInteger** trunc_type,
-                                      BasicType bt);
-
   // A 'main' loop has a pre-loop and a post-loop.  The 'main' loop
   // can run short a few iterations and may start a few iterations in.
   // It will be RCE'd and unrolled and aligned.
@@ -1029,8 +1024,6 @@ private:
   void rewire_old_target_loop_entry_dependency_to_new_entry(CountedLoopNode* target_loop_head,
                                                             const Node* old_target_loop_entry,
                                                             uint node_index_before_new_assertion_predicate_nodes);
-  void insert_loop_limit_check_predicate(ParsePredicateSuccessProj* loop_limit_check_parse_proj, Node* cmp_limit,
-                                         Node* bol);
   void log_loop_tree();
 
 public:
@@ -1294,7 +1287,7 @@ public:
   void recompute_dom_depth();
 
   // Is safept not required by an outer loop?
-  bool is_deleteable_safept(Node* sfpt);
+  bool is_deleteable_safept(Node* sfpt) const;
 
   // Replace parallel induction variable (parallel to trip counter)
   void replace_parallel_iv(IdealLoopTree *loop);
@@ -1345,21 +1338,109 @@ public:
   // Per-Node transform
   virtual Node* transform(Node* n) { return nullptr; }
 
-  Node* loop_exit_control(Node* x, IdealLoopTree* loop);
-  Node* loop_exit_test(Node* back_control, IdealLoopTree* loop, Node*& incr, Node*& limit, BoolTest::mask& bt, float& cl_prob);
-  Node* loop_iv_incr(Node* incr, Node* x, IdealLoopTree* loop, Node*& phi_incr);
-  Node* loop_iv_stride(Node* incr, Node*& xphi);
-  PhiNode* loop_iv_phi(Node* xphi, Node* phi_incr, Node* x);
+  Node* loop_exit_control(const IdealLoopTree* loop) const;
 
-  bool is_counted_loop(Node* x, IdealLoopTree*&loop, BasicType iv_bt);
+  class LoopExitTest {
+    bool _is_valid;
+
+    const Node* _back_control;
+    const IdealLoopTree* _loop;
+    PhaseIdealLoop* _phase;
+
+    Node* _cmp;
+    Node* _incr;
+    Node* _limit;
+    BoolTest::mask _mask;
+    float _cl_prob;
+
+  public:
+    LoopExitTest(const Node* back_control, const IdealLoopTree* loop, PhaseIdealLoop* phase) :
+      _is_valid(false),
+      _back_control(back_control),
+      _loop(loop),
+      _phase(phase),
+      _cmp(nullptr),
+      _incr(nullptr),
+      _limit(nullptr),
+      _mask(BoolTest::illegal),
+      _cl_prob(0.0f) {}
+
+    void build();
+    void canonicalize_mask(jlong stride_con);
+
+    bool is_valid_with_bt(BasicType bt) const {
+      return _is_valid && _cmp != nullptr && _cmp->Opcode() == Op_Cmp(bt);
+    }
+
+    bool should_include_limit() const { return _mask == BoolTest::le || _mask == BoolTest::ge; }
+
+    CmpNode* cmp() const { return _cmp->as_Cmp(); }
+    Node* incr() const { return _incr; }
+    Node* limit() const { return _limit; }
+    BoolTest::mask mask() const { return _mask; }
+    float cl_prob() const { return _cl_prob; }
+  };
+
+  class LoopIVIncr {
+    bool _is_valid;
+
+    const Node* _head;
+    const IdealLoopTree* _loop;
+
+    Node* _incr;
+    Node* _phi_incr;
+
+  public:
+    LoopIVIncr(const Node* head, const IdealLoopTree* loop) :
+      _is_valid(false),
+      _head(head),
+      _loop(loop),
+      _incr(nullptr),
+      _phi_incr(nullptr) {}
+
+    void build(Node* old_incr);
+
+    bool is_valid() const { return _is_valid; }
+    bool is_valid_with_bt(const BasicType bt) const {
+      return _is_valid && _incr->Opcode() == Op_Add(bt);
+    }
+
+    Node* incr() const { return _incr; }
+    Node* phi_incr() const { return _phi_incr; }
+  };
+
+  class LoopIVStride {
+    bool _is_valid;
+
+    BasicType _iv_bt;
+    Node* _stride_node;
+    Node* _xphi;
+
+  public:
+    LoopIVStride(BasicType iv_bt) :
+      _is_valid(false),
+      _iv_bt(iv_bt),
+      _stride_node(nullptr),
+      _xphi(nullptr) {}
+
+    void build(const Node* incr);
+
+    bool is_valid() const { return _is_valid && _stride_node != nullptr; }
+    Node* stride_node() const { return _stride_node; }
+    Node* xphi() const { return _xphi; }
+
+    jlong compute_non_zero_stride_con(BoolTest::mask mask, BasicType iv_bt) const;
+  };
+
+  static PhiNode* loop_iv_phi(const Node* xphi, const Node* phi_incr, const Node* head);
+
+  bool try_convert_to_counted_loop(Node* head, IdealLoopTree*& loop, BasicType iv_bt);
 
   Node* loop_nest_replace_iv(Node* iv_to_replace, Node* inner_iv, Node* outer_phi, Node* inner_head, BasicType bt);
   bool create_loop_nest(IdealLoopTree* loop, Node_List &old_new);
-#ifdef ASSERT
-  bool convert_to_long_loop(Node* cmp, Node* phi, IdealLoopTree* loop);
-#endif
+
   void add_parse_predicate(Deoptimization::DeoptReason reason, Node* inner_head, IdealLoopTree* loop, SafePointNode* sfpt);
-  SafePointNode* find_safepoint(Node* back_control, Node* x, IdealLoopTree* loop);
+  SafePointNode* find_safepoint(Node* back_control, const Node* head, const IdealLoopTree* loop);
 
   void add_parse_predicates(IdealLoopTree* outer_ilt, LoopNode* inner_head, SafePointNode* cloned_sfpt);
 
@@ -1496,8 +1577,6 @@ public:
   Node* clone_nodes_with_same_ctrl(Node* start_node, ProjNode* old_uncommon_proj, Node* new_uncommon_proj);
   void fix_cloned_data_node_controls(const ProjNode* orig, Node* new_uncommon_proj,
                                      const OrigToNewHashtable& orig_to_clone);
-  bool has_dominating_loop_limit_check(Node* init_trip, Node* limit, jlong stride_con, BasicType iv_bt,
-                                       Node* loop_entry);
 
  public:
   void register_control(Node* n, IdealLoopTree *loop, Node* pred, bool update_body = true);
@@ -1755,12 +1834,6 @@ public:
                        Node*& shift, Node*& offset);
 
 private:
-  // Return a type based on condition control flow
-  const TypeInt* filtered_type( Node *n, Node* n_ctrl);
-  const TypeInt* filtered_type( Node *n ) { return filtered_type(n, nullptr); }
- // Helpers for filtered type
-  const TypeInt* filtered_type_from_dominators( Node* val, Node *val_ctrl);
-
   // Helper functions
   Node *spinup( Node *iff, Node *new_false, Node *new_true, Node *region, Node *phi, small_cache *cache );
   Node *find_use_block( Node *use, Node *def, Node *old_false, Node *new_false, Node *old_true, Node *new_true );
@@ -1889,7 +1962,6 @@ public:
   static int _loop_work;        // Sum of PhaseIdealLoop x _unique
   static volatile int _long_loop_candidates;
   static volatile int _long_loop_nests;
-  static volatile int _long_loop_counted_loops;
 #endif
 
 #ifdef ASSERT
@@ -1901,7 +1973,7 @@ public:
 
   void rpo(Node* start, Node_Stack &stk, VectorSet &visited, Node_List &rpo_list) const;
 
-  void check_counted_loop_shape(IdealLoopTree* loop, Node* x, BasicType bt) NOT_DEBUG_RETURN;
+  void check_counted_loop_shape(IdealLoopTree* loop, Node* head, BasicType bt) NOT_DEBUG_RETURN;
 
   LoopNode* create_inner_head(IdealLoopTree* loop, BaseCountedLoopNode* head, IfNode* exit_test);
 
@@ -1979,6 +2051,146 @@ public:
   ConNode* zerocon(BasicType bt);
 };
 
+class CountedLoopConverter {
+  friend class PhaseIdealLoop;
+
+  // Match increment with optional truncation
+  class TruncatedIncrement {
+    bool _is_valid;
+
+    BasicType _bt;
+
+    Node* _incr;
+    Node* _outer_trunc;
+    Node* _inner_trunc;
+    const TypeInteger* _trunc_type;
+
+  public:
+    TruncatedIncrement(BasicType bt) :
+      _is_valid(false),
+      _bt(bt),
+      _incr(nullptr),
+      _outer_trunc(nullptr),
+      _inner_trunc(nullptr),
+      _trunc_type(nullptr) {}
+
+    void build(Node* expr);
+
+    bool is_valid() const { return _is_valid; }
+    Node* incr() const { return _incr; }
+
+    // Optional truncation for: CHAR: (i+1)&0x7fff, BYTE: ((i+1)<<8)>>8, or SHORT: ((i+1)<<16)>>16
+    Node* outer_trunc() const { return _outer_trunc; } // the outermost truncating node (either the & or the final >>)
+    Node* inner_trunc() const { return _inner_trunc; } // the inner truncating node, if applicable (the << in a <</>> pair)
+    const TypeInteger* trunc_type() const { return _trunc_type; }
+  };
+
+  class LoopStructure {
+    bool _is_valid;
+
+    const Node* _head;
+    const IdealLoopTree* _loop;
+    PhaseIdealLoop* _phase;
+    BasicType _iv_bt;
+
+    Node* _back_control;
+    PhaseIdealLoop::LoopExitTest _exit_test;
+    PhaseIdealLoop::LoopIVIncr _iv_incr;
+    TruncatedIncrement _truncated_increment;
+    PhaseIdealLoop::LoopIVStride _stride;
+    PhiNode* _phi;
+    SafePointNode* _safepoint;
+
+  public:
+    LoopStructure(const Node* head, const IdealLoopTree* loop, PhaseIdealLoop* phase, const BasicType iv_bt) :
+      _is_valid(false),
+      _head(head),
+      _loop(loop),
+      _phase(phase),
+      _iv_bt(iv_bt),
+      _back_control(_phase->loop_exit_control(_loop)),
+      _exit_test(_back_control, _loop, _phase),
+      _iv_incr(_head, _loop),
+      _truncated_increment(_iv_bt),
+      _stride(PhaseIdealLoop::LoopIVStride(_iv_bt)),
+      _phi(nullptr),
+      _safepoint(nullptr) {}
+
+    void build();
+
+    jlong final_limit_correction() const; // compute adjusted loop limit correction
+    bool is_infinite_loop() const;
+
+    bool is_valid() const { return _is_valid; }
+
+    Node* back_control() const { return _back_control; }
+    PhaseIdealLoop::LoopExitTest& exit_test() { return _exit_test; }
+    PhaseIdealLoop::LoopIVIncr& iv_incr() { return _iv_incr; }
+    TruncatedIncrement& truncated_increment() { return _truncated_increment; }
+    PhaseIdealLoop::LoopIVStride& stride() { return _stride; }
+    PhiNode* phi() const { return _phi; }
+    SafePointNode* sfpt() const { return _safepoint; }
+    jlong stride_con() const { return _stride.compute_non_zero_stride_con(_exit_test.mask(), _iv_bt); }
+    Node* limit() const { return _exit_test.limit(); }
+  };
+
+  PhaseIdealLoop* const _phase;
+  Node* const _head;
+  IdealLoopTree* const _loop;
+  const BasicType _iv_bt;
+
+  LoopStructure _structure;
+  bool _should_insert_stride_overflow_limit_check = false;
+  bool _should_insert_init_trip_limit_check = false;
+
+  DEBUG_ONLY(bool _checked_for_counted_loop = false;)
+
+  // stats for PhaseIdealLoop::print_statistics()
+  static volatile int _long_loop_counted_loops;
+
+  // Return a type based on condition control flow
+  const TypeInt* filtered_type(Node* n, Node* n_ctrl);
+  const TypeInt* filtered_type(Node* n) { return filtered_type(n, nullptr); }
+  // Helpers for filtered type
+  const TypeInt* filtered_type_from_dominators(Node* val, Node* val_ctrl);
+
+  void insert_loop_limit_check_predicate(const ParsePredicateSuccessProj* loop_limit_check_parse_proj, Node* bol) const;
+  void insert_stride_overflow_limit_check() const;
+  void insert_init_trip_limit_check() const;
+  bool has_dominating_loop_limit_check(Node* init_trip, Node* limit, jlong stride_con, BasicType iv_bt,
+                                       Node* loop_entry) const;
+
+  bool is_iv_overflowing(const TypeInteger* init_t, jlong stride_con, Node* phi_increment, BoolTest::mask mask) const;
+  bool has_truncation_wrap(const TruncatedIncrement& truncation, Node* phi, jlong stride_con);
+  SafePointNode* find_safepoint(Node* iftrue);
+  bool is_safepoint_invalid(SafePointNode* sfpt) const;
+
+ public:
+  CountedLoopConverter(PhaseIdealLoop* phase, Node* head, IdealLoopTree* loop, const BasicType iv_bt)
+      : _phase(phase),
+        _head(head),
+        _loop(loop),
+        _iv_bt(iv_bt),
+        _structure(LoopStructure(_head, _loop, _phase, _iv_bt)) {
+    assert(phase != nullptr, "must be"); // Fail early if mandatory parameters are null.
+    assert(head != nullptr, "must be");
+    assert(loop != nullptr, "must be");
+    assert(iv_bt == T_INT || iv_bt == T_LONG, "either int or long loops");
+  }
+
+  bool is_counted_loop();
+  IdealLoopTree* convert();
+
+  DEBUG_ONLY(bool should_stress_long_counted_loop();)
+  DEBUG_ONLY(bool stress_long_counted_loop();)
+
+  enum StrideOverflowState {
+    Overflow          = -1,
+    NoOverflow        = 0,
+    RequireLimitCheck = 1
+  };
+  static StrideOverflowState check_stride_overflow(jlong final_correction, const TypeInteger* limit_t, BasicType bt);
+};
 
 class AutoNodeBudget : public StackObj
 {

src/hotspot/share/opto/loopopts.cpp

@@ -2829,8 +2829,6 @@ void PhaseIdealLoop::clone_loop_body(const Node_List& body, Node_List &old_new,
 // with an optional truncation (left-shift followed by a right-shift)
 // of the add. Returns zero if not an iv.
 int PhaseIdealLoop::stride_of_possible_iv(Node* iff) {
-  Node* trunc1 = nullptr;
-  Node* trunc2 = nullptr;
   const TypeInteger* ttype = nullptr;
   if (!iff->is_If() || iff->in(1) == nullptr || !iff->in(1)->is_Bool()) {
     return 0;
@@ -2851,23 +2849,23 @@ int PhaseIdealLoop::stride_of_possible_iv(Node* iff) {
     Node* phi = cmp1;
     for (uint i = 1; i < phi->req(); i++) {
       Node* in = phi->in(i);
-      Node* add = CountedLoopNode::match_incr_with_optional_truncation(in,
-                                &trunc1, &trunc2, &ttype, T_INT);
-      if (add && add->in(1) == phi) {
-        add2 = add->in(2);
+      CountedLoopConverter::TruncatedIncrement add(T_INT);
+      add.build(in);
+      if (add.is_valid() && add.incr()->in(1) == phi) {
+        add2 = add.incr()->in(2);
         break;
       }
     }
   } else {
     // (If (Bool (CmpX addtrunc:(Optional-trunc((AddI (Phi ...addtrunc...) add2)) )))
     Node* addtrunc = cmp1;
-    Node* add = CountedLoopNode::match_incr_with_optional_truncation(addtrunc,
-                                &trunc1, &trunc2, &ttype, T_INT);
-    if (add && add->in(1)->is_Phi()) {
-      Node* phi = add->in(1);
+    CountedLoopConverter::TruncatedIncrement add(T_INT);
+    add.build(addtrunc);
+    if (add.is_valid() && add.incr()->in(1)->is_Phi()) {
+      Node* phi = add.incr()->in(1);
       for (uint i = 1; i < phi->req(); i++) {
         if (phi->in(i) == addtrunc) {
-          add2 = add->in(2);
+          add2 = add.incr()->in(2);
           break;
         }
       }
@@ -4294,54 +4292,50 @@ bool PhaseIdealLoop::duplicate_loop_backedge(IdealLoopTree *loop, Node_List &old
 #endif //ASSERT
   {
     // Is the shape of the loop that of a counted loop...
-    Node* back_control = loop_exit_control(head, loop);
+    Node* back_control = loop_exit_control(loop);
     if (back_control == nullptr) {
       return false;
     }
 
-    BoolTest::mask bt = BoolTest::illegal;
-    float cl_prob = 0;
-    Node* incr = nullptr;
-    Node* limit = nullptr;
-    Node* cmp = loop_exit_test(back_control, loop, incr, limit, bt, cl_prob);
-    if (cmp == nullptr || cmp->Opcode() != Op_CmpI) {
+    LoopExitTest loop_exit(back_control, loop, this);
+    loop_exit.build();
+    if (!loop_exit.is_valid_with_bt(T_INT)) {
       return false;
     }
 
+    const Node* loop_incr = loop_exit.incr();
+
     // With an extra phi for the candidate iv?
     // Or the region node is the loop head
-    if (!incr->is_Phi() || incr->in(0) == head) {
+    if (!loop_incr->is_Phi() || loop_incr->in(0) == head) {
       return false;
     }
 
     PathFrequency pf(head, this);
-    region = incr->in(0);
+    region = loop_incr->in(0);
 
     // Go over all paths for the extra phi's region and see if that
     // path is frequent enough and would match the expected iv shape
     // if the extra phi is removed
     inner = 0;
-    for (uint i = 1; i < incr->req(); ++i) {
-      Node* in = incr->in(i);
-      Node* trunc1 = nullptr;
-      Node* trunc2 = nullptr;
-      const TypeInteger* iv_trunc_t = nullptr;
-      Node* orig_in = in;
-      if (!(in = CountedLoopNode::match_incr_with_optional_truncation(in, &trunc1, &trunc2, &iv_trunc_t, T_INT))) {
+    for (uint i = 1; i < loop_incr->req(); ++i) {
+      CountedLoopConverter::TruncatedIncrement increment(T_INT);
+      increment.build(loop_incr->in(i));
+      if (!increment.is_valid()) {
         continue;
       }
-      assert(in->Opcode() == Op_AddI, "wrong increment code");
-      Node* xphi = nullptr;
-      Node* stride = loop_iv_stride(in, xphi);
+      assert(increment.incr()->Opcode() == Op_AddI, "wrong increment code");
 
-      if (stride == nullptr) {
+      LoopIVStride stride = LoopIVStride(T_INT);
+      stride.build(increment.incr());
+      if (!stride.is_valid()) {
         continue;
       }
 
-      PhiNode* phi = loop_iv_phi(xphi, nullptr, head);
+      PhiNode* phi = loop_iv_phi(stride.xphi(), nullptr, head);
       if (phi == nullptr ||
-          (trunc1 == nullptr && phi->in(LoopNode::LoopBackControl) != incr) ||
-          (trunc1 != nullptr && phi->in(LoopNode::LoopBackControl) != trunc1)) {
+          (increment.outer_trunc() == nullptr && phi->in(LoopNode::LoopBackControl) != loop_exit.incr()) ||
+          (increment.outer_trunc() != nullptr && phi->in(LoopNode::LoopBackControl) != increment.outer_trunc())) {
         return false;
       }
 

test/hotspot/jtreg/compiler/loopopts/TestStressLongCountedLoopLimitChecks.java

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2026 IBM and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package compiler.loopopts;
+
+import jdk.test.lib.Asserts;
+import jdk.test.lib.process.OutputAnalyzer;
+import jdk.test.lib.process.ProcessTools;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.stream.Stream;
+
+/**
+ * @test
+ * @bug 8353290
+ * @summary test loop limit checks are inserted when stressing int counted loops to long counted loops
+ * @library /test/lib
+ * @requires vm.debug == true
+ * @run driver compiler.loopopts.TestStressLongCountedLoopLimitChecks
+ */
+public class TestStressLongCountedLoopLimitChecks {
+    public static void main(String[] args) throws Exception {
+        test(BasicLauncher.class, 1, "-XX:StressLongCountedLoop=0");
+        test(BasicLauncher.class, 1, "-XX:StressLongCountedLoop=2000000");
+
+        test(LargeStrideLauncher.class, 2, "-XX:StressLongCountedLoop=0");
+        test(LargeStrideLauncher.class, 2, "-XX:StressLongCountedLoop=2000000");
+    }
+
+    private static void test(Class launcher, int limitChecks, String... flags) throws IOException {
+        ProcessBuilder pb = ProcessTools.createLimitedTestJavaProcessBuilder(
+                Stream.concat(Arrays.stream(flags), Stream.of(
+                        "-XX:+IgnoreUnrecognizedVMOptions",
+                        "-XX:+TraceLoopLimitCheck",
+                        "-XX:CompileOnly=" + launcher.getName() + "::test*",
+                        "-Xcomp",
+                        launcher.getName()
+                )).toList()
+        );
+
+        OutputAnalyzer analyzer = new OutputAnalyzer(pb.start());
+        analyzer.shouldHaveExitValue(0);
+
+        analyzer.outputTo(System.out);
+        analyzer.errorTo(System.err);
+
+        Asserts.assertEQ(
+                limitChecks,
+                (int) analyzer.asLines().stream().filter(
+                        l -> l.trim().matches("Counted Loop Limit Check generated:")
+                ).count(),
+                "wrong numbers of loop limit checks"
+        );
+    }
+
+    public static class BasicLauncher {
+        static int x, y, z;
+
+        public static void main(String[] args) throws Exception {
+            test();
+        }
+
+        static void test() {
+            int i = x; // Any int
+            do {
+                x += y;
+                i++; // Could overflow and thus we need a Loop Limit Check Predicate "i < z"
+            } while (i < z);
+        }
+    }
+
+    public static class LargeStrideLauncher {
+        static final int STRIDE = 100_000;
+
+        public static void main(String[] args) throws Exception {
+            Asserts.assertEQ(10_000_000L / STRIDE, test(0, 10_000_000), "loop not stopped");
+            Asserts.assertEQ(-1L, test(0, Integer.MAX_VALUE), "loop stopped prematurely");
+        }
+
+        static long ONE = 1; // Just so the compiler doesn't try to IV replace the whole thing
+
+        public static long test(int init, int limit) {
+            final int stride = 100_000;
+
+            long iterations = 0;
+            for (int i = init; i < limit; i += 100000) {
+                iterations += ONE;
+
+                if (iterations > (limit / stride) + 1) { // No it's not stopping, as we should expect.
+                    return -1;
+                }
+            }
+            return iterations; // Possibly stopping prematurely.
+        }
+    }
+}