8379816: C2: Possible integer overflow in BCEscapeAnalyzer::iterate_blocks

Reviewed-by: dlong, kvn

src/hotspot/share/ci/bcEscapeAnalyzer.cpp

@@ -34,6 +34,7 @@
 #include "utilities/align.hpp"
 #include "utilities/bitMap.inline.hpp"
 #include "utilities/copy.hpp"
+#include "utilities/integerCast.hpp"
 
 #ifndef PRODUCT
   #define TRACE_BCEA(level, code)                                            \
@@ -1077,17 +1078,32 @@ void BCEscapeAnalyzer::merge_block_states(StateInfo *blockstates, ciBlock *dest,
   }
 }
 
+bool BCEscapeAnalyzer::datasize_overflow(uint numblocks, uint stkSize, uint numLocals, size_t& datasize) {
+  uint64_t datacount64 = (uint64_t)(numblocks + 1) * (stkSize + numLocals);
+  if (datacount64 > SIZE_MAX / sizeof(ArgumentMap)) {
+    return true;
+  }
+  datasize = integer_cast_permit_tautology<size_t>(datacount64 * sizeof(ArgumentMap));
+  return false;
+}
+
 void BCEscapeAnalyzer::iterate_blocks(Arena *arena) {
-  int numblocks = _methodBlocks->num_blocks();
-  int stkSize   = _method->max_stack();
-  int numLocals = _method->max_locals();
+  uint numblocks = _methodBlocks->num_blocks();
+  uint stkSize   = _method->max_stack();
+  uint numLocals = _method->max_locals();
   StateInfo state;
 
-  int datacount = (numblocks + 1) * (stkSize + numLocals);
-  int datasize = datacount * sizeof(ArgumentMap);
+  size_t datasize;
+  if (datasize_overflow(numblocks, stkSize, numLocals, datasize)) {
+    _conservative = true;
+    return;
+  }
+  size_t datacount = datasize / sizeof(ArgumentMap);
   StateInfo *blockstates = (StateInfo *) arena->Amalloc(numblocks * sizeof(StateInfo));
   ArgumentMap *statedata  = (ArgumentMap *) arena->Amalloc(datasize);
-  for (int i = 0; i < datacount; i++) ::new ((void*)&statedata[i]) ArgumentMap();
+  for (size_t i = 0; i < datacount; i++) {
+    ::new ((void*)&statedata[i]) ArgumentMap();
+  }
   ArgumentMap *dp = statedata;
   state._vars = dp;
   dp += numLocals;
@@ -1095,7 +1111,7 @@ void BCEscapeAnalyzer::iterate_blocks(Arena *arena) {
   dp += stkSize;
   state._initialized = false;
   state._max_stack = stkSize;
-  for (int i = 0; i < numblocks; i++) {
+  for (uint i = 0; i < numblocks; i++) {
     blockstates[i]._vars = dp;
     dp += numLocals;
     blockstates[i]._stack = dp;
@@ -1142,7 +1158,7 @@ void BCEscapeAnalyzer::iterate_blocks(Arena *arena) {
     if (blk->is_handler() || blk->is_ret_target()) {
       // for an exception handler or a target of a ret instruction, we assume the worst case,
       // that any variable could contain any argument
-      for (int i = 0; i < numLocals; i++) {
+      for (uint i = 0; i < numLocals; i++) {
         state._vars[i] = allVars;
       }
       if (blk->is_handler()) {
@@ -1155,7 +1171,7 @@ void BCEscapeAnalyzer::iterate_blocks(Arena *arena) {
         state._stack[i] = allVars;
       }
     } else {
-      for (int i = 0; i < numLocals; i++) {
+      for (uint i = 0; i < numLocals; i++) {
         state._vars[i] = blkState->_vars[i];
       }
       for (int i = 0; i < blkState->_stack_height; i++) {
@@ -1170,7 +1186,7 @@ void BCEscapeAnalyzer::iterate_blocks(Arena *arena) {
       DEBUG_ONLY(int handler_count = 0;)
       int blk_start = blk->start_bci();
       int blk_end = blk->limit_bci();
-      for (int i = 0; i < numblocks; i++) {
+      for (uint i = 0; i < numblocks; i++) {
         ciBlock *b = _methodBlocks->block(i);
         if (b->is_handler()) {
           int ex_start = b->ex_start_bci();

src/hotspot/share/ci/bcEscapeAnalyzer.hpp

@@ -152,6 +152,12 @@ class BCEscapeAnalyzer : public ArenaObj {
   // Copy dependencies from this analysis into "deps"
   void copy_dependencies(Dependencies *deps);
 
+  // Returns true if the datasize computation for iterate_blocks would
+  // overflow, i.e. the allocation size exceeds what can be represented.
+  // On success, sets datasize to the computed allocation size in bytes.
+  // Extracted as a public static method for testability (JDK-8216486).
+  static bool datasize_overflow(uint numblocks, uint stkSize, uint numLocals, size_t& datasize);
+
 #ifndef PRODUCT
   // dump escape information
   void dump();

src/hotspot/share/ci/ciMethod.hpp

@@ -76,8 +76,8 @@ class ciMethod : public ciMetadata {
 
   // Code attributes.
   int _code_size;
-  int _max_stack;
-  int _max_locals;
+  uint _max_stack;
+  u2 _max_locals;
   vmIntrinsicID _intrinsic_id;
   int _handler_count;
   int _interpreter_invocation_count;

src/hotspot/share/ci/ciMethodBlocks.hpp

@@ -38,7 +38,7 @@ private:
   Arena *_arena;
   GrowableArray<ciBlock *>  *_blocks;
   ciBlock  **_bci_to_block;
-  int _num_blocks;
+  u2 _num_blocks;
   int _code_size;
 
   void do_analysis();

test/hotspot/jtreg/compiler/escapeAnalysis/TestBCEscapeAnalyzerOverflow.java

@@ -0,0 +1,160 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8216486
+ * @summary Verify BCEscapeAnalyzer handles methods where
+ *          (numblocks+1)*(max_stack+max_locals) overflows a 32-bit int.
+ *          On a UBSAN build the signed overflow would be caught as UB;
+ *          on a normal build the test verifies no crash from the bogus
+ *          allocation size that resulted from the overflow.
+ *
+ * @requires vm.compiler2.enabled
+ *
+ * @run main/othervm -Xcomp -XX:-TieredCompilation
+ *      compiler.escapeAnalysis.TestBCEscapeAnalyzerOverflow
+ */
+
+package compiler.escapeAnalysis;
+
+import java.lang.classfile.ClassFile;
+import java.lang.classfile.Label;
+import java.lang.constant.ClassDesc;
+import java.lang.constant.ConstantDescs;
+import java.lang.constant.MethodTypeDesc;
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.MethodType;
+
+public class TestBCEscapeAnalyzerOverflow {
+
+    // Number of goto instructions in the generated method.
+    // Creates NUM_GOTOS + 1 basic blocks.  With max_stack = 0xFFFF and
+    // max_locals = 0xFFFF the product (numblocks+1)*(max_stack+max_locals)
+    // is 16386 * 131070 = 2,147,713,020 which exceeds Integer.MAX_VALUE.
+    static final int NUM_GOTOS = 16384;
+    static final int TARGET_MAX_STACK = 0xFFFF;
+    static final int TARGET_MAX_LOCALS = 0xFFFF;
+
+    static final ClassDesc CD_HELPER =
+        ClassDesc.of("compiler.escapeAnalysis.BCEscapeOverflowHelper");
+
+    public static void main(String[] args) throws Throwable {
+        byte[] classBytes = buildClass();
+        MethodHandles.Lookup lookup = MethodHandles.lookup();
+        Class<?> cls = lookup.defineClass(classBytes);
+
+        // caller() allocates an Object and passes it to bigMethod() via
+        // invokestatic.  Under -Xcomp -XX:-TieredCompilation, C2 compiles
+        // caller() and invokes BCEscapeAnalyzer on bigMethod to determine
+        // whether the argument escapes.  Without the fix the 32-bit
+        // overflow in iterate_blocks leads to undefined behavior.
+        var mh = lookup.findStatic(cls, "caller",
+                     MethodType.methodType(void.class));
+        mh.invoke();
+    }
+
+    /**
+     * Builds a minimal class (version 50, no StackMapTable needed) with:
+     *   public static void bigMethod(Object o)  -- pathological method
+     *   public static void caller()              -- calls bigMethod
+     *
+     * The ClassFile API generates the bytecode; max_stack and max_locals
+     * of bigMethod are then patched to the target overflow-triggering values.
+     */
+    static byte[] buildClass() {
+        var mtd_Obj_void = MethodTypeDesc.of(ConstantDescs.CD_void,
+                                             ConstantDescs.CD_Object);
+        var mtd_void = MethodTypeDesc.of(ConstantDescs.CD_void);
+
+        byte[] bytes = ClassFile.of(ClassFile.StackMapsOption.DROP_STACK_MAPS)
+            .build(CD_HELPER, cb -> {
+                cb.withVersion(50, 0);
+                cb.withFlags(ClassFile.ACC_PUBLIC | ClassFile.ACC_SUPER);
+
+                // bigMethod(Object o): aload_0, pop, <goto chain>, return
+                cb.withMethod("bigMethod", mtd_Obj_void,
+                    ClassFile.ACC_PUBLIC | ClassFile.ACC_STATIC,
+                    mb -> mb.withCode(code -> {
+                        code.aload(0);
+                        code.pop();
+                        for (int i = 0; i < NUM_GOTOS; i++) {
+                            Label next = code.newLabel();
+                            code.goto_(next);
+                            code.labelBinding(next);
+                        }
+                        code.return_();
+                    }));
+
+                // caller(): new Object → dup → invokespecial <init> →
+                //           invokestatic bigMethod → return
+                cb.withMethod("caller", mtd_void,
+                    ClassFile.ACC_PUBLIC | ClassFile.ACC_STATIC,
+                    mb -> mb.withCode(code -> {
+                        code.new_(ConstantDescs.CD_Object);
+                        code.dup();
+                        code.invokespecial(ConstantDescs.CD_Object,
+                            "<init>", mtd_void);
+                        code.invokestatic(CD_HELPER,
+                            "bigMethod", mtd_Obj_void);
+                        code.return_();
+                    }));
+            });
+
+        patchBigMethodMaxes(bytes);
+        return bytes;
+    }
+
+    /**
+     * Locates bigMethod's Code attribute and patches max_stack/max_locals
+     * to TARGET_MAX_STACK/TARGET_MAX_LOCALS.  The ClassFile API computes
+     * small values (max_stack=1, max_locals=1); we inflate them to create
+     * the pathological overflow case.
+     *
+     * The Code attribute layout is:
+     *   attribute_name_index(u2), attribute_length(u4),
+     *   max_stack(u2), max_locals(u2), code_length(u4), code[...]...
+     *
+     * We search for bigMethod's unique code_length and patch the two u2
+     * fields immediately before it.
+     */
+    static void patchBigMethodMaxes(byte[] b) {
+        int expectedCodeLen = NUM_GOTOS * 3 + 3;
+        for (int i = 4; i <= b.length - 4; i++) {
+            int codeLen = ((b[i] & 0xFF) << 24) | ((b[i + 1] & 0xFF) << 16)
+                        | ((b[i + 2] & 0xFF) << 8) | (b[i + 3] & 0xFF);
+            if (codeLen == expectedCodeLen) {
+                int ms = ((b[i - 4] & 0xFF) << 8) | (b[i - 3] & 0xFF);
+                int ml = ((b[i - 2] & 0xFF) << 8) | (b[i - 1] & 0xFF);
+                if (ms <= 2 && ml <= 2) {
+                    b[i - 4] = (byte)(TARGET_MAX_STACK >>> 8);
+                    b[i - 3] = (byte)(TARGET_MAX_STACK);
+                    b[i - 2] = (byte)(TARGET_MAX_LOCALS >>> 8);
+                    b[i - 1] = (byte)(TARGET_MAX_LOCALS);
+                    return;
+                }
+            }
+        }
+        throw new RuntimeException("Could not find bigMethod Code attribute");
+    }
+}