From bd8072d22ced9a75da0d6578df66be65f886c746 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Thu, 25 Jun 2026 09:20:20 +0000 Subject: [PATCH] 8379816: C2: Possible integer overflow in BCEscapeAnalyzer::iterate_blocks Reviewed-by: dlong, kvn --- src/hotspot/share/ci/bcEscapeAnalyzer.cpp | 36 ++-- src/hotspot/share/ci/bcEscapeAnalyzer.hpp | 6 + src/hotspot/share/ci/ciMethod.hpp | 4 +- src/hotspot/share/ci/ciMethodBlocks.hpp | 2 +- .../TestBCEscapeAnalyzerOverflow.java | 160 ++++++++++++++++++ 5 files changed, 195 insertions(+), 13 deletions(-) create mode 100644 test/hotspot/jtreg/compiler/escapeAnalysis/TestBCEscapeAnalyzerOverflow.java diff --git a/src/hotspot/share/ci/bcEscapeAnalyzer.cpp b/src/hotspot/share/ci/bcEscapeAnalyzer.cpp index 712f7af4139..fd4ac33eb69 100644 --- a/src/hotspot/share/ci/bcEscapeAnalyzer.cpp +++ b/src/hotspot/share/ci/bcEscapeAnalyzer.cpp @@ -34,6 +34,7 @@ #include "utilities/align.hpp" #include "utilities/bitMap.inline.hpp" #include "utilities/copy.hpp" +#include "utilities/integerCast.hpp" #ifndef PRODUCT #define TRACE_BCEA(level, code) \ @@ -1077,17 +1078,32 @@ void BCEscapeAnalyzer::merge_block_states(StateInfo *blockstates, ciBlock *dest, } } +bool BCEscapeAnalyzer::datasize_overflow(uint numblocks, uint stkSize, uint numLocals, size_t& datasize) { + uint64_t datacount64 = (uint64_t)(numblocks + 1) * (stkSize + numLocals); + if (datacount64 > SIZE_MAX / sizeof(ArgumentMap)) { + return true; + } + datasize = integer_cast_permit_tautology(datacount64 * sizeof(ArgumentMap)); + return false; +} + void BCEscapeAnalyzer::iterate_blocks(Arena *arena) { - int numblocks = _methodBlocks->num_blocks(); - int stkSize = _method->max_stack(); - int numLocals = _method->max_locals(); + uint numblocks = _methodBlocks->num_blocks(); + uint stkSize = _method->max_stack(); + uint numLocals = _method->max_locals(); StateInfo state; - int datacount = (numblocks + 1) * (stkSize + numLocals); - int datasize = datacount * sizeof(ArgumentMap); + size_t datasize; + if (datasize_overflow(numblocks, stkSize, numLocals, datasize)) { + _conservative = true; + return; + } + size_t datacount = datasize / sizeof(ArgumentMap); StateInfo *blockstates = (StateInfo *) arena->Amalloc(numblocks * sizeof(StateInfo)); ArgumentMap *statedata = (ArgumentMap *) arena->Amalloc(datasize); - for (int i = 0; i < datacount; i++) ::new ((void*)&statedata[i]) ArgumentMap(); + for (size_t i = 0; i < datacount; i++) { + ::new ((void*)&statedata[i]) ArgumentMap(); + } ArgumentMap *dp = statedata; state._vars = dp; dp += numLocals; @@ -1095,7 +1111,7 @@ void BCEscapeAnalyzer::iterate_blocks(Arena *arena) { dp += stkSize; state._initialized = false; state._max_stack = stkSize; - for (int i = 0; i < numblocks; i++) { + for (uint i = 0; i < numblocks; i++) { blockstates[i]._vars = dp; dp += numLocals; blockstates[i]._stack = dp; @@ -1142,7 +1158,7 @@ void BCEscapeAnalyzer::iterate_blocks(Arena *arena) { if (blk->is_handler() || blk->is_ret_target()) { // for an exception handler or a target of a ret instruction, we assume the worst case, // that any variable could contain any argument - for (int i = 0; i < numLocals; i++) { + for (uint i = 0; i < numLocals; i++) { state._vars[i] = allVars; } if (blk->is_handler()) { @@ -1155,7 +1171,7 @@ void BCEscapeAnalyzer::iterate_blocks(Arena *arena) { state._stack[i] = allVars; } } else { - for (int i = 0; i < numLocals; i++) { + for (uint i = 0; i < numLocals; i++) { state._vars[i] = blkState->_vars[i]; } for (int i = 0; i < blkState->_stack_height; i++) { @@ -1170,7 +1186,7 @@ void BCEscapeAnalyzer::iterate_blocks(Arena *arena) { DEBUG_ONLY(int handler_count = 0;) int blk_start = blk->start_bci(); int blk_end = blk->limit_bci(); - for (int i = 0; i < numblocks; i++) { + for (uint i = 0; i < numblocks; i++) { ciBlock *b = _methodBlocks->block(i); if (b->is_handler()) { int ex_start = b->ex_start_bci(); diff --git a/src/hotspot/share/ci/bcEscapeAnalyzer.hpp b/src/hotspot/share/ci/bcEscapeAnalyzer.hpp index b75cb6a56f4..7bdd4a58146 100644 --- a/src/hotspot/share/ci/bcEscapeAnalyzer.hpp +++ b/src/hotspot/share/ci/bcEscapeAnalyzer.hpp @@ -152,6 +152,12 @@ class BCEscapeAnalyzer : public ArenaObj { // Copy dependencies from this analysis into "deps" void copy_dependencies(Dependencies *deps); + // Returns true if the datasize computation for iterate_blocks would + // overflow, i.e. the allocation size exceeds what can be represented. + // On success, sets datasize to the computed allocation size in bytes. + // Extracted as a public static method for testability (JDK-8216486). + static bool datasize_overflow(uint numblocks, uint stkSize, uint numLocals, size_t& datasize); + #ifndef PRODUCT // dump escape information void dump(); diff --git a/src/hotspot/share/ci/ciMethod.hpp b/src/hotspot/share/ci/ciMethod.hpp index eecd9427585..c3805b4a054 100644 --- a/src/hotspot/share/ci/ciMethod.hpp +++ b/src/hotspot/share/ci/ciMethod.hpp @@ -76,8 +76,8 @@ class ciMethod : public ciMetadata { // Code attributes. int _code_size; - int _max_stack; - int _max_locals; + uint _max_stack; + u2 _max_locals; vmIntrinsicID _intrinsic_id; int _handler_count; int _interpreter_invocation_count; diff --git a/src/hotspot/share/ci/ciMethodBlocks.hpp b/src/hotspot/share/ci/ciMethodBlocks.hpp index f1b446c2a87..567ea1e39b4 100644 --- a/src/hotspot/share/ci/ciMethodBlocks.hpp +++ b/src/hotspot/share/ci/ciMethodBlocks.hpp @@ -38,7 +38,7 @@ private: Arena *_arena; GrowableArray *_blocks; ciBlock **_bci_to_block; - int _num_blocks; + u2 _num_blocks; int _code_size; void do_analysis(); diff --git a/test/hotspot/jtreg/compiler/escapeAnalysis/TestBCEscapeAnalyzerOverflow.java b/test/hotspot/jtreg/compiler/escapeAnalysis/TestBCEscapeAnalyzerOverflow.java new file mode 100644 index 00000000000..f33a16785d1 --- /dev/null +++ b/test/hotspot/jtreg/compiler/escapeAnalysis/TestBCEscapeAnalyzerOverflow.java @@ -0,0 +1,160 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/* + * @test + * @bug 8216486 + * @summary Verify BCEscapeAnalyzer handles methods where + * (numblocks+1)*(max_stack+max_locals) overflows a 32-bit int. + * On a UBSAN build the signed overflow would be caught as UB; + * on a normal build the test verifies no crash from the bogus + * allocation size that resulted from the overflow. + * + * @requires vm.compiler2.enabled + * + * @run main/othervm -Xcomp -XX:-TieredCompilation + * compiler.escapeAnalysis.TestBCEscapeAnalyzerOverflow + */ + +package compiler.escapeAnalysis; + +import java.lang.classfile.ClassFile; +import java.lang.classfile.Label; +import java.lang.constant.ClassDesc; +import java.lang.constant.ConstantDescs; +import java.lang.constant.MethodTypeDesc; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; + +public class TestBCEscapeAnalyzerOverflow { + + // Number of goto instructions in the generated method. + // Creates NUM_GOTOS + 1 basic blocks. With max_stack = 0xFFFF and + // max_locals = 0xFFFF the product (numblocks+1)*(max_stack+max_locals) + // is 16386 * 131070 = 2,147,713,020 which exceeds Integer.MAX_VALUE. + static final int NUM_GOTOS = 16384; + static final int TARGET_MAX_STACK = 0xFFFF; + static final int TARGET_MAX_LOCALS = 0xFFFF; + + static final ClassDesc CD_HELPER = + ClassDesc.of("compiler.escapeAnalysis.BCEscapeOverflowHelper"); + + public static void main(String[] args) throws Throwable { + byte[] classBytes = buildClass(); + MethodHandles.Lookup lookup = MethodHandles.lookup(); + Class cls = lookup.defineClass(classBytes); + + // caller() allocates an Object and passes it to bigMethod() via + // invokestatic. Under -Xcomp -XX:-TieredCompilation, C2 compiles + // caller() and invokes BCEscapeAnalyzer on bigMethod to determine + // whether the argument escapes. Without the fix the 32-bit + // overflow in iterate_blocks leads to undefined behavior. + var mh = lookup.findStatic(cls, "caller", + MethodType.methodType(void.class)); + mh.invoke(); + } + + /** + * Builds a minimal class (version 50, no StackMapTable needed) with: + * public static void bigMethod(Object o) -- pathological method + * public static void caller() -- calls bigMethod + * + * The ClassFile API generates the bytecode; max_stack and max_locals + * of bigMethod are then patched to the target overflow-triggering values. + */ + static byte[] buildClass() { + var mtd_Obj_void = MethodTypeDesc.of(ConstantDescs.CD_void, + ConstantDescs.CD_Object); + var mtd_void = MethodTypeDesc.of(ConstantDescs.CD_void); + + byte[] bytes = ClassFile.of(ClassFile.StackMapsOption.DROP_STACK_MAPS) + .build(CD_HELPER, cb -> { + cb.withVersion(50, 0); + cb.withFlags(ClassFile.ACC_PUBLIC | ClassFile.ACC_SUPER); + + // bigMethod(Object o): aload_0, pop, , return + cb.withMethod("bigMethod", mtd_Obj_void, + ClassFile.ACC_PUBLIC | ClassFile.ACC_STATIC, + mb -> mb.withCode(code -> { + code.aload(0); + code.pop(); + for (int i = 0; i < NUM_GOTOS; i++) { + Label next = code.newLabel(); + code.goto_(next); + code.labelBinding(next); + } + code.return_(); + })); + + // caller(): new Object → dup → invokespecial → + // invokestatic bigMethod → return + cb.withMethod("caller", mtd_void, + ClassFile.ACC_PUBLIC | ClassFile.ACC_STATIC, + mb -> mb.withCode(code -> { + code.new_(ConstantDescs.CD_Object); + code.dup(); + code.invokespecial(ConstantDescs.CD_Object, + "", mtd_void); + code.invokestatic(CD_HELPER, + "bigMethod", mtd_Obj_void); + code.return_(); + })); + }); + + patchBigMethodMaxes(bytes); + return bytes; + } + + /** + * Locates bigMethod's Code attribute and patches max_stack/max_locals + * to TARGET_MAX_STACK/TARGET_MAX_LOCALS. The ClassFile API computes + * small values (max_stack=1, max_locals=1); we inflate them to create + * the pathological overflow case. + * + * The Code attribute layout is: + * attribute_name_index(u2), attribute_length(u4), + * max_stack(u2), max_locals(u2), code_length(u4), code[...]... + * + * We search for bigMethod's unique code_length and patch the two u2 + * fields immediately before it. + */ + static void patchBigMethodMaxes(byte[] b) { + int expectedCodeLen = NUM_GOTOS * 3 + 3; + for (int i = 4; i <= b.length - 4; i++) { + int codeLen = ((b[i] & 0xFF) << 24) | ((b[i + 1] & 0xFF) << 16) + | ((b[i + 2] & 0xFF) << 8) | (b[i + 3] & 0xFF); + if (codeLen == expectedCodeLen) { + int ms = ((b[i - 4] & 0xFF) << 8) | (b[i - 3] & 0xFF); + int ml = ((b[i - 2] & 0xFF) << 8) | (b[i - 1] & 0xFF); + if (ms <= 2 && ml <= 2) { + b[i - 4] = (byte)(TARGET_MAX_STACK >>> 8); + b[i - 3] = (byte)(TARGET_MAX_STACK); + b[i - 2] = (byte)(TARGET_MAX_LOCALS >>> 8); + b[i - 1] = (byte)(TARGET_MAX_LOCALS); + return; + } + } + } + throw new RuntimeException("Could not find bigMethod Code attribute"); + } +}