8387197: C2: Improve klass_ptr_type in GraphKit::gen_instanceof() similarly to GraphKit::gen_checkcast()

Reviewed-by: qamai, vlivanov

src/hotspot/share/opto/graphKit.cpp

@@ -3240,9 +3240,10 @@ Node* GraphKit::maybe_cast_profiled_obj(Node* obj,
 Node* GraphKit::gen_instanceof(Node* obj, Node* superklass, bool safe_for_replace) {
   kill_dead_locals();           // Benefit all the uncommon traps
   assert( !stopped(), "dead parse path should be checked in callers" );
-  assert(!TypePtr::NULL_PTR->higher_equal(_gvn.type(superklass)->is_klassptr()),
+  const TypeKlassPtr* klass_ptr_type = _gvn.type(superklass)->isa_klassptr();
+  assert(klass_ptr_type != nullptr && !TypePtr::NULL_PTR->higher_equal(klass_ptr_type),
          "must check for not-null not-dead klass in callers");
-
+  const TypeKlassPtr* improved_klass_ptr_type = klass_ptr_type->try_improve();
   // Make the merge point
   enum { _obj_path = 1, _fail_path, _null_path, PATH_LIMIT };
   RegionNode* region = new RegionNode(PATH_LIMIT);
@@ -3278,11 +3279,10 @@ Node* GraphKit::gen_instanceof(Node* obj, Node* superklass, bool safe_for_replac
 
   // Do we know the type check always succeed?
   bool known_statically = false;
-  if (_gvn.type(superklass)->singleton()) {
-    const TypeKlassPtr* superk = _gvn.type(superklass)->is_klassptr();
+  if (improved_klass_ptr_type->singleton()) {
     const TypeKlassPtr* subk = _gvn.type(obj)->is_oopptr()->as_klass_type();
     if (subk->is_loaded()) {
-      int static_res = C->static_subtype_check(superk, subk);
+      int static_res = C->static_subtype_check(improved_klass_ptr_type, subk);
       known_statically = (static_res == Compile::SSC_always_true || static_res == Compile::SSC_always_false);
     }
   }
@@ -3305,7 +3305,11 @@ Node* GraphKit::gen_instanceof(Node* obj, Node* superklass, bool safe_for_replac
   }
 
   // Generate the subtype check
-  Node* not_subtype_ctrl = gen_subtype_check(not_null_obj, superklass);
+  Node* improved_superklass = superklass;
+  if (improved_klass_ptr_type != klass_ptr_type && improved_klass_ptr_type->singleton()) {
+    improved_superklass = makecon(improved_klass_ptr_type);
+  }
+  Node* not_subtype_ctrl = gen_subtype_check(not_null_obj, improved_superklass);
 
   // Plug in the success path to the general merge in slot 1.
   region->init_req(_obj_path, control());

test/hotspot/jtreg/compiler/parsing/TestInstanceOfImprovedKlassPtrType.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8387197
+ * @summary Verify that improving klass_ptr_type in GraphKit::gen_instanceof() allows
+ *          eliminating SubTypeCheckNode when the receiver implements an interface
+ *          unrelated to the checked class.
+ * @library /test/lib /
+ * @run driver ${test.main.class}
+ */
+
+package compiler.parsing;
+
+import compiler.lib.ir_framework.*;
+import jdk.test.lib.Asserts;
+
+public class TestInstanceOfImprovedKlassPtrType {
+    static abstract class B {}
+    static final class C extends B {}
+
+    interface I {}
+    static class D implements I {}
+    static class E implements I {}
+
+    public static void main(String[] args) {
+        TestFramework.run();
+    }
+
+    @DontInline
+    int testHelper2(Object o) {
+        return 1;
+    }
+
+    @Test
+    @IR(counts = {IRNode.SUBTYPE_CHECK, "1"},
+        phase = CompilePhase.AFTER_PARSING)
+    int test1(Object o) {
+        Object o1 = (I) o;
+        if (o1 instanceof B) {
+            return testHelper2(o1);
+        } else {
+            return 2;
+        }
+    }
+
+    @Run(test = "test1")
+    @Warmup(0)
+    void runTest() {
+        int sum = 0;
+        Object[] arr = new Object[] {new C(), new D(), new E()};
+        for (int i = 0; i < 3; i++){
+            Object o = arr[i];
+            if (o instanceof I) {
+                sum += test1(o);
+            } else {
+                sum += 3;
+            }
+        }
+        Asserts.assertEquals(sum, 7);
+        return;
+    }
+}