From 52e2878cffd9cb704ad773b841dbab0d17eba896 Mon Sep 17 00:00:00 2001
From: Vicente Romero <vromero@openjdk.org>
Date: Thu, 16 Nov 2023 16:41:58 +0000
Subject: [PATCH] 8319987: compilation of sealed classes leads to infinite
 recursion

Reviewed-by: jlahoda
---
 .../com/sun/tools/javac/code/Types.java       | 59 +++++++++++--------
 .../CyclicHierarchyTest.java                  | 12 ++++
 .../CyclicHierarchyTest.out                   |  3 +
 3 files changed, 49 insertions(+), 25 deletions(-)
 create mode 100644 test/langtools/tools/javac/sealed/erroneous_hierarchy/CyclicHierarchyTest.java
 create mode 100644 test/langtools/tools/javac/sealed/erroneous_hierarchy/CyclicHierarchyTest.out

diff --git a/src/jdk.compiler/share/classes/com/sun/tools/javac/code/Types.java b/src/jdk.compiler/share/classes/com/sun/tools/javac/code/Types.java
index 980ec3d9003..7783c7fb4f4 100644
--- a/src/jdk.compiler/share/classes/com/sun/tools/javac/code/Types.java
+++ b/src/jdk.compiler/share/classes/com/sun/tools/javac/code/Types.java
@@ -1665,37 +1665,46 @@ public class Types {
                 && (t.tsym.isSealed() || s.tsym.isSealed())) {
             return (t.isCompound() || s.isCompound()) ?
                     true :
-                    !areDisjoint((ClassSymbol)t.tsym, (ClassSymbol)s.tsym);
+                    !(new DisjointChecker().areDisjoint((ClassSymbol)t.tsym, (ClassSymbol)s.tsym));
         }
         return result;
     }
     // where
-        private boolean areDisjoint(ClassSymbol ts, ClassSymbol ss) {
-            if (isSubtype(erasure(ts.type), erasure(ss.type))) {
-                return false;
-            }
-            // if both are classes or both are interfaces, shortcut
-            if (ts.isInterface() == ss.isInterface() && isSubtype(erasure(ss.type), erasure(ts.type))) {
-                return false;
-            }
-            if (ts.isInterface() && !ss.isInterface()) {
-                /* so ts is interface but ss is a class
-                 * an interface is disjoint from a class if the class is disjoint form the interface
+        class DisjointChecker {
+            Set<Pair<ClassSymbol, ClassSymbol>> pairsSeen = new HashSet<>();
+            private boolean areDisjoint(ClassSymbol ts, ClassSymbol ss) {
+                Pair<ClassSymbol, ClassSymbol> newPair = new Pair<>(ts, ss);
+                /* if we are seeing the same pair again then there is an issue with the sealed hierarchy
+                 * bail out, a detailed error will be reported downstream
                  */
-                return areDisjoint(ss, ts);
+                if (!pairsSeen.add(newPair))
+                    return false;
+                if (isSubtype(erasure(ts.type), erasure(ss.type))) {
+                    return false;
+                }
+                // if both are classes or both are interfaces, shortcut
+                if (ts.isInterface() == ss.isInterface() && isSubtype(erasure(ss.type), erasure(ts.type))) {
+                    return false;
+                }
+                if (ts.isInterface() && !ss.isInterface()) {
+                    /* so ts is interface but ss is a class
+                     * an interface is disjoint from a class if the class is disjoint form the interface
+                     */
+                    return areDisjoint(ss, ts);
+                }
+                // a final class that is not subtype of ss is disjoint
+                if (!ts.isInterface() && ts.isFinal()) {
+                    return true;
+                }
+                // if at least one is sealed
+                if (ts.isSealed() || ss.isSealed()) {
+                    // permitted subtypes have to be disjoint with the other symbol
+                    ClassSymbol sealedOne = ts.isSealed() ? ts : ss;
+                    ClassSymbol other = sealedOne == ts ? ss : ts;
+                    return sealedOne.permitted.stream().allMatch(sym -> areDisjoint((ClassSymbol)sym, other));
+                }
+                return false;
             }
-            // a final class that is not subtype of ss is disjoint
-            if (!ts.isInterface() && ts.isFinal()) {
-                return true;
-            }
-            // if at least one is sealed
-            if (ts.isSealed() || ss.isSealed()) {
-                // permitted subtypes have to be disjoint with the other symbol
-                ClassSymbol sealedOne = ts.isSealed() ? ts : ss;
-                ClassSymbol other = sealedOne == ts ? ss : ts;
-                return sealedOne.permitted.stream().allMatch(sym -> areDisjoint((ClassSymbol)sym, other));
-            }
-            return false;
         }
 
         private TypeRelation isCastable = new TypeRelation() {
diff --git a/test/langtools/tools/javac/sealed/erroneous_hierarchy/CyclicHierarchyTest.java b/test/langtools/tools/javac/sealed/erroneous_hierarchy/CyclicHierarchyTest.java
new file mode 100644
index 00000000000..b3f30c4297a
--- /dev/null
+++ b/test/langtools/tools/javac/sealed/erroneous_hierarchy/CyclicHierarchyTest.java
@@ -0,0 +1,12 @@
+/*
+ * @test /nodynamiccopyright/
+ * @bug 8319987
+ * @summary compilation of sealed classes leads to infinite recursion
+ * @compile/fail/ref=CyclicHierarchyTest.out -XDrawDiagnostics CyclicHierarchyTest.java
+ */
+
+class CyclicHierarchyTest {
+    sealed interface Action permits Add {}
+    sealed interface MathOp permits Add {}
+    sealed static class Add implements MathOp permits Add {}
+}
diff --git a/test/langtools/tools/javac/sealed/erroneous_hierarchy/CyclicHierarchyTest.out b/test/langtools/tools/javac/sealed/erroneous_hierarchy/CyclicHierarchyTest.out
new file mode 100644
index 00000000000..c9420a0dc09
--- /dev/null
+++ b/test/langtools/tools/javac/sealed/erroneous_hierarchy/CyclicHierarchyTest.out
@@ -0,0 +1,3 @@
+CyclicHierarchyTest.java:9:37: compiler.err.invalid.permits.clause: (compiler.misc.doesnt.extend.sealed: CyclicHierarchyTest.Add)
+CyclicHierarchyTest.java:11:55: compiler.err.invalid.permits.clause: (compiler.misc.must.not.be.same.class)
+2 errors