8156097: docs build fails with StackOverflowError on Solaris

Avoiding deep recursion in TypeEnter.MembersPhase

Reviewed-by: mcimadamore

langtools/src/jdk.compiler/share/classes/com/sun/tools/javac/comp/TypeEnter.java

@@ -841,6 +841,28 @@ public class TypeEnter implements Completer {
             super(CompletionCause.MEMBERS_PHASE, null);
         }
 
+        private boolean completing;
+        private List<Env<AttrContext>> todo = List.nil();
+
+        @Override
+        protected void doCompleteEnvs(List<Env<AttrContext>> envs) {
+            todo = todo.prependList(envs);
+            if (completing) {
+                return ; //the top-level invocation will handle all envs
+            }
+            boolean prevCompleting = completing;
+            completing = true;
+            try {
+                while (todo.nonEmpty()) {
+                    Env<AttrContext> head = todo.head;
+                    todo = todo.tail;
+                    super.doCompleteEnvs(List.of(head));
+                }
+            } finally {
+                completing = prevCompleting;
+            }
+        }
+
         @Override
         protected void runPhase(Env<AttrContext> env) {
             JCClassDecl tree = env.enclClass;

langtools/test/tools/javac/importscope/TestDeepFinishClassStack.java

@@ -0,0 +1,67 @@
+/*
+ * Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+ * @test
+ * @bug 8156097
+ * @summary Check that nested TypeEnter.MembersPhase invocations don't cause StackOverflowError
+ * @library /tools/lib
+ * @modules jdk.compiler/com.sun.tools.javac.api
+ *          jdk.compiler/com.sun.tools.javac.main
+ */
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import toolbox.JavacTask;
+import toolbox.ToolBox;
+
+public class TestDeepFinishClassStack {
+    public static void main(String... args) throws IOException {
+        new TestDeepFinishClassStack().run();
+    }
+
+    int depth = 1000;
+
+    void run() throws IOException {
+        Path src = Paths.get("src");
+
+        Files.createDirectories(src);
+
+        ToolBox tb = new ToolBox();
+
+        for (int i = 0; i < depth; i++) {
+            tb.writeJavaFiles(src, "public class C" + i + " { public void test(C" + (i + 1) + " c) { } }");
+        }
+
+        tb.writeJavaFiles(src, "public class C" + depth + " { }");
+
+        new JavacTask(tb).files(src.resolve("C0.java"))
+                         .sourcepath(src)
+                         .outdir(".")
+                         .run()
+                         .writeAll();
+    }
+
+}