8208752: Calling a deserialized Lambda might fail with ClassCastException

src/jdk.compiler/share/classes/com/sun/tools/javac/comp/LambdaToMethod.java

@@ -697,7 +697,7 @@ public class LambdaToMethod extends TreeTranslator {
                 rs.resolveConstructor(null, attrEnv, ctype, TreeInfo.types(args), List.nil()));
     }
 
-    private void addDeserializationCase(MethodHandleSymbol refSym, Type targetType, MethodSymbol samSym,
+    private void addDeserializationCase(MethodHandleSymbol refSym, Type targetType, MethodSymbol samSym, Type samType,
                                         DiagnosticPosition pos, List<LoadableConstant> staticArgs, MethodType indyType) {
         String functionalInterfaceClass = classSig(targetType);
         String functionalInterfaceMethodName = samSym.getSimpleName().toString();
@@ -712,6 +712,7 @@ public class LambdaToMethod extends TreeTranslator {
         String implClass = classSig(types.erasure(refSym.owner.type));
         String implMethodName = refSym.getQualifiedName().toString();
         String implMethodSignature = typeSig(types.erasure(refSym.type));
+        String instantiatedMethodType = typeSig(types.erasure(samType));
 
         JCExpression kindTest = eqTest(syms.intType, deserGetter("getImplMethodKind", syms.intType),
                 make.Literal(refSym.referenceKind()));
@@ -724,13 +725,14 @@ public class LambdaToMethod extends TreeTranslator {
             ++i;
         }
         JCStatement stmt = make.If(
-                deserTest(deserTest(deserTest(deserTest(deserTest(
-                                                        kindTest,
-                                                        "getFunctionalInterfaceClass", functionalInterfaceClass),
-                                                "getFunctionalInterfaceMethodName", functionalInterfaceMethodName),
-                                        "getFunctionalInterfaceMethodSignature", functionalInterfaceMethodSignature),
-                                "getImplClass", implClass),
-                        "getImplMethodSignature", implMethodSignature),
+                deserTests(kindTest, Map.of(
+                        "getFunctionalInterfaceClass", functionalInterfaceClass,
+                        "getFunctionalInterfaceMethodName", functionalInterfaceMethodName,
+                        "getFunctionalInterfaceMethodSignature", functionalInterfaceMethodSignature,
+                        "getImplClass", implClass,
+                        "getImplMethodSignature", implMethodSignature,
+                        "getInstantiatedMethodType", instantiatedMethodType
+                )),
                 make.Return(makeIndyCall(
                         pos,
                         syms.lambdaMetafactory,
@@ -751,6 +753,7 @@ public class LambdaToMethod extends TreeTranslator {
         System.err.printf("*implClass: '%s'\n", implClass);
         System.err.printf("*implMethodName: '%s'\n", implMethodName);
         System.err.printf("*implMethodSignature: '%s'\n", implMethodSignature);
+        System.err.printf("*instantiatedMethodType: '%s'\n", instantiatedMethodType);
         ****/
         stmts.append(stmt);
     }
@@ -762,6 +765,15 @@ public class LambdaToMethod extends TreeTranslator {
         return testExpr;
     }
 
+    private JCExpression deserTests(JCExpression prev, Map<String, String> tests) {
+        for (Map.Entry<String, String> entry : tests.entrySet()) {
+            String func = entry.getKey();
+            String lit = entry.getValue();
+            prev = deserTest(prev, func, lit);
+        }
+        return prev;
+    }
+
     private JCExpression deserTest(JCExpression prev, String func, String lit) {
         MethodType eqmt = new MethodType(List.of(syms.objectType), syms.booleanType, List.nil(), syms.methodClass);
         Symbol eqsym = rs.resolveQualifiedMethod(null, attrEnv, syms.objectType, names.equals, List.of(syms.objectType), List.nil());
@@ -812,10 +824,11 @@ public class LambdaToMethod extends TreeTranslator {
                                                  List<JCExpression> indy_args) {
         //determine the static bsm args
         MethodSymbol samSym = (MethodSymbol) types.findDescriptorSymbol(tree.target.tsym);
+        MethodType samType = typeToMethodType(tree.getDescriptorType(types));
         List<LoadableConstant> staticArgs = List.of(
                 typeToMethodType(samSym.type),
                 refSym.asHandle(),
-                typeToMethodType(tree.getDescriptorType(types)));
+                samType);
 
         //computed indy arg types
         ListBuffer<Type> indy_args_types = new ListBuffer<>();
@@ -879,7 +892,7 @@ public class LambdaToMethod extends TreeTranslator {
                 int prevPos = make.pos;
                 try {
                     make.at(kInfo.clazz);
-                    addDeserializationCase(refSym, tree.type, samSym,
+                    addDeserializationCase(refSym, tree.type, samSym, samType,
                             tree, staticArgs, indyType);
                 } finally {
                     make.at(prevPos);

test/langtools/tools/javac/lambda/LambdaSerializedClassCastException.java

@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2024, Alphabet LLC. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/*
+@test
+@bug 8208752
+@summary NPE generating serializedLambdaName for nested lambda
+*/
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.Serializable;
+import java.util.function.Function;
+
+public class LambdaSerializedClassCastException {
+
+    public static void main(String[] args) throws Exception {
+
+        Function<String, String> lambda1 =
+                (Function<String, String> & Serializable) Object::toString;
+        Function<Object, String> lambda2 =
+                (Function<Object, String> & Serializable) Object::toString;
+
+        Function<Object, String> deserial = serialDeserial(lambda2);
+        deserial.apply(new Object());
+    }
+
+    @SuppressWarnings("unchecked")
+    static <T> T serialDeserial(T object) throws Exception {
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        ObjectOutputStream oos = new ObjectOutputStream(baos);
+        oos.writeObject(object);
+        oos.close();
+        ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+        ObjectInputStream ois = new ObjectInputStream(bais);
+        T result = (T) ois.readObject();
+        ois.close();
+        return result;
+    }
+}