8011960: [2,1].sort(null) should throw TypeError

Reviewed-by: hannesw, lagergren

nashorn/src/jdk/nashorn/internal/objects/NativeArray.java

@@ -793,11 +793,15 @@ public final class NativeArray extends ScriptObject {
     }
 
     private static ScriptFunction compareFunction(final Object comparefn) {
-        try {
-            return (ScriptFunction)comparefn;
-        } catch (final ClassCastException e) {
-            return null; //undefined or null
+        if (comparefn == ScriptRuntime.UNDEFINED) {
+            return null;
         }
+
+        if (! (comparefn instanceof ScriptFunction)) {
+            throw typeError("not.a.function", ScriptRuntime.safeToString(comparefn));
+        }
+
+        return (ScriptFunction)comparefn;
     }
 
     private static Object[] sort(final Object[] array, final Object comparefn) {

nashorn/test/script/basic/JDK-8011960.js

@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2010, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ * 
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ * 
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ * 
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ * 
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * JDK-8011960: [2,1].sort(null) should throw TypeError
+ *
+ * @test
+ * @run
+ */
+
+function check(func) {
+    try {
+        [2,1].sort(func);
+        fail("should have thrown TypeError for :" + func);
+    } catch (e) {
+        if (! (e instanceof TypeError)) {
+            fail("TypeError not thrown for sort comparefn: " + func);
+        }
+    }
+}
+
+// should not result in TypeError for undefined
+[1, 2].sort(undefined);
+
+// TypeError for null
+check(null);
+
+// should result in TypeError other non-callable params
+check(32);
+check("foo");
+check(false);
+check({});
+check([]);