6726779: ConvolveOp on USHORT raster can cause the JVM crash

Reviewed-by: igor, prr

jdk/src/share/native/sun/awt/medialib/awt_ImagingLib.c

@@ -216,6 +216,16 @@ printMedialibError(int status) {
 
 #endif /* ! DEBUG */
 
+static int
+getMlibEdgeHint(jint edgeHint) {
+    switch (edgeHint) {
+    case java_awt_image_ConvolveOp_EDGE_NO_OP:
+        return MLIB_EDGE_DST_COPY_SRC;
+    case java_awt_image_ConvolveOp_EDGE_ZERO_FILL:
+    default:
+        return MLIB_EDGE_DST_FILL_ZERO;
+    }
+}
 
 /***************************************************************************
  *                          External Functions                             *
@@ -400,22 +410,10 @@ Java_sun_awt_image_ImagingLib_convolveBI(JNIEnv *env, jobject this,
         }
     }
 
-    if (edgeHint == java_awt_image_ConvolveOp_EDGE_NO_OP) {
-        int kw2 = kwidth>>1;
-        int kh2 = kheight>>1;
-        int bsize = mlib_ImageGetChannels(src)*
-            (mlib_ImageGetType(src) == MLIB_BYTE ? 1 : 2);
-
-        void *dstDataP = mlib_ImageGetData(dst);
-        void *srcDataP = mlib_ImageGetData(src);
-        /* REMIND: Copy a smaller area */
-        memcpy(dstDataP, srcDataP, dst->width*dst->height*bsize);
-    }
-
     cmask = (1<<src->channels)-1;
     status = (*sMlibFns[MLIB_CONVMxN].fptr)(dst, src, kdata, w, h,
                                (w-1)/2, (h-1)/2, scale, cmask,
-                               MLIB_EDGE_DST_NO_WRITE);
+                               getMlibEdgeHint(edgeHint));
 
     if (status != MLIB_SUCCESS) {
         printMedialibError(status);
@@ -660,22 +658,10 @@ Java_sun_awt_image_ImagingLib_convolveRaster(JNIEnv *env, jobject this,
         }
     }
 
-    if (edgeHint == java_awt_image_ConvolveOp_EDGE_NO_OP) {
-        int kw2 = kwidth>>1;
-        int kh2 = kheight>>1;
-        int bsize = mlib_ImageGetChannels(src)*
-            (mlib_ImageGetType(src) == MLIB_BYTE ? 1 : 2);
-
-        void *dstDataP = mlib_ImageGetData(dst);
-        void *srcDataP = mlib_ImageGetData(src);
-        /* REMIND: Copy a smaller area */
-        memcpy(dstDataP, srcDataP, dst->width*dst->height*bsize);
-    }
-
     cmask = (1<<src->channels)-1;
     status = (*sMlibFns[MLIB_CONVMxN].fptr)(dst, src, kdata, w, h,
                                (w-1)/2, (h-1)/2, scale, cmask,
-                               MLIB_EDGE_DST_NO_WRITE);
+                               getMlibEdgeHint(edgeHint));
 
     if (status != MLIB_SUCCESS) {
         printMedialibError(status);

jdk/test/java/awt/image/ConvolveOp/EdgeNoOpCrash.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright 2008 Sun Microsystems, Inc.  All Rights Reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
+ * CA 95054 USA or visit www.sun.com if you need additional information or
+ * have any questions.
+ */
+
+/*
+ * @test
+ * @bug     6726779
+ * @summary Test verifies that ConvolveOp with the EDGE_NO_OP edge condition
+ *          does not cause JVM crash if size of source raster elements is
+ *          greather than size of the destination raster element.
+ *
+ * @run     main EdgeNoOpCrash
+ */
+import java.awt.Point;
+import java.awt.image.ConvolveOp;
+import java.awt.image.DataBuffer;
+import java.awt.image.ImagingOpException;
+import java.awt.image.Kernel;
+import java.awt.image.Raster;
+import java.awt.image.WritableRaster;
+import java.util.Arrays;
+
+public class EdgeNoOpCrash {
+    private static final int w = 3000;
+    private static final int h = 200;
+
+    public static void main(String[] args) {
+        crashTest();
+    }
+
+    private static void crashTest() {
+        Raster src = createSrcRaster();
+        WritableRaster dst = createDstRaster();
+        ConvolveOp op = createConvolveOp(ConvolveOp.EDGE_NO_OP);
+        try {
+            op.filter(src, dst);
+        } catch (ImagingOpException e) {
+            /*
+             * The test pair of source and destination rasters
+             * may cause failure of the medialib convolution routine,
+             * so this exception is expected.
+             *
+             * The JVM crash is the only manifestation of this
+             * test failure.
+             */
+        }
+        System.out.println("Test PASSED.");
+    }
+
+    private static Raster createSrcRaster() {
+        WritableRaster r = Raster.createInterleavedRaster(DataBuffer.TYPE_USHORT,
+                w, h, 4, new Point(0, 0));
+
+        return r;
+    }
+
+    private static WritableRaster createDstRaster() {
+        WritableRaster r = Raster.createInterleavedRaster(DataBuffer.TYPE_BYTE,
+                w, h, 4, new Point(0, 0));
+
+        return r;
+    }
+
+    private static ConvolveOp createConvolveOp(int edgeHint) {
+        final int kw = 3;
+        final int kh = 3;
+        float[] kdata = new float[kw * kh];
+        float v = 1f / kdata.length;
+        Arrays.fill(kdata, v);
+
+        Kernel k = new Kernel(kw, kh, kdata);
+        ConvolveOp op = new ConvolveOp(k, edgeHint, null);
+
+        return op;
+    }
+}