8248574: Improve jpeg processing

Reviewed-by: serb, jdv, mschoene, rhalade

src/java.desktop/share/native/libjavajpeg/jdhuff.c

@@ -121,7 +121,8 @@ start_pass_huff_decoder (j_decompress_ptr cinfo)
     compptr = cinfo->cur_comp_info[ci];
     /* Precalculate which table to use for each block */
     entropy->dc_cur_tbls[blkn] = entropy->dc_derived_tbls[compptr->dc_tbl_no];
-    entropy->ac_cur_tbls[blkn] = entropy->ac_derived_tbls[compptr->ac_tbl_no];
+    entropy->ac_cur_tbls[blkn] =    /* AC needs no table when not present */
+      cinfo->lim_Se ? entropy->ac_derived_tbls[compptr->ac_tbl_no] : NULL;
     /* Decide whether we really care about the coefficient values */
     if (compptr->component_needed) {
       entropy->dc_needed[blkn] = TRUE;

src/java.desktop/share/native/libjavajpeg/jdinput.c

@@ -74,6 +74,39 @@ initial_setup (j_decompress_ptr cinfo)
                                    compptr->v_samp_factor);
   }
 
+  /* Derive lim_Se */
+  if (cinfo->is_baseline || (cinfo->progressive_mode &&
+      cinfo->comps_in_scan)) { /* no pseudo SOS marker */
+    cinfo->lim_Se = DCTSIZE2-1;
+  } else {
+    switch (cinfo->Se) {
+    case (1*1-1):
+    case (2*2-1):
+    case (3*3-1):
+    case (4*4-1):
+    case (5*5-1):
+    case (6*6-1):
+    case (7*7-1):
+         cinfo->lim_Se = cinfo->Se;
+         break;
+    case (8*8-1):
+    case (9*9-1):
+    case (10*10-1):
+    case (11*11-1):
+    case (12*12-1):
+    case (13*13-1):
+    case (14*14-1):
+    case (15*15-1):
+    case (16*16-1):
+         cinfo->lim_Se = DCTSIZE2-1;
+         break;
+    default:
+      ERREXIT4(cinfo, JERR_BAD_PROGRESSION,
+               cinfo->Ss, cinfo->Se, cinfo->Ah, cinfo->Al);
+      break;
+    }
+  }
+
   /* We initialize DCT_scaled_size and min_DCT_scaled_size to DCTSIZE.
    * In the full decompressor, this will be overridden by jdmaster.c;
    * but in the transcoder, jdmaster.c is not used, so we must do it here.

src/java.desktop/share/native/libjavajpeg/jdmarker.c

@@ -238,7 +238,7 @@ get_soi (j_decompress_ptr cinfo)
 
 
 LOCAL(boolean)
-get_sof (j_decompress_ptr cinfo, boolean is_prog, boolean is_arith)
+get_sof (j_decompress_ptr cinfo, boolean is_baseline, boolean is_prog, boolean is_arith)
 /* Process a SOFn marker */
 {
   INT32 length;
@@ -246,6 +246,7 @@ get_sof (j_decompress_ptr cinfo, boolean is_prog, boolean is_arith)
   jpeg_component_info * compptr;
   INPUT_VARS(cinfo);
 
+  cinfo->is_baseline = is_baseline;
   cinfo->progressive_mode = is_prog;
   cinfo->arith_code = is_arith;
 
@@ -998,23 +999,27 @@ read_markers (j_decompress_ptr cinfo)
       break;
 
     case M_SOF0:                /* Baseline */
+      if (! get_sof(cinfo, TRUE, FALSE, FALSE))
+        return JPEG_SUSPENDED;
+      break;
+
     case M_SOF1:                /* Extended sequential, Huffman */
-      if (! get_sof(cinfo, FALSE, FALSE))
+      if (! get_sof(cinfo, FALSE, FALSE, FALSE))
         return JPEG_SUSPENDED;
       break;
 
     case M_SOF2:                /* Progressive, Huffman */
-      if (! get_sof(cinfo, TRUE, FALSE))
+      if (! get_sof(cinfo, FALSE, TRUE, FALSE))
         return JPEG_SUSPENDED;
       break;
 
     case M_SOF9:                /* Extended sequential, arithmetic */
-      if (! get_sof(cinfo, FALSE, TRUE))
+      if (! get_sof(cinfo, FALSE, FALSE, TRUE))
         return JPEG_SUSPENDED;
       break;
 
     case M_SOF10:               /* Progressive, arithmetic */
-      if (! get_sof(cinfo, TRUE, TRUE))
+      if (! get_sof(cinfo, FALSE, TRUE, TRUE))
         return JPEG_SUSPENDED;
       break;
 

src/java.desktop/share/native/libjavajpeg/jmemnobs.c

@@ -70,13 +70,16 @@ jpeg_free_large (j_common_ptr cinfo, void FAR * object, size_t sizeofobject)
 
 /*
  * This routine computes the total memory space available for allocation.
- * Here we always say, "we got all you want bud!"
  */
 
 GLOBAL(size_t)
 jpeg_mem_available (j_common_ptr cinfo, size_t min_bytes_needed,
                     size_t max_bytes_needed, size_t already_allocated)
 {
+  if (cinfo->mem->max_memory_to_use)
+    return cinfo->mem->max_memory_to_use - already_allocated;
+
+  /* Here we say, "we got all you want bud!" */
   return max_bytes_needed;
 }
 

src/java.desktop/share/native/libjavajpeg/jpeglib.h

@@ -539,6 +539,7 @@ struct jpeg_decompress_struct {
   jpeg_component_info * comp_info;
   /* comp_info[i] describes component that appears i'th in SOF */
 
+  boolean is_baseline;          /* TRUE if Baseline SOF0 encountered */
   boolean progressive_mode;     /* TRUE if SOFn specifies progressive mode */
   boolean arith_code;           /* TRUE=arithmetic coding, FALSE=Huffman */
 
@@ -611,6 +612,8 @@ struct jpeg_decompress_struct {
 
   int Ss, Se, Ah, Al;           /* progressive JPEG parameters for scan */
 
+  int lim_Se;                   /* min( Se, DCTSIZE2-1 ) for entropy decode */
+
   /* This field is shared between entropy decoder and marker parser.
    * It is either zero or the code of a JPEG marker that has been
    * read from the data source, but has not yet been processed.