[trunk] Import commit f4139d702559649e577a5df9cfd64b0ca6107a7a from ghostpdl

Several functions accept a buffer size but never actually check whether
the buffer overflows during reading/writing. This fixes all cases where
a size variable has explicitly been marked as unused (through a (void)
cast).

This was discovered while investigating an assertion caused by
7cc691f332f26802c64cdc47e17bff8b_signal_sigabrt_7ffff6d59425_2247_2509.pdf
among others.

Thanks to Mateusz Jurczyk and Gynvael Coldwind of the Google Security
Team for providing the example files.

src/lib/openjp2/event.c

@@ -118,7 +118,7 @@ OPJ_BOOL opj_event_msg(opj_event_mgr_t* p_event_mgr, OPJ_INT32 event_type, const
 		str_length = (strlen(fmt) > OPJ_MSG_SIZE) ? OPJ_MSG_SIZE : strlen(fmt);
         (void)str_length;
 		/* parse the format string and put the result in 'message' */
-		vsprintf(message, fmt, arg); /* UniPG */
+		vsnprintf(message, OPJ_MSG_SIZE, fmt, arg); /* UniPG */
 		/* deinitialize the optional parameter list */
 		va_end(arg);
 

src/lib/openjp2/jp2.c

@@ -837,6 +837,7 @@ OPJ_BOOL opj_jp2_read_pclr(	opj_jp2_t *jp2,
 	OPJ_UINT16 nr_entries,nr_channels;
 	OPJ_UINT16 i, j;
 	OPJ_UINT32 l_value;
+	OPJ_BYTE *orig_header_data = p_pclr_header_data;
 
 	/* preconditions */
 	assert(p_pclr_header_data != 00);
@@ -847,6 +848,9 @@ OPJ_BOOL opj_jp2_read_pclr(	opj_jp2_t *jp2,
 	if(jp2->color.jp2_pclr)
 		return OPJ_FALSE;
 
+	if (p_pclr_header_size < 3)
+		return OPJ_FALSE;
+
 	opj_read_bytes(p_pclr_header_data, &l_value , 2);	/* NE */
 	p_pclr_header_data += 2;
 	nr_entries = (OPJ_UINT16) l_value;
@@ -855,6 +859,9 @@ OPJ_BOOL opj_jp2_read_pclr(	opj_jp2_t *jp2,
 	++p_pclr_header_data;
 	nr_channels = (OPJ_UINT16) l_value;
 
+	if (p_pclr_header_size < 3 + (OPJ_UINT32)nr_channels || nr_channels == 0 || nr_entries >= (OPJ_UINT32)-1 / nr_channels)
+		return OPJ_FALSE;
+
 	entries = (OPJ_UINT32*) opj_malloc(nr_channels * nr_entries * sizeof(OPJ_UINT32));
     if (!entries)
         return OPJ_FALSE;
@@ -902,6 +909,11 @@ OPJ_BOOL opj_jp2_read_pclr(	opj_jp2_t *jp2,
 		for(i = 0; i < nr_channels; ++i) {
 			OPJ_INT32 bytes_to_read = (channel_size[i]+7)>>3;
 
+			if (bytes_to_read > sizeof(OPJ_UINT32))
+				bytes_to_read = sizeof(OPJ_UINT32);
+			if ((ptrdiff_t)p_pclr_header_size < p_pclr_header_data - orig_header_data + bytes_to_read)
+				return OPJ_FALSE;
+
 			opj_read_bytes(p_pclr_header_data, &l_value , bytes_to_read);	/* Cji */
 			p_pclr_header_data += bytes_to_read;
 			*entries = (OPJ_UINT32) l_value;
@@ -943,6 +955,11 @@ OPJ_BOOL opj_jp2_read_cmap(	opj_jp2_t * jp2,
 	}
 
 	nr_channels = jp2->color.jp2_pclr->nr_channels;
+	if (p_cmap_header_size < (OPJ_UINT32)nr_channels * 4) {
+		opj_event_msg(p_manager, EVT_ERROR, "Insufficient data for CMAP box.\n");
+		return OPJ_FALSE;
+	}
+
 	cmap = (opj_jp2_cmap_comp_t*) opj_malloc(nr_channels * sizeof(opj_jp2_cmap_comp_t));
     if (!cmap)
         return OPJ_FALSE;
@@ -1022,6 +1039,11 @@ OPJ_BOOL opj_jp2_read_cdef(	opj_jp2_t * jp2,
 	 * inside a JP2 Header box.'*/
 	if(jp2->color.jp2_cdef) return OPJ_FALSE;
 
+	if (p_cdef_header_size < 2) {
+		opj_event_msg(p_manager, EVT_ERROR, "Insufficient data for CDEF box.\n");
+		return OPJ_FALSE;
+	}
+
 	opj_read_bytes(p_cdef_header_data,&l_value ,2);			/* N */
 	p_cdef_header_data+= 2;
 
@@ -1030,6 +1052,11 @@ OPJ_BOOL opj_jp2_read_cdef(	opj_jp2_t * jp2,
 		return OPJ_FALSE;
 	}
 
+	if (p_cdef_header_size < 2 + (OPJ_UINT32)(OPJ_UINT16)l_value * 6) {
+		opj_event_msg(p_manager, EVT_ERROR, "Insufficient data for CDEF box.\n");
+		return OPJ_FALSE;
+	}
+
 	cdef_info = (opj_jp2_cdef_info_t*) opj_malloc(l_value * sizeof(opj_jp2_cdef_info_t));
     if (!cdef_info)
         return OPJ_FALSE;