vmdav: Try to fix unpack_rle()

This fixes out of array accesses
The code prior to this commit could not have worked, thus obviously
was untested. I was also not able to find a valid sample that uses this
code.
This fix is thus only based on the description of the format

If someone has a sample that uses unpack_rle(), please mail me.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
This commit is contained in:
Michael Niedermayer 2013-05-01 23:46:38 +02:00
parent d602f16a37
commit c1f2c4c3b4

View File

@ -151,7 +151,7 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest,
int src_count, int src_size, int dest_len)
{
unsigned char *pd;
int i, l;
int i, j, l;
unsigned char *dest_end = dest + dest_len;
GetByteContext gb;
@ -176,13 +176,15 @@ static int rle_unpack(const unsigned char *src, unsigned char *dest,
bytestream2_get_bufferu(&gb, pd, l);
pd += l;
} else {
if (dest_end - pd < i || bytestream2_get_bytes_left(&gb) < 2)
int ps[2];
if (dest_end - pd < 2*l || bytestream2_get_bytes_left(&gb) < 2)
return bytestream2_tell(&gb);
for (i = 0; i < l; i++) {
*pd++ = bytestream2_get_byteu(&gb);
*pd++ = bytestream2_get_byteu(&gb);
ps[0] = bytestream2_get_byteu(&gb);
ps[1] = bytestream2_get_byteu(&gb);
for (j = 0; j < l; j++) {
*pd++ = ps[0];
*pd++ = ps[1];
}
bytestream2_skip(&gb, 2);
}
i += l;
} while (i < src_count);