jpeg2000: fix overflow in dequantization

Fixes decoding of file generated with:
ffmpeg -f lavfi -i smptehdbars=hd720 -pix_fmt rgb48 /tmp/o.jp2

Reviewed-by: Nicolas BERTRAND <nicoinattendu@gmail.com>
Signed-off-by: Michael Niedermayer <michaelni@gmx.at>
(cherry picked from commit f57119b8e5)
This commit is contained in:
Michael Niedermayer
2013-07-13 02:24:56 +02:00
committed by Carl Eugen Hoyos
parent e0d88cfd18
commit fa6b6dad3d
3 changed files with 4 additions and 4 deletions

View File

@@ -320,7 +320,7 @@ int ff_jpeg2000_init_component(Jpeg2000Component *comp,
if (!av_codec_is_encoder(avctx->codec))
band->f_stepsize *= 0.5;
band->i_stepsize = band->f_stepsize * (1 << 16);
band->i_stepsize = band->f_stepsize * (1 << 15);
/* computation of tbx_0, tbx_1, tby_0, tby_1
* see ISO/IEC 15444-1:2002 B.5 eq. B-15 and tbl B.1