From f6006295c037e8ac2706874ee7b4b97bdda5c387 Mon Sep 17 00:00:00 2001 From: "Ronald S. Bultje" Date: Fri, 23 Oct 2015 11:11:53 -0400 Subject: [PATCH] videodsp: don't overread edges in vfix3 emu_edge. Fixes trac ticket 3226. Also see Andreas' analysis in https://bugs.debian.org/801745, which was very helpful. (cherry picked from commit 52f84d82bdf1851ecfcc412c1719e5f6f3396209) --- libavcodec/x86/videodsp.asm | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/libavcodec/x86/videodsp.asm b/libavcodec/x86/videodsp.asm index 1ac02574d6..77189fa6ef 100644 --- a/libavcodec/x86/videodsp.asm +++ b/libavcodec/x86/videodsp.asm @@ -185,8 +185,12 @@ hvar_fn %elif (%2-%%off) == 2 mov valw, [srcq+%2-2] %elifidn %1, body - mov vald, [srcq+%2-3] -%else + mov valb, [srcq+%2-1] + sal vald, 16 + mov valw, [srcq+%2-3] +%elifidn %1, bottom + movd mm %+ %%mmx_idx, [srcq+%2-4] +%else ; top movd mm %+ %%mmx_idx, [srcq+%2-3] %endif %endif ; (%2-%%off) >= 1 @@ -242,12 +246,15 @@ hvar_fn mov [dstq+%2-2], valw %elifidn %1, body mov [dstq+%2-3], valw - shr vald, 16 + sar vald, 16 mov [dstq+%2-1], valb %else movd vald, mm %+ %%mmx_idx +%ifidn %1, bottom + sar vald, 8 +%endif mov [dstq+%2-3], valw - shr vald, 16 + sar vald, 16 mov [dstq+%2-1], valb %endif %endif ; (%2-%%off) >= 1