;****************************************************************************** ;* VP9 MC SIMD optimizations ;* ;* Copyright (c) 2013 Ronald S. Bultje <rsbultje gmail com> ;* ;* This file is part of FFmpeg. ;* ;* FFmpeg is free software; you can redistribute it and/or ;* modify it under the terms of the GNU Lesser General Public ;* License as published by the Free Software Foundation; either ;* version 2.1 of the License, or (at your option) any later version. ;* ;* FFmpeg is distributed in the hope that it will be useful, ;* but WITHOUT ANY WARRANTY; without even the implied warranty of ;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU ;* Lesser General Public License for more details. ;* ;* You should have received a copy of the GNU Lesser General Public ;* License along with FFmpeg; if not, write to the Free Software ;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA ;****************************************************************************** %include "libavutil/x86/x86util.asm" SECTION_RODATA ; FIXME share with vp8dsp.asm pw_256: times 8 dw 256 %macro F8_TAPS 8 times 8 db %1, %2 times 8 db %3, %4 times 8 db %5, %6 times 8 db %7, %8 %endmacro ; int8_t ff_filters_ssse3[3][15][4][16] const filters_ssse3 ; smooth F8_TAPS -3, -1, 32, 64, 38, 1, -3, 0 F8_TAPS -2, -2, 29, 63, 41, 2, -3, 0 F8_TAPS -2, -2, 26, 63, 43, 4, -4, 0 F8_TAPS -2, -3, 24, 62, 46, 5, -4, 0 F8_TAPS -2, -3, 21, 60, 49, 7, -4, 0 F8_TAPS -1, -4, 18, 59, 51, 9, -4, 0 F8_TAPS -1, -4, 16, 57, 53, 12, -4, -1 F8_TAPS -1, -4, 14, 55, 55, 14, -4, -1 F8_TAPS -1, -4, 12, 53, 57, 16, -4, -1 F8_TAPS 0, -4, 9, 51, 59, 18, -4, -1 F8_TAPS 0, -4, 7, 49, 60, 21, -3, -2 F8_TAPS 0, -4, 5, 46, 62, 24, -3, -2 F8_TAPS 0, -4, 4, 43, 63, 26, -2, -2 F8_TAPS 0, -3, 2, 41, 63, 29, -2, -2 F8_TAPS 0, -3, 1, 38, 64, 32, -1, -3 ; regular F8_TAPS 0, 1, -5, 126, 8, -3, 1, 0 F8_TAPS -1, 3, -10, 122, 18, -6, 2, 0 F8_TAPS -1, 4, -13, 118, 27, -9, 3, -1 F8_TAPS -1, 4, -16, 112, 37, -11, 4, -1 F8_TAPS -1, 5, -18, 105, 48, -14, 4, -1 F8_TAPS -1, 5, -19, 97, 58, -16, 5, -1 F8_TAPS -1, 6, -19, 88, 68, -18, 5, -1 F8_TAPS -1, 6, -19, 78, 78, -19, 6, -1 F8_TAPS -1, 5, -18, 68, 88, -19, 6, -1 F8_TAPS -1, 5, -16, 58, 97, -19, 5, -1 F8_TAPS -1, 4, -14, 48, 105, -18, 5, -1 F8_TAPS -1, 4, -11, 37, 112, -16, 4, -1 F8_TAPS -1, 3, -9, 27, 118, -13, 4, -1 F8_TAPS 0, 2, -6, 18, 122, -10, 3, -1 F8_TAPS 0, 1, -3, 8, 126, -5, 1, 0 ; sharp F8_TAPS -1, 3, -7, 127, 8, -3, 1, 0 F8_TAPS -2, 5, -13, 125, 17, -6, 3, -1 F8_TAPS -3, 7, -17, 121, 27, -10, 5, -2 F8_TAPS -4, 9, -20, 115, 37, -13, 6, -2 F8_TAPS -4, 10, -23, 108, 48, -16, 8, -3 F8_TAPS -4, 10, -24, 100, 59, -19, 9, -3 F8_TAPS -4, 11, -24, 90, 70, -21, 10, -4 F8_TAPS -4, 11, -23, 80, 80, -23, 11, -4 F8_TAPS -4, 10, -21, 70, 90, -24, 11, -4 F8_TAPS -3, 9, -19, 59, 100, -24, 10, -4 F8_TAPS -3, 8, -16, 48, 108, -23, 10, -4 F8_TAPS -2, 6, -13, 37, 115, -20, 9, -4 F8_TAPS -2, 5, -10, 27, 121, -17, 7, -3 F8_TAPS -1, 3, -6, 17, 125, -13, 5, -2 F8_TAPS 0, 1, -3, 8, 127, -7, 3, -1 SECTION .text %macro filter_h_fn 1 %assign %%px mmsize/2 cglobal %1_8tap_1d_h_ %+ %%px, 6, 6, 11, dst, dstride, src, sstride, h, filtery mova m6, [pw_256] mova m7, [filteryq+ 0] %if ARCH_X86_64 && mmsize > 8 mova m8, [filteryq+16] mova m9, [filteryq+32] mova m10, [filteryq+48] %endif .loop: movh m0, [srcq-3] movh m1, [srcq-2] movh m2, [srcq-1] movh m3, [srcq+0] movh m4, [srcq+1] movh m5, [srcq+2] punpcklbw m0, m1 punpcklbw m2, m3 movh m1, [srcq+3] movh m3, [srcq+4] add srcq, sstrideq punpcklbw m4, m5 punpcklbw m1, m3 pmaddubsw m0, m7 %if ARCH_X86_64 && mmsize > 8 pmaddubsw m2, m8 pmaddubsw m4, m9 pmaddubsw m1, m10 %else pmaddubsw m2, [filteryq+16] pmaddubsw m4, [filteryq+32] pmaddubsw m1, [filteryq+48] %endif paddw m0, m2 paddw m4, m1 paddsw m0, m4 pmulhrsw m0, m6 %ifidn %1, avg movh m1, [dstq] %endif packuswb m0, m0 %ifidn %1, avg pavgb m0, m1 %endif movh [dstq], m0 add dstq, dstrideq dec hd jg .loop RET %endmacro INIT_MMX ssse3 filter_h_fn put filter_h_fn avg INIT_XMM ssse3 filter_h_fn put filter_h_fn avg %if ARCH_X86_64 %macro filter_hx2_fn 1 %assign %%px mmsize cglobal %1_8tap_1d_h_ %+ %%px, 6, 6, 14, dst, dstride, src, sstride, h, filtery mova m13, [pw_256] mova m8, [filteryq+ 0] mova m9, [filteryq+16] mova m10, [filteryq+32] mova m11, [filteryq+48] .loop: movu m0, [srcq-3] movu m1, [srcq-2] movu m2, [srcq-1] movu m3, [srcq+0] movu m4, [srcq+1] movu m5, [srcq+2] movu m6, [srcq+3] movu m7, [srcq+4] add srcq, sstrideq SBUTTERFLY bw, 0, 1, 12 SBUTTERFLY bw, 2, 3, 12 SBUTTERFLY bw, 4, 5, 12 SBUTTERFLY bw, 6, 7, 12 pmaddubsw m0, m8 pmaddubsw m1, m8 pmaddubsw m2, m9 pmaddubsw m3, m9 pmaddubsw m4, m10 pmaddubsw m5, m10 pmaddubsw m6, m11 pmaddubsw m7, m11 paddw m0, m2 paddw m1, m3 paddw m4, m6 paddw m5, m7 paddsw m0, m4 paddsw m1, m5 pmulhrsw m0, m13 pmulhrsw m1, m13 packuswb m0, m1 %ifidn %1, avg pavgb m0, [dstq] %endif mova [dstq], m0 add dstq, dstrideq dec hd jg .loop RET %endmacro INIT_XMM ssse3 filter_hx2_fn put filter_hx2_fn avg %endif ; ARCH_X86_64 %macro filter_v_fn 1 %assign %%px mmsize/2 %if ARCH_X86_64 cglobal %1_8tap_1d_v_ %+ %%px, 6, 8, 11, dst, dstride, src, sstride, h, filtery, src4, sstride3 %else cglobal %1_8tap_1d_v_ %+ %%px, 4, 7, 11, dst, dstride, src, sstride, filtery, src4, sstride3 mov filteryq, r5mp %define hd r4mp %endif mova m6, [pw_256] lea sstride3q, [sstrideq*3] lea src4q, [srcq+sstrideq] sub srcq, sstride3q mova m7, [filteryq+ 0] %if ARCH_X86_64 && mmsize > 8 mova m8, [filteryq+16] mova m9, [filteryq+32] mova m10, [filteryq+48] %endif .loop: ; FIXME maybe reuse loads from previous rows, or just ; more generally unroll this to prevent multiple loads of ; the same data? movh m0, [srcq] movh m1, [srcq+sstrideq] movh m2, [srcq+sstrideq*2] movh m3, [srcq+sstride3q] movh m4, [src4q] movh m5, [src4q+sstrideq] punpcklbw m0, m1 punpcklbw m2, m3 movh m1, [src4q+sstrideq*2] movh m3, [src4q+sstride3q] add srcq, sstrideq add src4q, sstrideq punpcklbw m4, m5 punpcklbw m1, m3 pmaddubsw m0, m7 %if ARCH_X86_64 && mmsize > 8 pmaddubsw m2, m8 pmaddubsw m4, m9 pmaddubsw m1, m10 %else pmaddubsw m2, [filteryq+16] pmaddubsw m4, [filteryq+32] pmaddubsw m1, [filteryq+48] %endif paddw m0, m2 paddw m4, m1 paddsw m0, m4 pmulhrsw m0, m6 %ifidn %1, avg movh m1, [dstq] %endif packuswb m0, m0 %ifidn %1, avg pavgb m0, m1 %endif movh [dstq], m0 add dstq, dstrideq dec hd jg .loop RET %endmacro INIT_MMX ssse3 filter_v_fn put filter_v_fn avg INIT_XMM ssse3 filter_v_fn put filter_v_fn avg %if ARCH_X86_64 %macro filter_vx2_fn 1 %assign %%px mmsize cglobal %1_8tap_1d_v_ %+ %%px, 6, 8, 14, dst, dstride, src, sstride, h, filtery, src4, sstride3 mova m13, [pw_256] lea sstride3q, [sstrideq*3] lea src4q, [srcq+sstrideq] sub srcq, sstride3q mova m8, [filteryq+ 0] mova m9, [filteryq+16] mova m10, [filteryq+32] mova m11, [filteryq+48] .loop: ; FIXME maybe reuse loads from previous rows, or just ; more generally unroll this to prevent multiple loads of ; the same data? movu m0, [srcq] movu m1, [srcq+sstrideq] movu m2, [srcq+sstrideq*2] movu m3, [srcq+sstride3q] movu m4, [src4q] movu m5, [src4q+sstrideq] movu m6, [src4q+sstrideq*2] movu m7, [src4q+sstride3q] add srcq, sstrideq add src4q, sstrideq SBUTTERFLY bw, 0, 1, 12 SBUTTERFLY bw, 2, 3, 12 SBUTTERFLY bw, 4, 5, 12 SBUTTERFLY bw, 6, 7, 12 pmaddubsw m0, m8 pmaddubsw m1, m8 pmaddubsw m2, m9 pmaddubsw m3, m9 pmaddubsw m4, m10 pmaddubsw m5, m10 pmaddubsw m6, m11 pmaddubsw m7, m11 paddw m0, m2 paddw m1, m3 paddw m4, m6 paddw m5, m7 paddsw m0, m4 paddsw m1, m5 pmulhrsw m0, m13 pmulhrsw m1, m13 packuswb m0, m1 %ifidn %1, avg pavgb m0, [dstq] %endif mova [dstq], m0 add dstq, dstrideq dec hd jg .loop RET %endmacro INIT_XMM ssse3 filter_vx2_fn put filter_vx2_fn avg %endif ; ARCH_X86_64 %macro fpel_fn 6 %if %2 == 4 %define %%srcfn movh %define %%dstfn movh %else %define %%srcfn movu %define %%dstfn mova %endif %if %2 <= 16 cglobal %1%2, 5, 7, 4, dst, dstride, src, sstride, h, dstride3, sstride3 lea sstride3q, [sstrideq*3] lea dstride3q, [dstrideq*3] %else cglobal %1%2, 5, 5, 4, dst, dstride, src, sstride, h %endif .loop: %%srcfn m0, [srcq] %%srcfn m1, [srcq+s%3] %%srcfn m2, [srcq+s%4] %%srcfn m3, [srcq+s%5] lea srcq, [srcq+sstrideq*%6] %ifidn %1, avg pavgb m0, [dstq] pavgb m1, [dstq+d%3] pavgb m2, [dstq+d%4] pavgb m3, [dstq+d%5] %endif %%dstfn [dstq], m0 %%dstfn [dstq+d%3], m1 %%dstfn [dstq+d%4], m2 %%dstfn [dstq+d%5], m3 lea dstq, [dstq+dstrideq*%6] sub hd, %6 jnz .loop RET %endmacro %define d16 16 %define s16 16 INIT_MMX mmx fpel_fn put, 4, strideq, strideq*2, stride3q, 4 fpel_fn put, 8, strideq, strideq*2, stride3q, 4 INIT_MMX mmxext fpel_fn avg, 4, strideq, strideq*2, stride3q, 4 fpel_fn avg, 8, strideq, strideq*2, stride3q, 4 INIT_XMM sse fpel_fn put, 16, strideq, strideq*2, stride3q, 4 fpel_fn put, 32, mmsize, strideq, strideq+mmsize, 2 fpel_fn put, 64, mmsize, mmsize*2, mmsize*3, 1 INIT_XMM sse2 fpel_fn avg, 16, strideq, strideq*2, stride3q, 4 fpel_fn avg, 32, mmsize, strideq, strideq+mmsize, 2 fpel_fn avg, 64, mmsize, mmsize*2, mmsize*3, 1 %undef s16 %undef d16