vpx/vp8/common/x86/subpixel_ssse3.asm

;
;  Copyright (c) 2010 The WebM project authors. All Rights Reserved.
;
;  Use of this source code is governed by a BSD-style license
;  that can be found in the LICENSE file in the root of the source
;  tree. An additional intellectual property rights grant can be found
;  in the file PATENTS.  All contributing project authors may
;  be found in the AUTHORS file in the root of the source tree.
;


%include "vpx_ports/x86_abi_support.asm"

%define BLOCK_HEIGHT_WIDTH 4
%define VP8_FILTER_WEIGHT 128
%define VP8_FILTER_SHIFT  7


;/************************************************************************************
; Notes: filter_block1d_h6 applies a 6 tap filter horizontally to the input pixels. The
; input pixel array has output_height rows. This routine assumes that output_height is an
; even number. This function handles 8 pixels in horizontal direction, calculating ONE
; rows each iteration to take advantage of the 128 bits operations.
;
; This is an implementation of some of the SSE optimizations first seen in ffvp8
;
;*************************************************************************************/
;void vp8_filter_block1d8_h6_ssse3
;(
;    unsigned char  *src_ptr,
;    unsigned int    src_pixels_per_line,
;    unsigned char *output_ptr,
;    unsigned int    output_pitch,
;    unsigned int    output_height,
;    unsigned int    vp8_filter_index
;)
global sym(vp8_filter_block1d8_h6_ssse3) PRIVATE
sym(vp8_filter_block1d8_h6_ssse3):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 6
    SAVE_XMM 7
    GET_GOT     rbx
    push        rsi
    push        rdi
    ; end prolog

    movsxd      rdx, DWORD PTR arg(5)   ;table index
    xor         rsi, rsi
    shl         rdx, 4

    movdqa      xmm7, [GLOBAL(rd)]

    lea         rax, [GLOBAL(k0_k5)]
    add         rax, rdx
    mov         rdi, arg(2)             ;output_ptr

    cmp         esi, DWORD PTR [rax]
    je          vp8_filter_block1d8_h4_ssse3

    movdqa      xmm4, XMMWORD PTR [rax]         ;k0_k5
    movdqa      xmm5, XMMWORD PTR [rax+256]     ;k2_k4
    movdqa      xmm6, XMMWORD PTR [rax+128]     ;k1_k3

    mov         rsi, arg(0)             ;src_ptr
    movsxd      rax, dword ptr arg(1)   ;src_pixels_per_line
    movsxd      rcx, dword ptr arg(4)   ;output_height

    movsxd      rdx, dword ptr arg(3)   ;output_pitch

    sub         rdi, rdx
;xmm3 free
.filter_block1d8_h6_rowloop_ssse3:
    movq        xmm0,   MMWORD PTR [rsi - 2]    ; -2 -1  0  1  2  3  4  5

    movq        xmm2,   MMWORD PTR [rsi + 3]    ;  3  4  5  6  7  8  9 10

    punpcklbw   xmm0,   xmm2                    ; -2  3 -1  4  0  5  1  6  2  7  3  8  4  9  5 10

    movdqa      xmm1,   xmm0
    pmaddubsw   xmm0,   xmm4

    movdqa      xmm2,   xmm1
    pshufb      xmm1,   [GLOBAL(shuf2bfrom1)]

    pshufb      xmm2,   [GLOBAL(shuf3bfrom1)]
    pmaddubsw   xmm1,   xmm5

    lea         rdi,    [rdi + rdx]
    pmaddubsw   xmm2,   xmm6

    lea         rsi,    [rsi + rax]
    dec         rcx

    paddsw      xmm0,   xmm1
    paddsw      xmm2,   xmm7

    paddsw      xmm0,   xmm2

    psraw       xmm0,   7

    packuswb    xmm0,   xmm0

    movq        MMWORD Ptr [rdi], xmm0
    jnz         .filter_block1d8_h6_rowloop_ssse3

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

vp8_filter_block1d8_h4_ssse3:
    movdqa      xmm5, XMMWORD PTR [rax+256]     ;k2_k4
    movdqa      xmm6, XMMWORD PTR [rax+128]     ;k1_k3

    movdqa      xmm3, XMMWORD PTR [GLOBAL(shuf2bfrom1)]
    movdqa      xmm4, XMMWORD PTR [GLOBAL(shuf3bfrom1)]

    mov         rsi, arg(0)             ;src_ptr

    movsxd      rax, dword ptr arg(1)   ;src_pixels_per_line
    movsxd      rcx, dword ptr arg(4)   ;output_height

    movsxd      rdx, dword ptr arg(3)   ;output_pitch

    sub         rdi, rdx

.filter_block1d8_h4_rowloop_ssse3:
    movq        xmm0,   MMWORD PTR [rsi - 2]    ; -2 -1  0  1  2  3  4  5

    movq        xmm1,   MMWORD PTR [rsi + 3]    ;  3  4  5  6  7  8  9 10

    punpcklbw   xmm0,   xmm1                    ; -2  3 -1  4  0  5  1  6  2  7  3  8  4  9  5 10

    movdqa      xmm2,   xmm0
    pshufb      xmm0,   xmm3

    pshufb      xmm2,   xmm4
    pmaddubsw   xmm0,   xmm5

    lea         rdi,    [rdi + rdx]
    pmaddubsw   xmm2,   xmm6

    lea         rsi,    [rsi + rax]
    dec         rcx

    paddsw      xmm0,   xmm7

    paddsw      xmm0,   xmm2

    psraw       xmm0,   7

    packuswb    xmm0,   xmm0

    movq        MMWORD Ptr [rdi], xmm0

    jnz         .filter_block1d8_h4_rowloop_ssse3

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret
;void vp8_filter_block1d16_h6_ssse3
;(
;    unsigned char  *src_ptr,
;    unsigned int    src_pixels_per_line,
;    unsigned char  *output_ptr,
;    unsigned int    output_pitch,
;    unsigned int    output_height,
;    unsigned int    vp8_filter_index
;)
global sym(vp8_filter_block1d16_h6_ssse3) PRIVATE
sym(vp8_filter_block1d16_h6_ssse3):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 6
    SAVE_XMM 7
    GET_GOT     rbx
    push        rsi
    push        rdi
    ; end prolog

    movsxd      rdx, DWORD PTR arg(5)           ;table index
    xor         rsi, rsi
    shl         rdx, 4      ;

    lea         rax, [GLOBAL(k0_k5)]
    add         rax, rdx

    mov         rdi, arg(2)                     ;output_ptr

    mov         rsi, arg(0)                     ;src_ptr

    movdqa      xmm4, XMMWORD PTR [rax]         ;k0_k5
    movdqa      xmm5, XMMWORD PTR [rax+256]     ;k2_k4
    movdqa      xmm6, XMMWORD PTR [rax+128]     ;k1_k3

    movsxd      rax, dword ptr arg(1)           ;src_pixels_per_line
    movsxd      rcx, dword ptr arg(4)           ;output_height
    movsxd      rdx, dword ptr arg(3)           ;output_pitch

.filter_block1d16_h6_rowloop_ssse3:
    movq        xmm0,   MMWORD PTR [rsi - 2]    ; -2 -1  0  1  2  3  4  5

    movq        xmm3,   MMWORD PTR [rsi + 3]    ;  3  4  5  6  7  8  9 10

    punpcklbw   xmm0,   xmm3                    ; -2  3 -1  4  0  5  1  6  2  7  3  8  4  9  5 10

    movdqa      xmm1,   xmm0
    pmaddubsw   xmm0,   xmm4

    movdqa      xmm2,   xmm1
    pshufb      xmm1,   [GLOBAL(shuf2bfrom1)]

    pshufb      xmm2,   [GLOBAL(shuf3bfrom1)]
    movq        xmm3,   MMWORD PTR [rsi +  6]

    pmaddubsw   xmm1,   xmm5
    movq        xmm7,   MMWORD PTR [rsi + 11]

    pmaddubsw   xmm2,   xmm6
    punpcklbw   xmm3,   xmm7

    paddsw      xmm0,   xmm1
    movdqa      xmm1,   xmm3

    pmaddubsw   xmm3,   xmm4
    paddsw      xmm0,   xmm2

    movdqa      xmm2,   xmm1
    paddsw      xmm0,   [GLOBAL(rd)]

    pshufb      xmm1,   [GLOBAL(shuf2bfrom1)]
    pshufb      xmm2,   [GLOBAL(shuf3bfrom1)]

    psraw       xmm0,   7
    pmaddubsw   xmm1,   xmm5

    pmaddubsw   xmm2,   xmm6
    packuswb    xmm0,   xmm0

    lea         rsi,    [rsi + rax]
    paddsw      xmm3,   xmm1

    paddsw      xmm3,   xmm2

    paddsw      xmm3,   [GLOBAL(rd)]

    psraw       xmm3,   7

    packuswb    xmm3,   xmm3

    punpcklqdq  xmm0,   xmm3

    movdqa      XMMWORD Ptr [rdi], xmm0

    lea         rdi,    [rdi + rdx]
    dec         rcx
    jnz         .filter_block1d16_h6_rowloop_ssse3

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

;void vp8_filter_block1d4_h6_ssse3
;(
;    unsigned char  *src_ptr,
;    unsigned int    src_pixels_per_line,
;    unsigned char  *output_ptr,
;    unsigned int    output_pitch,
;    unsigned int    output_height,
;    unsigned int    vp8_filter_index
;)
global sym(vp8_filter_block1d4_h6_ssse3) PRIVATE
sym(vp8_filter_block1d4_h6_ssse3):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 6
    SAVE_XMM 7
    GET_GOT     rbx
    push        rsi
    push        rdi
    ; end prolog

    movsxd      rdx, DWORD PTR arg(5)   ;table index
    xor         rsi, rsi
    shl         rdx, 4      ;

    lea         rax, [GLOBAL(k0_k5)]
    add         rax, rdx
    movdqa      xmm7, [GLOBAL(rd)]

    cmp         esi, DWORD PTR [rax]
    je          .vp8_filter_block1d4_h4_ssse3

    movdqa      xmm4, XMMWORD PTR [rax]         ;k0_k5
    movdqa      xmm5, XMMWORD PTR [rax+256]     ;k2_k4
    movdqa      xmm6, XMMWORD PTR [rax+128]     ;k1_k3

    mov         rsi, arg(0)             ;src_ptr
    mov         rdi, arg(2)             ;output_ptr
    movsxd      rax, dword ptr arg(1)   ;src_pixels_per_line
    movsxd      rcx, dword ptr arg(4)   ;output_height

    movsxd      rdx, dword ptr arg(3)   ;output_pitch

;xmm3 free
.filter_block1d4_h6_rowloop_ssse3:
    movdqu      xmm0,   XMMWORD PTR [rsi - 2]

    movdqa      xmm1, xmm0
    pshufb      xmm0, [GLOBAL(shuf1b)]

    movdqa      xmm2, xmm1
    pshufb      xmm1, [GLOBAL(shuf2b)]
    pmaddubsw   xmm0, xmm4
    pshufb      xmm2, [GLOBAL(shuf3b)]
    pmaddubsw   xmm1, xmm5

;--
    pmaddubsw   xmm2, xmm6

    lea         rsi,    [rsi + rax]
;--
    paddsw      xmm0, xmm1
    paddsw      xmm0, xmm7
    pxor        xmm1, xmm1
    paddsw      xmm0, xmm2
    psraw       xmm0, 7
    packuswb    xmm0, xmm0

    movd        DWORD PTR [rdi], xmm0

    add         rdi, rdx
    dec         rcx
    jnz         .filter_block1d4_h6_rowloop_ssse3

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    UNSHADOW_ARGS
    pop         rbp
    ret

.vp8_filter_block1d4_h4_ssse3:
    movdqa      xmm5, XMMWORD PTR [rax+256]     ;k2_k4
    movdqa      xmm6, XMMWORD PTR [rax+128]     ;k1_k3
    movdqa      xmm0, XMMWORD PTR [GLOBAL(shuf2b)]
    movdqa      xmm3, XMMWORD PTR [GLOBAL(shuf3b)]

    mov         rsi, arg(0)             ;src_ptr
    mov         rdi, arg(2)             ;output_ptr
    movsxd      rax, dword ptr arg(1)   ;src_pixels_per_line
    movsxd      rcx, dword ptr arg(4)   ;output_height

    movsxd      rdx, dword ptr arg(3)   ;output_pitch

.filter_block1d4_h4_rowloop_ssse3:
    movdqu      xmm1,   XMMWORD PTR [rsi - 2]

    movdqa      xmm2, xmm1
    pshufb      xmm1, xmm0 ;;[GLOBAL(shuf2b)]
    pshufb      xmm2, xmm3 ;;[GLOBAL(shuf3b)]
    pmaddubsw   xmm1, xmm5

;--
    pmaddubsw   xmm2, xmm6

    lea         rsi,    [rsi + rax]
;--
    paddsw      xmm1, xmm7
    paddsw      xmm1, xmm2
    psraw       xmm1, 7
    packuswb    xmm1, xmm1

    movd        DWORD PTR [rdi], xmm1

    add         rdi, rdx
    dec         rcx
    jnz         .filter_block1d4_h4_rowloop_ssse3

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret



;void vp8_filter_block1d16_v6_ssse3
;(
;    unsigned char *src_ptr,
;    unsigned int   src_pitch,
;    unsigned char *output_ptr,
;    unsigned int   out_pitch,
;    unsigned int   output_height,
;    unsigned int   vp8_filter_index
;)
global sym(vp8_filter_block1d16_v6_ssse3) PRIVATE
sym(vp8_filter_block1d16_v6_ssse3):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 6
    SAVE_XMM 7
    GET_GOT     rbx
    push        rsi
    push        rdi
    ; end prolog

    movsxd      rdx, DWORD PTR arg(5)   ;table index
    xor         rsi, rsi
    shl         rdx, 4      ;

    lea         rax, [GLOBAL(k0_k5)]
    add         rax, rdx

    cmp         esi, DWORD PTR [rax]
    je          .vp8_filter_block1d16_v4_ssse3

    movdqa      xmm5, XMMWORD PTR [rax]         ;k0_k5
    movdqa      xmm6, XMMWORD PTR [rax+256]     ;k2_k4
    movdqa      xmm7, XMMWORD PTR [rax+128]     ;k1_k3

    mov         rsi, arg(0)             ;src_ptr
    movsxd      rdx, DWORD PTR arg(1)   ;pixels_per_line
    mov         rdi, arg(2)             ;output_ptr

%if ABI_IS_32BIT=0
    movsxd      r8, DWORD PTR arg(3)    ;out_pitch
%endif
    mov         rax, rsi
    movsxd      rcx, DWORD PTR arg(4)   ;output_height
    add         rax, rdx


.vp8_filter_block1d16_v6_ssse3_loop:
    movq        xmm1, MMWORD PTR [rsi]                  ;A
    movq        xmm2, MMWORD PTR [rsi + rdx]            ;B
    movq        xmm3, MMWORD PTR [rsi + rdx * 2]        ;C
    movq        xmm4, MMWORD PTR [rax + rdx * 2]        ;D
    movq        xmm0, MMWORD PTR [rsi + rdx * 4]        ;E

    punpcklbw   xmm2, xmm4                  ;B D
    punpcklbw   xmm3, xmm0                  ;C E

    movq        xmm0, MMWORD PTR [rax + rdx * 4]        ;F

    pmaddubsw   xmm3, xmm6
    punpcklbw   xmm1, xmm0                  ;A F
    pmaddubsw   xmm2, xmm7
    pmaddubsw   xmm1, xmm5

    paddsw      xmm2, xmm3
    paddsw      xmm2, xmm1
    paddsw      xmm2, [GLOBAL(rd)]
    psraw       xmm2, 7
    packuswb    xmm2, xmm2

    movq        MMWORD PTR [rdi], xmm2          ;store the results

    movq        xmm1, MMWORD PTR [rsi + 8]                  ;A
    movq        xmm2, MMWORD PTR [rsi + rdx + 8]            ;B
    movq        xmm3, MMWORD PTR [rsi + rdx * 2 + 8]        ;C
    movq        xmm4, MMWORD PTR [rax + rdx * 2 + 8]        ;D
    movq        xmm0, MMWORD PTR [rsi + rdx * 4 + 8]        ;E

    punpcklbw   xmm2, xmm4                  ;B D
    punpcklbw   xmm3, xmm0                  ;C E

    movq        xmm0, MMWORD PTR [rax + rdx * 4 + 8]        ;F
    pmaddubsw   xmm3, xmm6
    punpcklbw   xmm1, xmm0                  ;A F
    pmaddubsw   xmm2, xmm7
    pmaddubsw   xmm1, xmm5

    add         rsi,  rdx
    add         rax,  rdx
;--
;--
    paddsw      xmm2, xmm3
    paddsw      xmm2, xmm1
    paddsw      xmm2, [GLOBAL(rd)]
    psraw       xmm2, 7
    packuswb    xmm2, xmm2

    movq        MMWORD PTR [rdi+8], xmm2

%if ABI_IS_32BIT
    add         rdi,        DWORD PTR arg(3) ;out_pitch
%else
    add         rdi,        r8
%endif
    dec         rcx
    jnz         .vp8_filter_block1d16_v6_ssse3_loop

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

.vp8_filter_block1d16_v4_ssse3:
    movdqa      xmm6, XMMWORD PTR [rax+256]     ;k2_k4
    movdqa      xmm7, XMMWORD PTR [rax+128]     ;k1_k3

    mov         rsi, arg(0)             ;src_ptr
    movsxd      rdx, DWORD PTR arg(1)   ;pixels_per_line
    mov         rdi, arg(2)             ;output_ptr

%if ABI_IS_32BIT=0
    movsxd      r8, DWORD PTR arg(3)    ;out_pitch
%endif
    mov         rax, rsi
    movsxd      rcx, DWORD PTR arg(4)   ;output_height
    add         rax, rdx

.vp8_filter_block1d16_v4_ssse3_loop:
    movq        xmm2, MMWORD PTR [rsi + rdx]            ;B
    movq        xmm3, MMWORD PTR [rsi + rdx * 2]        ;C
    movq        xmm4, MMWORD PTR [rax + rdx * 2]        ;D
    movq        xmm0, MMWORD PTR [rsi + rdx * 4]        ;E

    punpcklbw   xmm2, xmm4                  ;B D
    punpcklbw   xmm3, xmm0                  ;C E

    pmaddubsw   xmm3, xmm6
    pmaddubsw   xmm2, xmm7
    movq        xmm5, MMWORD PTR [rsi + rdx + 8]            ;B
    movq        xmm1, MMWORD PTR [rsi + rdx * 2 + 8]        ;C
    movq        xmm4, MMWORD PTR [rax + rdx * 2 + 8]        ;D
    movq        xmm0, MMWORD PTR [rsi + rdx * 4 + 8]        ;E

    paddsw      xmm2, [GLOBAL(rd)]
    paddsw      xmm2, xmm3
    psraw       xmm2, 7
    packuswb    xmm2, xmm2

    punpcklbw   xmm5, xmm4                  ;B D
    punpcklbw   xmm1, xmm0                  ;C E

    pmaddubsw   xmm1, xmm6
    pmaddubsw   xmm5, xmm7

    movdqa      xmm4, [GLOBAL(rd)]
    add         rsi,  rdx
    add         rax,  rdx
;--
;--
    paddsw      xmm5, xmm1
    paddsw      xmm5, xmm4
    psraw       xmm5, 7
    packuswb    xmm5, xmm5

    punpcklqdq  xmm2, xmm5

    movdqa       XMMWORD PTR [rdi], xmm2

%if ABI_IS_32BIT
    add         rdi,        DWORD PTR arg(3) ;out_pitch
%else
    add         rdi,        r8
%endif
    dec         rcx
    jnz         .vp8_filter_block1d16_v4_ssse3_loop

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

;void vp8_filter_block1d8_v6_ssse3
;(
;    unsigned char *src_ptr,
;    unsigned int   src_pitch,
;    unsigned char *output_ptr,
;    unsigned int   out_pitch,
;    unsigned int   output_height,
;    unsigned int   vp8_filter_index
;)
global sym(vp8_filter_block1d8_v6_ssse3) PRIVATE
sym(vp8_filter_block1d8_v6_ssse3):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 6
    SAVE_XMM 7
    GET_GOT     rbx
    push        rsi
    push        rdi
    ; end prolog

    movsxd      rdx, DWORD PTR arg(5)   ;table index
    xor         rsi, rsi
    shl         rdx, 4      ;

    lea         rax, [GLOBAL(k0_k5)]
    add         rax, rdx

    movsxd      rdx, DWORD PTR arg(1)   ;pixels_per_line
    mov         rdi, arg(2)             ;output_ptr
%if ABI_IS_32BIT=0
    movsxd      r8, DWORD PTR arg(3)    ; out_pitch
%endif
    movsxd      rcx, DWORD PTR arg(4)   ;[output_height]

    cmp         esi, DWORD PTR [rax]
    je          .vp8_filter_block1d8_v4_ssse3

    movdqa      xmm5, XMMWORD PTR [rax]         ;k0_k5
    movdqa      xmm6, XMMWORD PTR [rax+256]     ;k2_k4
    movdqa      xmm7, XMMWORD PTR [rax+128]     ;k1_k3

    mov         rsi, arg(0)             ;src_ptr

    mov         rax, rsi
    add         rax, rdx

.vp8_filter_block1d8_v6_ssse3_loop:
    movq        xmm1, MMWORD PTR [rsi]                  ;A
    movq        xmm2, MMWORD PTR [rsi + rdx]            ;B
    movq        xmm3, MMWORD PTR [rsi + rdx * 2]        ;C
    movq        xmm4, MMWORD PTR [rax + rdx * 2]        ;D
    movq        xmm0, MMWORD PTR [rsi + rdx * 4]        ;E

    punpcklbw   xmm2, xmm4                  ;B D
    punpcklbw   xmm3, xmm0                  ;C E

    movq        xmm0, MMWORD PTR [rax + rdx * 4]        ;F
    movdqa      xmm4, [GLOBAL(rd)]

    pmaddubsw   xmm3, xmm6
    punpcklbw   xmm1, xmm0                  ;A F
    pmaddubsw   xmm2, xmm7
    pmaddubsw   xmm1, xmm5
    add         rsi,  rdx
    add         rax,  rdx
;--
;--
    paddsw      xmm2, xmm3
    paddsw      xmm2, xmm1
    paddsw      xmm2, xmm4
    psraw       xmm2, 7
    packuswb    xmm2, xmm2

    movq        MMWORD PTR [rdi], xmm2

%if ABI_IS_32BIT
    add         rdi,        DWORD PTR arg(3) ;[out_pitch]
%else
    add         rdi,        r8
%endif
    dec         rcx
    jnz         .vp8_filter_block1d8_v6_ssse3_loop

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

.vp8_filter_block1d8_v4_ssse3:
    movdqa      xmm6, XMMWORD PTR [rax+256]     ;k2_k4
    movdqa      xmm7, XMMWORD PTR [rax+128]     ;k1_k3
    movdqa      xmm5, [GLOBAL(rd)]

    mov         rsi, arg(0)             ;src_ptr

    mov         rax, rsi
    add         rax, rdx

.vp8_filter_block1d8_v4_ssse3_loop:
    movq        xmm2, MMWORD PTR [rsi + rdx]            ;B
    movq        xmm3, MMWORD PTR [rsi + rdx * 2]        ;C
    movq        xmm4, MMWORD PTR [rax + rdx * 2]        ;D
    movq        xmm0, MMWORD PTR [rsi + rdx * 4]        ;E

    punpcklbw   xmm2, xmm4                  ;B D
    punpcklbw   xmm3, xmm0                  ;C E

    pmaddubsw   xmm3, xmm6
    pmaddubsw   xmm2, xmm7
    add         rsi,  rdx
    add         rax,  rdx
;--
;--
    paddsw      xmm2, xmm3
    paddsw      xmm2, xmm5
    psraw       xmm2, 7
    packuswb    xmm2, xmm2

    movq        MMWORD PTR [rdi], xmm2

%if ABI_IS_32BIT
    add         rdi,        DWORD PTR arg(3) ;[out_pitch]
%else
    add         rdi,        r8
%endif
    dec         rcx
    jnz         .vp8_filter_block1d8_v4_ssse3_loop

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret
;void vp8_filter_block1d4_v6_ssse3
;(
;    unsigned char *src_ptr,
;    unsigned int   src_pitch,
;    unsigned char *output_ptr,
;    unsigned int   out_pitch,
;    unsigned int   output_height,
;    unsigned int   vp8_filter_index
;)
global sym(vp8_filter_block1d4_v6_ssse3) PRIVATE
sym(vp8_filter_block1d4_v6_ssse3):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 6
    GET_GOT     rbx
    push        rsi
    push        rdi
    ; end prolog

    movsxd      rdx, DWORD PTR arg(5)   ;table index
    xor         rsi, rsi
    shl         rdx, 4      ;

    lea         rax, [GLOBAL(k0_k5)]
    add         rax, rdx

    movsxd      rdx, DWORD PTR arg(1)   ;pixels_per_line
    mov         rdi, arg(2)             ;output_ptr
%if ABI_IS_32BIT=0
    movsxd      r8, DWORD PTR arg(3)    ; out_pitch
%endif
    movsxd      rcx, DWORD PTR arg(4)   ;[output_height]

    cmp         esi, DWORD PTR [rax]
    je          .vp8_filter_block1d4_v4_ssse3

    movq        mm5, MMWORD PTR [rax]         ;k0_k5
    movq        mm6, MMWORD PTR [rax+256]     ;k2_k4
    movq        mm7, MMWORD PTR [rax+128]     ;k1_k3

    mov         rsi, arg(0)             ;src_ptr

    mov         rax, rsi
    add         rax, rdx

.vp8_filter_block1d4_v6_ssse3_loop:
    movd        mm1, DWORD PTR [rsi]                  ;A
    movd        mm2, DWORD PTR [rsi + rdx]            ;B
    movd        mm3, DWORD PTR [rsi + rdx * 2]        ;C
    movd        mm4, DWORD PTR [rax + rdx * 2]        ;D
    movd        mm0, DWORD PTR [rsi + rdx * 4]        ;E

    punpcklbw   mm2, mm4                  ;B D
    punpcklbw   mm3, mm0                  ;C E

    movd        mm0, DWORD PTR [rax + rdx * 4]        ;F

    movq        mm4, [GLOBAL(rd)]

    pmaddubsw   mm3, mm6
    punpcklbw   mm1, mm0                  ;A F
    pmaddubsw   mm2, mm7
    pmaddubsw   mm1, mm5
    add         rsi,  rdx
    add         rax,  rdx
;--
;--
    paddsw      mm2, mm3
    paddsw      mm2, mm1
    paddsw      mm2, mm4
    psraw       mm2, 7
    packuswb    mm2, mm2

    movd        DWORD PTR [rdi], mm2

%if ABI_IS_32BIT
    add         rdi,        DWORD PTR arg(3) ;[out_pitch]
%else
    add         rdi,        r8
%endif
    dec         rcx
    jnz         .vp8_filter_block1d4_v6_ssse3_loop

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    UNSHADOW_ARGS
    pop         rbp
    ret

.vp8_filter_block1d4_v4_ssse3:
    movq        mm6, MMWORD PTR [rax+256]     ;k2_k4
    movq        mm7, MMWORD PTR [rax+128]     ;k1_k3
    movq        mm5, MMWORD PTR [GLOBAL(rd)]

    mov         rsi, arg(0)             ;src_ptr

    mov         rax, rsi
    add         rax, rdx

.vp8_filter_block1d4_v4_ssse3_loop:
    movd        mm2, DWORD PTR [rsi + rdx]            ;B
    movd        mm3, DWORD PTR [rsi + rdx * 2]        ;C
    movd        mm4, DWORD PTR [rax + rdx * 2]        ;D
    movd        mm0, DWORD PTR [rsi + rdx * 4]        ;E

    punpcklbw   mm2, mm4                  ;B D
    punpcklbw   mm3, mm0                  ;C E

    pmaddubsw   mm3, mm6
    pmaddubsw   mm2, mm7
    add         rsi,  rdx
    add         rax,  rdx
;--
;--
    paddsw      mm2, mm3
    paddsw      mm2, mm5
    psraw       mm2, 7
    packuswb    mm2, mm2

    movd        DWORD PTR [rdi], mm2

%if ABI_IS_32BIT
    add         rdi,        DWORD PTR arg(3) ;[out_pitch]
%else
    add         rdi,        r8
%endif
    dec         rcx
    jnz         .vp8_filter_block1d4_v4_ssse3_loop

    ; begin epilog
    pop rdi
    pop rsi
    RESTORE_GOT
    UNSHADOW_ARGS
    pop         rbp
    ret

;void vp8_bilinear_predict16x16_ssse3
;(
;    unsigned char  *src_ptr,
;    int   src_pixels_per_line,
;    int  xoffset,
;    int  yoffset,
;    unsigned char *dst_ptr,
;    int dst_pitch
;)
global sym(vp8_bilinear_predict16x16_ssse3) PRIVATE
sym(vp8_bilinear_predict16x16_ssse3):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 6
    SAVE_XMM 7
    GET_GOT     rbx
    push        rsi
    push        rdi
    ; end prolog

        lea         rcx,        [GLOBAL(vp8_bilinear_filters_ssse3)]
        movsxd      rax,        dword ptr arg(2)    ; xoffset

        cmp         rax,        0                   ; skip first_pass filter if xoffset=0
        je          .b16x16_sp_only

        shl         rax,        4
        lea         rax,        [rax + rcx]         ; HFilter

        mov         rdi,        arg(4)              ; dst_ptr
        mov         rsi,        arg(0)              ; src_ptr
        movsxd      rdx,        dword ptr arg(5)    ; dst_pitch

        movdqa      xmm1,       [rax]

        movsxd      rax,        dword ptr arg(3)    ; yoffset

        cmp         rax,        0                   ; skip second_pass filter if yoffset=0
        je          .b16x16_fp_only

        shl         rax,        4
        lea         rax,        [rax + rcx]         ; VFilter

        lea         rcx,        [rdi+rdx*8]
        lea         rcx,        [rcx+rdx*8]
        movsxd      rdx,        dword ptr arg(1)    ; src_pixels_per_line

        movdqa      xmm2,       [rax]

%if ABI_IS_32BIT=0
        movsxd      r8,         dword ptr arg(5)    ; dst_pitch
%endif
        movq        xmm3,       [rsi]               ; 00 01 02 03 04 05 06 07
        movq        xmm5,       [rsi+1]             ; 01 02 03 04 05 06 07 08

        punpcklbw   xmm3,       xmm5                ; 00 01 01 02 02 03 03 04 04 05 05 06 06 07 07 08
        movq        xmm4,       [rsi+8]             ; 08 09 10 11 12 13 14 15

        movq        xmm5,       [rsi+9]             ; 09 10 11 12 13 14 15 16

        lea         rsi,        [rsi + rdx]         ; next line

        pmaddubsw   xmm3,       xmm1                ; 00 02 04 06 08 10 12 14

        punpcklbw   xmm4,       xmm5                ; 08 09 09 10 10 11 11 12 12 13 13 14 14 15 15 16
        pmaddubsw   xmm4,       xmm1                ; 01 03 05 07 09 11 13 15

        paddw       xmm3,       [GLOBAL(rd)]        ; xmm3 += round value
        psraw       xmm3,       VP8_FILTER_SHIFT    ; xmm3 /= 128

        paddw       xmm4,       [GLOBAL(rd)]        ; xmm4 += round value
        psraw       xmm4,       VP8_FILTER_SHIFT    ; xmm4 /= 128

        movdqa      xmm7,       xmm3
        packuswb    xmm7,       xmm4                ; 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15

.next_row:
        movq        xmm6,       [rsi]               ; 00 01 02 03 04 05 06 07
        movq        xmm5,       [rsi+1]             ; 01 02 03 04 05 06 07 08

        punpcklbw   xmm6,       xmm5
        movq        xmm4,       [rsi+8]             ; 08 09 10 11 12 13 14 15

        movq        xmm5,       [rsi+9]             ; 09 10 11 12 13 14 15 16
        lea         rsi,        [rsi + rdx]         ; next line

        pmaddubsw   xmm6,       xmm1

        punpcklbw   xmm4,       xmm5
        pmaddubsw   xmm4,       xmm1

        paddw       xmm6,       [GLOBAL(rd)]        ; xmm6 += round value
        psraw       xmm6,       VP8_FILTER_SHIFT    ; xmm6 /= 128

        paddw       xmm4,       [GLOBAL(rd)]        ; xmm4 += round value
        psraw       xmm4,       VP8_FILTER_SHIFT    ; xmm4 /= 128

        packuswb    xmm6,       xmm4
        movdqa      xmm5,       xmm7

        punpcklbw   xmm5,       xmm6
        pmaddubsw   xmm5,       xmm2

        punpckhbw   xmm7,       xmm6
        pmaddubsw   xmm7,       xmm2

        paddw       xmm5,       [GLOBAL(rd)]        ; xmm5 += round value
        psraw       xmm5,       VP8_FILTER_SHIFT    ; xmm5 /= 128

        paddw       xmm7,       [GLOBAL(rd)]        ; xmm7 += round value
        psraw       xmm7,       VP8_FILTER_SHIFT    ; xmm7 /= 128

        packuswb    xmm5,       xmm7
        movdqa      xmm7,       xmm6

        movdqa      [rdi],      xmm5                ; store the results in the destination
%if ABI_IS_32BIT
        add         rdi,        DWORD PTR arg(5)    ; dst_pitch
%else
        add         rdi,        r8
%endif

        cmp         rdi,        rcx
        jne         .next_row

        jmp         .done

.b16x16_sp_only:
        movsxd      rax,        dword ptr arg(3)    ; yoffset
        shl         rax,        4
        lea         rax,        [rax + rcx]         ; VFilter

        mov         rdi,        arg(4)              ; dst_ptr
        mov         rsi,        arg(0)              ; src_ptr
        movsxd      rdx,        dword ptr arg(5)    ; dst_pitch

        movdqa      xmm1,       [rax]               ; VFilter

        lea         rcx,        [rdi+rdx*8]
        lea         rcx,        [rcx+rdx*8]
        movsxd      rax,        dword ptr arg(1)    ; src_pixels_per_line

        ; get the first horizontal line done
        movq        xmm4,       [rsi]               ; load row 0
        movq        xmm2,       [rsi + 8]           ; load row 0

        lea         rsi,        [rsi + rax]         ; next line
.next_row_sp:
        movq        xmm3,       [rsi]               ; load row + 1
        movq        xmm5,       [rsi + 8]           ; load row + 1

        punpcklbw   xmm4,       xmm3
        punpcklbw   xmm2,       xmm5

        pmaddubsw   xmm4,       xmm1
        movq        xmm7,       [rsi + rax]         ; load row + 2

        pmaddubsw   xmm2,       xmm1
        movq        xmm6,       [rsi + rax + 8]     ; load row + 2

        punpcklbw   xmm3,       xmm7
        punpcklbw   xmm5,       xmm6

        pmaddubsw   xmm3,       xmm1
        paddw       xmm4,       [GLOBAL(rd)]

        pmaddubsw   xmm5,       xmm1
        paddw       xmm2,       [GLOBAL(rd)]

        psraw       xmm4,       VP8_FILTER_SHIFT
        psraw       xmm2,       VP8_FILTER_SHIFT

        packuswb    xmm4,       xmm2
        paddw       xmm3,       [GLOBAL(rd)]

        movdqa      [rdi],      xmm4                ; store row 0
        paddw       xmm5,       [GLOBAL(rd)]

        psraw       xmm3,       VP8_FILTER_SHIFT
        psraw       xmm5,       VP8_FILTER_SHIFT

        packuswb    xmm3,       xmm5
        movdqa      xmm4,       xmm7

        movdqa      [rdi + rdx],xmm3                ; store row 1
        lea         rsi,        [rsi + 2*rax]

        movdqa      xmm2,       xmm6
        lea         rdi,        [rdi + 2*rdx]

        cmp         rdi,        rcx
        jne         .next_row_sp

        jmp         .done

.b16x16_fp_only:
        lea         rcx,        [rdi+rdx*8]
        lea         rcx,        [rcx+rdx*8]
        movsxd      rax,        dword ptr arg(1)    ; src_pixels_per_line

.next_row_fp:
        movq        xmm2,       [rsi]               ; 00 01 02 03 04 05 06 07
        movq        xmm4,       [rsi+1]             ; 01 02 03 04 05 06 07 08

        punpcklbw   xmm2,       xmm4
        movq        xmm3,       [rsi+8]             ; 08 09 10 11 12 13 14 15

        pmaddubsw   xmm2,       xmm1
        movq        xmm4,       [rsi+9]             ; 09 10 11 12 13 14 15 16

        lea         rsi,        [rsi + rax]         ; next line
        punpcklbw   xmm3,       xmm4

        pmaddubsw   xmm3,       xmm1
        movq        xmm5,       [rsi]

        paddw       xmm2,       [GLOBAL(rd)]
        movq        xmm7,       [rsi+1]

        movq        xmm6,       [rsi+8]
        psraw       xmm2,       VP8_FILTER_SHIFT

        punpcklbw   xmm5,       xmm7
        movq        xmm7,       [rsi+9]

        paddw       xmm3,       [GLOBAL(rd)]
        pmaddubsw   xmm5,       xmm1

        psraw       xmm3,       VP8_FILTER_SHIFT
        punpcklbw   xmm6,       xmm7

        packuswb    xmm2,       xmm3
        pmaddubsw   xmm6,       xmm1

        movdqa      [rdi],      xmm2                ; store the results in the destination
        paddw       xmm5,       [GLOBAL(rd)]

        lea         rdi,        [rdi + rdx]         ; dst_pitch
        psraw       xmm5,       VP8_FILTER_SHIFT

        paddw       xmm6,       [GLOBAL(rd)]
        psraw       xmm6,       VP8_FILTER_SHIFT

        packuswb    xmm5,       xmm6
        lea         rsi,        [rsi + rax]         ; next line

        movdqa      [rdi],      xmm5                ; store the results in the destination
        lea         rdi,        [rdi + rdx]         ; dst_pitch

        cmp         rdi,        rcx

        jne         .next_row_fp

.done:
    ; begin epilog
    pop         rdi
    pop         rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

;void vp8_bilinear_predict8x8_ssse3
;(
;    unsigned char  *src_ptr,
;    int   src_pixels_per_line,
;    int  xoffset,
;    int  yoffset,
;    unsigned char *dst_ptr,
;    int dst_pitch
;)
global sym(vp8_bilinear_predict8x8_ssse3) PRIVATE
sym(vp8_bilinear_predict8x8_ssse3):
    push        rbp
    mov         rbp, rsp
    SHADOW_ARGS_TO_STACK 6
    SAVE_XMM 7
    GET_GOT     rbx
    push        rsi
    push        rdi
    ; end prolog

    ALIGN_STACK 16, rax
    sub         rsp, 144                         ; reserve 144 bytes

        lea         rcx,        [GLOBAL(vp8_bilinear_filters_ssse3)]

        mov         rsi,        arg(0) ;src_ptr
        movsxd      rdx,        dword ptr arg(1) ;src_pixels_per_line

    ;Read 9-line unaligned data in and put them on stack. This gives a big
    ;performance boost.
        movdqu      xmm0,       [rsi]
        lea         rax,        [rdx + rdx*2]
        movdqu      xmm1,       [rsi+rdx]
        movdqu      xmm2,       [rsi+rdx*2]
        add         rsi,        rax
        movdqu      xmm3,       [rsi]
        movdqu      xmm4,       [rsi+rdx]
        movdqu      xmm5,       [rsi+rdx*2]
        add         rsi,        rax
        movdqu      xmm6,       [rsi]
        movdqu      xmm7,       [rsi+rdx]

        movdqa      XMMWORD PTR [rsp],            xmm0

        movdqu      xmm0,       [rsi+rdx*2]

        movdqa      XMMWORD PTR [rsp+16],         xmm1
        movdqa      XMMWORD PTR [rsp+32],         xmm2
        movdqa      XMMWORD PTR [rsp+48],         xmm3
        movdqa      XMMWORD PTR [rsp+64],         xmm4
        movdqa      XMMWORD PTR [rsp+80],         xmm5
        movdqa      XMMWORD PTR [rsp+96],         xmm6
        movdqa      XMMWORD PTR [rsp+112],        xmm7
        movdqa      XMMWORD PTR [rsp+128],        xmm0

        movsxd      rax,        dword ptr arg(2)    ; xoffset
        cmp         rax,        0                   ; skip first_pass filter if xoffset=0
        je          .b8x8_sp_only

        shl         rax,        4
        add         rax,        rcx                 ; HFilter

        mov         rdi,        arg(4)              ; dst_ptr
        movsxd      rdx,        dword ptr arg(5)    ; dst_pitch

        movdqa      xmm0,       [rax]

        movsxd      rax,        dword ptr arg(3)    ; yoffset
        cmp         rax,        0                   ; skip second_pass filter if yoffset=0
        je          .b8x8_fp_only

        shl         rax,        4
        lea         rax,        [rax + rcx]         ; VFilter

        lea         rcx,        [rdi+rdx*8]

        movdqa      xmm1,       [rax]

        ; get the first horizontal line done
        movdqa      xmm3,       [rsp]               ; 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15
        movdqa      xmm5,       xmm3                ; 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15 xx

        psrldq      xmm5,       1
        lea         rsp,        [rsp + 16]          ; next line

        punpcklbw   xmm3,       xmm5                ; 00 01 01 02 02 03 03 04 04 05 05 06 06 07 07 08
        pmaddubsw   xmm3,       xmm0                ; 00 02 04 06 08 10 12 14

        paddw       xmm3,       [GLOBAL(rd)]        ; xmm3 += round value
        psraw       xmm3,       VP8_FILTER_SHIFT    ; xmm3 /= 128

        movdqa      xmm7,       xmm3
        packuswb    xmm7,       xmm7                ; 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15

.next_row:
        movdqa      xmm6,       [rsp]               ; 00 01 02 03 04 05 06 07 08 09 10 11 12 13 14 15
        lea         rsp,        [rsp + 16]          ; next line

        movdqa      xmm5,       xmm6

        psrldq      xmm5,       1

        punpcklbw   xmm6,       xmm5
        pmaddubsw   xmm6,       xmm0

        paddw       xmm6,       [GLOBAL(rd)]        ; xmm6 += round value
        psraw       xmm6,       VP8_FILTER_SHIFT    ; xmm6 /= 128

        packuswb    xmm6,       xmm6

        punpcklbw   xmm7,       xmm6
        pmaddubsw   xmm7,       xmm1

        paddw       xmm7,       [GLOBAL(rd)]        ; xmm7 += round value
        psraw       xmm7,       VP8_FILTER_SHIFT    ; xmm7 /= 128

        packuswb    xmm7,       xmm7

        movq        [rdi],      xmm7                ; store the results in the destination
        lea         rdi,        [rdi + rdx]

        movdqa      xmm7,       xmm6

        cmp         rdi,        rcx
        jne         .next_row

        jmp         .done8x8

.b8x8_sp_only:
        movsxd      rax,        dword ptr arg(3)    ; yoffset
        shl         rax,        4
        lea         rax,        [rax + rcx]         ; VFilter

        mov         rdi,        arg(4) ;dst_ptr
        movsxd      rdx,        dword ptr arg(5)    ; dst_pitch

        movdqa      xmm0,       [rax]               ; VFilter

        movq        xmm1,       XMMWORD PTR [rsp]
        movq        xmm2,       XMMWORD PTR [rsp+16]

        movq        xmm3,       XMMWORD PTR [rsp+32]
        punpcklbw   xmm1,       xmm2

        movq        xmm4,       XMMWORD PTR [rsp+48]
        punpcklbw   xmm2,       xmm3

        movq        xmm5,       XMMWORD PTR [rsp+64]
        punpcklbw   xmm3,       xmm4

        movq        xmm6,       XMMWORD PTR [rsp+80]
        punpcklbw   xmm4,       xmm5

        movq        xmm7,       XMMWORD PTR [rsp+96]
        punpcklbw   xmm5,       xmm6

        pmaddubsw   xmm1,       xmm0
        pmaddubsw   xmm2,       xmm0

        pmaddubsw   xmm3,       xmm0
        pmaddubsw   xmm4,       xmm0

        pmaddubsw   xmm5,       xmm0
        punpcklbw   xmm6,       xmm7

        pmaddubsw   xmm6,       xmm0
        paddw       xmm1,       [GLOBAL(rd)]

        paddw       xmm2,       [GLOBAL(rd)]
        psraw       xmm1,       VP8_FILTER_SHIFT

        paddw       xmm3,       [GLOBAL(rd)]
        psraw       xmm2,       VP8_FILTER_SHIFT

        paddw       xmm4,       [GLOBAL(rd)]
        psraw       xmm3,       VP8_FILTER_SHIFT

        paddw       xmm5,       [GLOBAL(rd)]
        psraw       xmm4,       VP8_FILTER_SHIFT

        paddw       xmm6,       [GLOBAL(rd)]
        psraw       xmm5,       VP8_FILTER_SHIFT

        psraw       xmm6,       VP8_FILTER_SHIFT
        packuswb    xmm1,       xmm1

        packuswb    xmm2,       xmm2
        movq        [rdi],      xmm1

        packuswb    xmm3,       xmm3
        movq        [rdi+rdx],  xmm2

        packuswb    xmm4,       xmm4
        movq        xmm1,       XMMWORD PTR [rsp+112]

        lea         rdi,        [rdi + 2*rdx]
        movq        xmm2,       XMMWORD PTR [rsp+128]

        packuswb    xmm5,       xmm5
        movq        [rdi],      xmm3

        packuswb    xmm6,       xmm6
        movq        [rdi+rdx],  xmm4

        lea         rdi,        [rdi + 2*rdx]
        punpcklbw   xmm7,       xmm1

        movq        [rdi],      xmm5
        pmaddubsw   xmm7,       xmm0

        movq        [rdi+rdx],  xmm6
        punpcklbw   xmm1,       xmm2

        pmaddubsw   xmm1,       xmm0
        paddw       xmm7,       [GLOBAL(rd)]

        psraw       xmm7,       VP8_FILTER_SHIFT
        paddw       xmm1,       [GLOBAL(rd)]

        psraw       xmm1,       VP8_FILTER_SHIFT
        packuswb    xmm7,       xmm7

        packuswb    xmm1,       xmm1
        lea         rdi,        [rdi + 2*rdx]

        movq        [rdi],      xmm7

        movq        [rdi+rdx],  xmm1
        lea         rsp,        [rsp + 144]

        jmp         .done8x8

.b8x8_fp_only:
        lea         rcx,        [rdi+rdx*8]

.next_row_fp:
        movdqa      xmm1,       XMMWORD PTR [rsp]
        movdqa      xmm3,       XMMWORD PTR [rsp+16]

        movdqa      xmm2,       xmm1
        movdqa      xmm5,       XMMWORD PTR [rsp+32]

        psrldq      xmm2,       1
        movdqa      xmm7,       XMMWORD PTR [rsp+48]

        movdqa      xmm4,       xmm3
        psrldq      xmm4,       1

        movdqa      xmm6,       xmm5
        psrldq      xmm6,       1

        punpcklbw   xmm1,       xmm2
        pmaddubsw   xmm1,       xmm0

        punpcklbw   xmm3,       xmm4
        pmaddubsw   xmm3,       xmm0

        punpcklbw   xmm5,       xmm6
        pmaddubsw   xmm5,       xmm0

        movdqa      xmm2,       xmm7
        psrldq      xmm2,       1

        punpcklbw   xmm7,       xmm2
        pmaddubsw   xmm7,       xmm0

        paddw       xmm1,       [GLOBAL(rd)]
        psraw       xmm1,       VP8_FILTER_SHIFT

        paddw       xmm3,       [GLOBAL(rd)]
        psraw       xmm3,       VP8_FILTER_SHIFT

        paddw       xmm5,       [GLOBAL(rd)]
        psraw       xmm5,       VP8_FILTER_SHIFT

        paddw       xmm7,       [GLOBAL(rd)]
        psraw       xmm7,       VP8_FILTER_SHIFT

        packuswb    xmm1,       xmm1
        packuswb    xmm3,       xmm3

        packuswb    xmm5,       xmm5
        movq        [rdi],      xmm1

        packuswb    xmm7,       xmm7
        movq        [rdi+rdx],  xmm3

        lea         rdi,        [rdi + 2*rdx]
        movq        [rdi],      xmm5

        lea         rsp,        [rsp + 4*16]
        movq        [rdi+rdx],  xmm7

        lea         rdi,        [rdi + 2*rdx]
        cmp         rdi,        rcx

        jne         .next_row_fp

        lea         rsp,        [rsp + 16]

.done8x8:
    ;add rsp, 144
    pop         rsp
    ; begin epilog
    pop         rdi
    pop         rsi
    RESTORE_GOT
    RESTORE_XMM
    UNSHADOW_ARGS
    pop         rbp
    ret

SECTION_RODATA
align 16
shuf1b:
    db 0, 5, 1, 6, 2, 7, 3, 8, 4, 9, 5, 10, 6, 11, 7, 12
shuf2b:
    db 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7, 9, 8, 10, 9, 11
shuf3b:
    db 1, 3, 2, 4, 3, 5, 4, 6, 5, 7, 6, 8, 7, 9, 8, 10

align 16
shuf2bfrom1:
    db  4, 8, 6, 1, 8, 3, 1, 5, 3, 7, 5, 9, 7,11, 9,13
align 16
shuf3bfrom1:
    db  2, 6, 4, 8, 6, 1, 8, 3, 1, 5, 3, 7, 5, 9, 7,11

align 16
rd:
    times 8 dw 0x40

align 16
k0_k5:
    times 8 db 0, 0             ;placeholder
    times 8 db 0, 0
    times 8 db 2, 1
    times 8 db 0, 0
    times 8 db 3, 3
    times 8 db 0, 0
    times 8 db 1, 2
    times 8 db 0, 0
k1_k3:
    times 8 db  0,    0         ;placeholder
    times 8 db  -6,  12
    times 8 db -11,  36
    times 8 db  -9,  50
    times 8 db -16,  77
    times 8 db  -6,  93
    times 8 db  -8, 108
    times 8 db  -1, 123
k2_k4:
    times 8 db 128,    0        ;placeholder
    times 8 db 123,   -1
    times 8 db 108,   -8
    times 8 db  93,   -6
    times 8 db  77,  -16
    times 8 db  50,   -9
    times 8 db  36,  -11
    times 8 db  12,   -6
align 16
vp8_bilinear_filters_ssse3:
    times 8 db 128, 0
    times 8 db 112, 16
    times 8 db 96,  32
    times 8 db 80,  48
    times 8 db 64,  64
    times 8 db 48,  80
    times 8 db 32,  96
    times 8 db 16,  112