;*****************************************************************************
;* sad-a.asm: h264 encoder library
;*****************************************************************************
;* Copyright (C) 2003-2008 x264 project
;*
;* Authors: Loren Merritt <lorenm@u.washington.edu>
;*          Laurent Aimar <fenrir@via.ecp.fr>
;*          Jason Garrett-Glaser <darkshikari@gmail.com>
;*          Alex Izvorski <aizvorksi@gmail.com>
;*
;* This program is free software; you can redistribute it and/or modify
;* it under the terms of the GNU General Public License as published by
;* the Free Software Foundation; either version 2 of the License, or
;* (at your option) any later version.
;*
;* This program is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;* GNU General Public License for more details.
;*
;* You should have received a copy of the GNU General Public License
;* along with this program; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
;*****************************************************************************

%include "x86inc.asm"
%include "x86util.asm"

SECTION_RODATA
pb_3: times 16 db 3
sw_64: dd 64

SECTION .text

;=============================================================================
; SAD MMX
;=============================================================================

%macro SAD_INC_2x16P 0
    movq    mm1,    [r0]
    movq    mm2,    [r0+8]
    movq    mm3,    [r0+r1]
    movq    mm4,    [r0+r1+8]
    psadbw  mm1,    [r2]
    psadbw  mm2,    [r2+8]
    psadbw  mm3,    [r2+r3]
    psadbw  mm4,    [r2+r3+8]
    lea     r0,     [r0+2*r1]
    paddw   mm1,    mm2
    paddw   mm3,    mm4
    lea     r2,     [r2+2*r3]
    paddw   mm0,    mm1
    paddw   mm0,    mm3
%endmacro

%macro SAD_INC_2x8P 0
    movq    mm1,    [r0]
    movq    mm2,    [r0+r1]
    psadbw  mm1,    [r2]
    psadbw  mm2,    [r2+r3]
    lea     r0,     [r0+2*r1]
    paddw   mm0,    mm1
    paddw   mm0,    mm2
    lea     r2,     [r2+2*r3]
%endmacro

%macro SAD_INC_2x4P 0
    movd    mm1,    [r0]
    movd    mm2,    [r2]
    punpckldq mm1,  [r0+r1]
    punpckldq mm2,  [r2+r3]
    psadbw  mm1,    mm2
    paddw   mm0,    mm1
    lea     r0,     [r0+2*r1]
    lea     r2,     [r2+2*r3]
%endmacro

;-----------------------------------------------------------------------------
; int x264_pixel_sad_16x16_mmxext (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
%macro SAD 2
cglobal x264_pixel_sad_%1x%2_mmxext, 4,4
    pxor    mm0, mm0
%rep %2/2
    SAD_INC_2x%1P
%endrep
    movd    eax, mm0
    RET
%endmacro

SAD 16, 16
SAD 16,  8
SAD  8, 16
SAD  8,  8
SAD  8,  4
SAD  4,  8
SAD  4,  4


;=============================================================================
; SAD XMM
;=============================================================================

%macro SAD_END_SSE2 0
    movhlps m1, m0
    paddw   m0, m1
    movd   eax, m0
    RET
%endmacro

%macro SAD_W16 1
;-----------------------------------------------------------------------------
; int x264_pixel_sad_16x16_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_sad_16x16_%1, 4,4
    movdqu  m0, [r2]
    movdqu  m1, [r2+r3]
    lea     r2, [r2+2*r3]
    movdqu  m2, [r2]
    movdqu  m3, [r2+r3]
    lea     r2, [r2+2*r3]
    psadbw  m0, [r0]
    psadbw  m1, [r0+r1]
    lea     r0, [r0+2*r1]
    movdqu  m4, [r2]
    paddw   m0, m1
    psadbw  m2, [r0]
    psadbw  m3, [r0+r1]
    lea     r0, [r0+2*r1]
    movdqu  m5, [r2+r3]
    lea     r2, [r2+2*r3]
    paddw   m2, m3
    movdqu  m6, [r2]
    movdqu  m7, [r2+r3]
    lea     r2, [r2+2*r3]
    paddw   m0, m2
    psadbw  m4, [r0]
    psadbw  m5, [r0+r1]
    lea     r0, [r0+2*r1]
    movdqu  m1, [r2]
    paddw   m4, m5
    psadbw  m6, [r0]
    psadbw  m7, [r0+r1]
    lea     r0, [r0+2*r1]
    movdqu  m2, [r2+r3]
    lea     r2, [r2+2*r3]
    paddw   m6, m7
    movdqu  m3, [r2]
    paddw   m0, m4
    movdqu  m4, [r2+r3]
    lea     r2, [r2+2*r3]
    paddw   m0, m6
    psadbw  m1, [r0]
    psadbw  m2, [r0+r1]
    lea     r0, [r0+2*r1]
    movdqu  m5, [r2]
    paddw   m1, m2
    psadbw  m3, [r0]
    psadbw  m4, [r0+r1]
    lea     r0, [r0+2*r1]
    movdqu  m6, [r2+r3]
    lea     r2, [r2+2*r3]
    paddw   m3, m4
    movdqu  m7, [r2]
    paddw   m0, m1
    movdqu  m1, [r2+r3]
    paddw   m0, m3
    psadbw  m5, [r0]
    psadbw  m6, [r0+r1]
    lea     r0, [r0+2*r1]
    paddw   m5, m6
    psadbw  m7, [r0]
    psadbw  m1, [r0+r1]
    paddw   m7, m1
    paddw   m0, m5
    paddw   m0, m7
    SAD_END_SSE2

;-----------------------------------------------------------------------------
; int x264_pixel_sad_16x8_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_sad_16x8_%1, 4,4
    movdqu  m0, [r2]
    movdqu  m2, [r2+r3]
    lea     r2, [r2+2*r3]
    movdqu  m3, [r2]
    movdqu  m4, [r2+r3]
    psadbw  m0, [r0]
    psadbw  m2, [r0+r1]
    lea     r0, [r0+2*r1]
    psadbw  m3, [r0]
    psadbw  m4, [r0+r1]
    lea     r0, [r0+2*r1]
    lea     r2, [r2+2*r3]
    paddw   m0, m2
    paddw   m3, m4
    paddw   m0, m3
    movdqu  m1, [r2]
    movdqu  m2, [r2+r3]
    lea     r2, [r2+2*r3]
    movdqu  m3, [r2]
    movdqu  m4, [r2+r3]
    psadbw  m1, [r0]
    psadbw  m2, [r0+r1]
    lea     r0, [r0+2*r1]
    psadbw  m3, [r0]
    psadbw  m4, [r0+r1]
    lea     r0, [r0+2*r1]
    lea     r2, [r2+2*r3]
    paddw   m1, m2
    paddw   m3, m4
    paddw   m0, m1
    paddw   m0, m3
    SAD_END_SSE2
%endmacro

INIT_XMM
SAD_W16 sse2
%define movdqu lddqu
SAD_W16 sse3
%define movdqu movdqa
SAD_W16 sse2_aligned
%undef movdqu

%macro SAD_INC_4x8P_SSE 1
    movq    m1, [r0]
    movq    m2, [r0+r1]
    lea     r0, [r0+2*r1]
    movq    m3, [r2]
    movq    m4, [r2+r3]
    lea     r2, [r2+2*r3]
    movhps  m1, [r0]
    movhps  m2, [r0+r1]
    movhps  m3, [r2]
    movhps  m4, [r2+r3]
    lea     r0, [r0+2*r1]
    psadbw  m1, m3
    psadbw  m2, m4
    lea     r2, [r2+2*r3]
%if %1
    paddw   m0, m1
%else
    SWAP    m0, m1
%endif
    paddw   m0, m2
%endmacro

;Even on Nehalem, no sizes other than 8x16 benefit from this method.
cglobal x264_pixel_sad_8x16_sse2, 4,4
    SAD_INC_4x8P_SSE 0
    SAD_INC_4x8P_SSE 1
    SAD_INC_4x8P_SSE 1
    SAD_INC_4x8P_SSE 1
    SAD_END_SSE2
    RET

;-----------------------------------------------------------------------------
; void intra_sad_x3_16x16 ( uint8_t *fenc, uint8_t *fdec, int res[3] );
;-----------------------------------------------------------------------------

;xmm7: DC prediction    xmm6: H prediction  xmm5: V prediction
;xmm4: DC pred score    xmm3: H pred score  xmm2: V pred score
%macro INTRA_SAD16 1
cglobal x264_intra_sad_x3_16x16_%1,3,5
    pxor    mm0, mm0
    pxor    mm1, mm1
    psadbw  mm0, [r1-FDEC_STRIDE+0]
    psadbw  mm1, [r1-FDEC_STRIDE+8]
    paddw   mm0, mm1
    movd    r3d, mm0
%ifidn %1, ssse3
    mova  m1, [pb_3 GLOBAL]
%endif
%assign n 0
%rep 16
    movzx   r4d, byte [r1-1+FDEC_STRIDE*n]
    add     r3d, r4d
%assign n n+1
%endrep
    add     r3d, 16
    shr     r3d, 5
    imul    r3d, 0x01010101
    movd    m7, r3d
    mova    m5, [r1-FDEC_STRIDE]
%if mmsize==16
    pshufd  m7, m7, 0
%else
    mova    m1, [r1-FDEC_STRIDE+8]
    punpckldq m7, m7
%endif
    pxor    m4, m4
    pxor    m3, m3
    pxor    m2, m2
    mov     r3d, 15*FENC_STRIDE
.vloop:
    SPLATB  m6, r1+r3*2-1, m1
    mova    m0, [r0+r3]
    psadbw  m0, m7
    paddw   m4, m0
    mova    m0, [r0+r3]
    psadbw  m0, m5
    paddw   m2, m0
%if mmsize==8
    mova    m0, [r0+r3]
    psadbw  m0, m6
    paddw   m3, m0
    mova    m0, [r0+r3+8]
    psadbw  m0, m7
    paddw   m4, m0
    mova    m0, [r0+r3+8]
    psadbw  m0, m1
    paddw   m2, m0
    psadbw  m6, [r0+r3+8]
    paddw   m3, m6
%else
    psadbw  m6, [r0+r3]
    paddw   m3, m6
%endif
    add     r3d, -FENC_STRIDE
    jge .vloop
%if mmsize==16
    pslldq  m3, 4
    por     m3, m2
    movhlps m1, m3
    paddw   m3, m1
    movq  [r2+0], m3
    movhlps m1, m4
    paddw   m4, m1
%else
    movd  [r2+0], m2
    movd  [r2+4], m3
%endif
    movd  [r2+8], m4
    RET
%endmacro

INIT_MMX
%define SPLATB SPLATB_MMX
INTRA_SAD16 mmxext
INIT_XMM
INTRA_SAD16 sse2
%define SPLATB SPLATB_SSSE3
INTRA_SAD16 ssse3


;=============================================================================
; SAD x3/x4 MMX
;=============================================================================

%macro SAD_X3_START_1x8P 0
    movq    mm3,    [r0]
    movq    mm0,    [r1]
    movq    mm1,    [r2]
    movq    mm2,    [r3]
    psadbw  mm0,    mm3
    psadbw  mm1,    mm3
    psadbw  mm2,    mm3
%endmacro

%macro SAD_X3_1x8P 2
    movq    mm3,    [r0+%1]
    movq    mm4,    [r1+%2]
    movq    mm5,    [r2+%2]
    movq    mm6,    [r3+%2]
    psadbw  mm4,    mm3
    psadbw  mm5,    mm3
    psadbw  mm6,    mm3
    paddw   mm0,    mm4
    paddw   mm1,    mm5
    paddw   mm2,    mm6
%endmacro

%macro SAD_X3_START_2x4P 3
    movd      mm3,  [r0]
    movd      %1,   [r1]
    movd      %2,   [r2]
    movd      %3,   [r3]
    punpckldq mm3,  [r0+FENC_STRIDE]
    punpckldq %1,   [r1+r4]
    punpckldq %2,   [r2+r4]
    punpckldq %3,   [r3+r4]
    psadbw    %1,   mm3
    psadbw    %2,   mm3
    psadbw    %3,   mm3
%endmacro

%macro SAD_X3_2x16P 1
%if %1
    SAD_X3_START_1x8P
%else
    SAD_X3_1x8P 0, 0
%endif
    SAD_X3_1x8P 8, 8
    SAD_X3_1x8P FENC_STRIDE, r4
    SAD_X3_1x8P FENC_STRIDE+8, r4+8
    add     r0, 2*FENC_STRIDE
    lea     r1, [r1+2*r4]
    lea     r2, [r2+2*r4]
    lea     r3, [r3+2*r4]
%endmacro

%macro SAD_X3_2x8P 1
%if %1
    SAD_X3_START_1x8P
%else
    SAD_X3_1x8P 0, 0
%endif
    SAD_X3_1x8P FENC_STRIDE, r4
    add     r0, 2*FENC_STRIDE
    lea     r1, [r1+2*r4]
    lea     r2, [r2+2*r4]
    lea     r3, [r3+2*r4]
%endmacro

%macro SAD_X3_2x4P 1
%if %1
    SAD_X3_START_2x4P mm0, mm1, mm2
%else
    SAD_X3_START_2x4P mm4, mm5, mm6
    paddw     mm0,  mm4
    paddw     mm1,  mm5
    paddw     mm2,  mm6
%endif
    add     r0, 2*FENC_STRIDE
    lea     r1, [r1+2*r4]
    lea     r2, [r2+2*r4]
    lea     r3, [r3+2*r4]
%endmacro

%macro SAD_X4_START_1x8P 0
    movq    mm7,    [r0]
    movq    mm0,    [r1]
    movq    mm1,    [r2]
    movq    mm2,    [r3]
    movq    mm3,    [r4]
    psadbw  mm0,    mm7
    psadbw  mm1,    mm7
    psadbw  mm2,    mm7
    psadbw  mm3,    mm7
%endmacro

%macro SAD_X4_1x8P 2
    movq    mm7,    [r0+%1]
    movq    mm4,    [r1+%2]
    movq    mm5,    [r2+%2]
    movq    mm6,    [r3+%2]
    psadbw  mm4,    mm7
    psadbw  mm5,    mm7
    psadbw  mm6,    mm7
    psadbw  mm7,    [r4+%2]
    paddw   mm0,    mm4
    paddw   mm1,    mm5
    paddw   mm2,    mm6
    paddw   mm3,    mm7
%endmacro

%macro SAD_X4_START_2x4P 0
    movd      mm7,  [r0]
    movd      mm0,  [r1]
    movd      mm1,  [r2]
    movd      mm2,  [r3]
    movd      mm3,  [r4]
    punpckldq mm7,  [r0+FENC_STRIDE]
    punpckldq mm0,  [r1+r5]
    punpckldq mm1,  [r2+r5]
    punpckldq mm2,  [r3+r5]
    punpckldq mm3,  [r4+r5]
    psadbw    mm0,  mm7
    psadbw    mm1,  mm7
    psadbw    mm2,  mm7
    psadbw    mm3,  mm7
%endmacro

%macro SAD_X4_INC_2x4P 0
    movd      mm7,  [r0]
    movd      mm4,  [r1]
    movd      mm5,  [r2]
    punpckldq mm7,  [r0+FENC_STRIDE]
    punpckldq mm4,  [r1+r5]
    punpckldq mm5,  [r2+r5]
    psadbw    mm4,  mm7
    psadbw    mm5,  mm7
    paddw     mm0,  mm4
    paddw     mm1,  mm5
    movd      mm4,  [r3]
    movd      mm5,  [r4]
    punpckldq mm4,  [r3+r5]
    punpckldq mm5,  [r4+r5]
    psadbw    mm4,  mm7
    psadbw    mm5,  mm7
    paddw     mm2,  mm4
    paddw     mm3,  mm5
%endmacro

%macro SAD_X4_2x16P 1
%if %1
    SAD_X4_START_1x8P
%else
    SAD_X4_1x8P 0, 0
%endif
    SAD_X4_1x8P 8, 8
    SAD_X4_1x8P FENC_STRIDE, r5
    SAD_X4_1x8P FENC_STRIDE+8, r5+8
    add     r0, 2*FENC_STRIDE
    lea     r1, [r1+2*r5]
    lea     r2, [r2+2*r5]
    lea     r3, [r3+2*r5]
    lea     r4, [r4+2*r5]
%endmacro

%macro SAD_X4_2x8P 1
%if %1
    SAD_X4_START_1x8P
%else
    SAD_X4_1x8P 0, 0
%endif
    SAD_X4_1x8P FENC_STRIDE, r5
    add     r0, 2*FENC_STRIDE
    lea     r1, [r1+2*r5]
    lea     r2, [r2+2*r5]
    lea     r3, [r3+2*r5]
    lea     r4, [r4+2*r5]
%endmacro

%macro SAD_X4_2x4P 1
%if %1
    SAD_X4_START_2x4P
%else
    SAD_X4_INC_2x4P
%endif
    add     r0, 2*FENC_STRIDE
    lea     r1, [r1+2*r5]
    lea     r2, [r2+2*r5]
    lea     r3, [r3+2*r5]
    lea     r4, [r4+2*r5]
%endmacro

%macro SAD_X3_END 0
%ifdef ARCH_X86_64
    movd    [r5+0], mm0
    movd    [r5+4], mm1
    movd    [r5+8], mm2
%else
    mov     r0, r5m
    movd    [r0+0], mm0
    movd    [r0+4], mm1
    movd    [r0+8], mm2
%endif
    RET
%endmacro

%macro SAD_X4_END 0
    mov     r0, r6m
    movd    [r0+0], mm0
    movd    [r0+4], mm1
    movd    [r0+8], mm2
    movd    [r0+12], mm3
    RET
%endmacro

;-----------------------------------------------------------------------------
; void x264_pixel_sad_x3_16x16_mmxext( uint8_t *fenc, uint8_t *pix0, uint8_t *pix1,
;                                      uint8_t *pix2, int i_stride, int scores[3] )
;-----------------------------------------------------------------------------
%macro SAD_X 3
cglobal x264_pixel_sad_x%1_%2x%3_mmxext, %1+2, %1+2
    SAD_X%1_2x%2P 1
%rep %3/2-1
    SAD_X%1_2x%2P 0
%endrep
    SAD_X%1_END
%endmacro

SAD_X 3, 16, 16
SAD_X 3, 16,  8
SAD_X 3,  8, 16
SAD_X 3,  8,  8
SAD_X 3,  8,  4
SAD_X 3,  4,  8
SAD_X 3,  4,  4
SAD_X 4, 16, 16
SAD_X 4, 16,  8
SAD_X 4,  8, 16
SAD_X 4,  8,  8
SAD_X 4,  8,  4
SAD_X 4,  4,  8
SAD_X 4,  4,  4


;=============================================================================
; SAD x3/x4 XMM
;=============================================================================

%macro SAD_X3_START_1x16P_SSE2 0
    movdqa xmm3, [r0]
    movdqu xmm0, [r1]
    movdqu xmm1, [r2]
    movdqu xmm2, [r3]
    psadbw xmm0, xmm3
    psadbw xmm1, xmm3
    psadbw xmm2, xmm3
%endmacro

%macro SAD_X3_1x16P_SSE2 2
    movdqa xmm3, [r0+%1]
    movdqu xmm4, [r1+%2]
    movdqu xmm5, [r2+%2]
    movdqu xmm6, [r3+%2]
    psadbw xmm4, xmm3
    psadbw xmm5, xmm3
    psadbw xmm6, xmm3
    paddw  xmm0, xmm4
    paddw  xmm1, xmm5
    paddw  xmm2, xmm6
%endmacro

%macro SAD_X3_2x16P_SSE2 1
%if %1
    SAD_X3_START_1x16P_SSE2
%else
    SAD_X3_1x16P_SSE2 0, 0
%endif
    SAD_X3_1x16P_SSE2 FENC_STRIDE, r4
    add  r0, 2*FENC_STRIDE
    lea  r1, [r1+2*r4]
    lea  r2, [r2+2*r4]
    lea  r3, [r3+2*r4]
%endmacro

%macro SAD_X3_START_2x8P_SSE2 0
    movq    xmm7, [r0]
    movq    xmm0, [r1]
    movq    xmm1, [r2]
    movq    xmm2, [r3]
    movhps  xmm7, [r0+FENC_STRIDE]
    movhps  xmm0, [r1+r4]
    movhps  xmm1, [r2+r4]
    movhps  xmm2, [r3+r4]
    psadbw  xmm0, xmm7
    psadbw  xmm1, xmm7
    psadbw  xmm2, xmm7
%endmacro

%macro SAD_X3_2x8P_SSE2 0
    movq    xmm7, [r0]
    movq    xmm3, [r1]
    movq    xmm4, [r2]
    movq    xmm5, [r3]
    movhps  xmm7, [r0+FENC_STRIDE]
    movhps  xmm3, [r1+r4]
    movhps  xmm4, [r2+r4]
    movhps  xmm5, [r3+r4]
    psadbw  xmm3, xmm7
    psadbw  xmm4, xmm7
    psadbw  xmm5, xmm7
    paddw   xmm0, xmm3
    paddw   xmm1, xmm4
    paddw   xmm2, xmm5
%endmacro

%macro SAD_X4_START_2x8P_SSE2 0
    movq    xmm7, [r0]
    movq    xmm0, [r1]
    movq    xmm1, [r2]
    movq    xmm2, [r3]
    movq    xmm3, [r4]
    movhps  xmm7, [r0+FENC_STRIDE]
    movhps  xmm0, [r1+r5]
    movhps  xmm1, [r2+r5]
    movhps  xmm2, [r3+r5]
    movhps  xmm3, [r4+r5]
    psadbw  xmm0, xmm7
    psadbw  xmm1, xmm7
    psadbw  xmm2, xmm7
    psadbw  xmm3, xmm7
%endmacro

%macro SAD_X4_2x8P_SSE2 0
    movq    xmm7, [r0]
    movq    xmm4, [r1]
    movq    xmm5, [r2]
%ifdef ARCH_X86_64
    movq    xmm6, [r3]
    movq    xmm8, [r4]
    movhps  xmm7, [r0+FENC_STRIDE]
    movhps  xmm4, [r1+r5]
    movhps  xmm5, [r2+r5]
    movhps  xmm6, [r3+r5]
    movhps  xmm8, [r4+r5]
    psadbw  xmm4, xmm7
    psadbw  xmm5, xmm7
    psadbw  xmm6, xmm7
    psadbw  xmm8, xmm7
    paddw   xmm0, xmm4
    paddw   xmm1, xmm5
    paddw   xmm2, xmm6
    paddw   xmm3, xmm8
%else
    movhps  xmm7, [r0+FENC_STRIDE]
    movhps  xmm4, [r1+r5]
    movhps  xmm5, [r2+r5]
    psadbw  xmm4, xmm7
    psadbw  xmm5, xmm7
    paddw   xmm0, xmm4
    paddw   xmm1, xmm5
    movq    xmm6, [r3]
    movq    xmm4, [r4]
    movhps  xmm6, [r3+r5]
    movhps  xmm4, [r4+r5]
    psadbw  xmm6, xmm7
    psadbw  xmm4, xmm7
    paddw   xmm2, xmm6
    paddw   xmm3, xmm4
%endif
%endmacro

%macro SAD_X4_START_1x16P_SSE2 0
    movdqa xmm7, [r0]
    movdqu xmm0, [r1]
    movdqu xmm1, [r2]
    movdqu xmm2, [r3]
    movdqu xmm3, [r4]
    psadbw xmm0, xmm7
    psadbw xmm1, xmm7
    psadbw xmm2, xmm7
    psadbw xmm3, xmm7
%endmacro

%macro SAD_X4_1x16P_SSE2 2
    movdqa xmm7, [r0+%1]
    movdqu xmm4, [r1+%2]
    movdqu xmm5, [r2+%2]
    movdqu xmm6, [r3+%2]
%ifdef ARCH_X86_64
    movdqu xmm8, [r4+%2]
    psadbw xmm4, xmm7
    psadbw xmm5, xmm7
    psadbw xmm6, xmm7
    psadbw xmm8, xmm7
    paddw  xmm0, xmm4
    paddw  xmm1, xmm5
    paddw  xmm2, xmm6
    paddw  xmm3, xmm8
%else
    psadbw xmm4, xmm7
    psadbw xmm5, xmm7
    paddw  xmm0, xmm4
    psadbw xmm6, xmm7
    movdqu xmm4, [r4+%2]
    paddw  xmm1, xmm5
    psadbw xmm4, xmm7
    paddw  xmm2, xmm6
    paddw  xmm3, xmm4
%endif
%endmacro

%macro SAD_X4_2x16P_SSE2 1
%if %1
    SAD_X4_START_1x16P_SSE2
%else
    SAD_X4_1x16P_SSE2 0, 0
%endif
    SAD_X4_1x16P_SSE2 FENC_STRIDE, r5
    add  r0, 2*FENC_STRIDE
    lea  r1, [r1+2*r5]
    lea  r2, [r2+2*r5]
    lea  r3, [r3+2*r5]
    lea  r4, [r4+2*r5]
%endmacro

%macro SAD_X3_2x8P_SSE2 1
%if %1
    SAD_X3_START_2x8P_SSE2
%else
    SAD_X3_2x8P_SSE2
%endif
    add  r0, 2*FENC_STRIDE
    lea  r1, [r1+2*r4]
    lea  r2, [r2+2*r4]
    lea  r3, [r3+2*r4]
%endmacro

%macro SAD_X4_2x8P_SSE2 1
%if %1
    SAD_X4_START_2x8P_SSE2
%else
    SAD_X4_2x8P_SSE2
%endif
    add  r0, 2*FENC_STRIDE
    lea  r1, [r1+2*r5]
    lea  r2, [r2+2*r5]
    lea  r3, [r3+2*r5]
    lea  r4, [r4+2*r5]
%endmacro

%macro SAD_X3_END_SSE2 0
    movhlps xmm4, xmm0
    movhlps xmm5, xmm1
    movhlps xmm6, xmm2
    paddw   xmm0, xmm4
    paddw   xmm1, xmm5
    paddw   xmm2, xmm6
%ifdef ARCH_X86_64
    movd [r5+0], xmm0
    movd [r5+4], xmm1
    movd [r5+8], xmm2
%else
    mov      r0, r5m
    movd [r0+0], xmm0
    movd [r0+4], xmm1
    movd [r0+8], xmm2
%endif
    RET
%endmacro

%macro SAD_X4_END_SSE2 0
    mov       r0, r6m
    psllq   xmm1, 32
    psllq   xmm3, 32
    paddw   xmm0, xmm1
    paddw   xmm2, xmm3
    movhlps xmm1, xmm0
    movhlps xmm3, xmm2
    paddw   xmm0, xmm1
    paddw   xmm2, xmm3
    movq  [r0+0], xmm0
    movq  [r0+8], xmm2
    RET
%endmacro

%macro SAD_X3_START_1x16P_SSE2_MISALIGN 0
    movdqa xmm2, [r0]
    movdqu xmm0, [r1]
    movdqu xmm1, [r2]
    psadbw xmm0, xmm2
    psadbw xmm1, xmm2
    psadbw xmm2, [r3]
%endmacro

%macro SAD_X3_1x16P_SSE2_MISALIGN 2
    movdqa xmm3, [r0+%1]
    movdqu xmm4, [r1+%2]
    movdqu xmm5, [r2+%2]
    psadbw xmm4, xmm3
    psadbw xmm5, xmm3
    psadbw xmm3, [r3+%2]
    paddw  xmm0, xmm4
    paddw  xmm1, xmm5
    paddw  xmm2, xmm3
%endmacro

%macro SAD_X4_START_1x16P_SSE2_MISALIGN 0
    movdqa xmm3, [r0]
    movdqu xmm0, [r1]
    movdqu xmm1, [r2]
    movdqu xmm2, [r3]
    psadbw xmm0, xmm3
    psadbw xmm1, xmm3
    psadbw xmm2, xmm3
    psadbw xmm3, [r4]
%endmacro

%macro SAD_X4_1x16P_SSE2_MISALIGN 2
    movdqa xmm7, [r0+%1]
    movdqu xmm4, [r1+%2]
    movdqu xmm5, [r2+%2]
    movdqu xmm6, [r3+%2]
    psadbw xmm4, xmm7
    psadbw xmm5, xmm7
    psadbw xmm6, xmm7
    psadbw xmm7, [r4+%2]
    paddw  xmm0, xmm4
    paddw  xmm1, xmm5
    paddw  xmm2, xmm6
    paddw  xmm3, xmm7
%endmacro

%macro SAD_X3_2x16P_SSE2_MISALIGN 1
%if %1
    SAD_X3_START_1x16P_SSE2_MISALIGN
%else
    SAD_X3_1x16P_SSE2_MISALIGN 0, 0
%endif
    SAD_X3_1x16P_SSE2_MISALIGN FENC_STRIDE, r4
    add  r0, 2*FENC_STRIDE
    lea  r1, [r1+2*r4]
    lea  r2, [r2+2*r4]
    lea  r3, [r3+2*r4]
%endmacro

%macro SAD_X4_2x16P_SSE2_MISALIGN 1
%if %1
    SAD_X4_START_1x16P_SSE2_MISALIGN
%else
    SAD_X4_1x16P_SSE2_MISALIGN 0, 0
%endif
    SAD_X4_1x16P_SSE2_MISALIGN FENC_STRIDE, r5
    add  r0, 2*FENC_STRIDE
    lea  r1, [r1+2*r5]
    lea  r2, [r2+2*r5]
    lea  r3, [r3+2*r5]
    lea  r4, [r4+2*r5]
%endmacro

;-----------------------------------------------------------------------------
; void x264_pixel_sad_x3_16x16_sse2( uint8_t *fenc, uint8_t *pix0, uint8_t *pix1,
;                                    uint8_t *pix2, int i_stride, int scores[3] )
;-----------------------------------------------------------------------------
%macro SAD_X_SSE2 4
cglobal x264_pixel_sad_x%1_%2x%3_%4, 2+%1,2+%1
    SAD_X%1_2x%2P_SSE2 1
%rep %3/2-1
    SAD_X%1_2x%2P_SSE2 0
%endrep
    SAD_X%1_END_SSE2
%endmacro

%macro SAD_X_SSE2_MISALIGN 4
cglobal x264_pixel_sad_x%1_%2x%3_%4_misalign, 2+%1,2+%1
    SAD_X%1_2x%2P_SSE2_MISALIGN 1
%rep %3/2-1
    SAD_X%1_2x%2P_SSE2_MISALIGN 0
%endrep
    SAD_X%1_END_SSE2
%endmacro

SAD_X_SSE2 3, 16, 16, sse2
SAD_X_SSE2 3, 16,  8, sse2
SAD_X_SSE2 3,  8, 16, sse2
SAD_X_SSE2 3,  8,  8, sse2
SAD_X_SSE2 3,  8,  4, sse2
SAD_X_SSE2 4, 16, 16, sse2
SAD_X_SSE2 4, 16,  8, sse2
SAD_X_SSE2 4,  8, 16, sse2
SAD_X_SSE2 4,  8,  8, sse2
SAD_X_SSE2 4,  8,  4, sse2

SAD_X_SSE2_MISALIGN 3, 16, 16, sse2
SAD_X_SSE2_MISALIGN 3, 16,  8, sse2
SAD_X_SSE2_MISALIGN 4, 16, 16, sse2
SAD_X_SSE2_MISALIGN 4, 16,  8, sse2

%define movdqu lddqu
SAD_X_SSE2 3, 16, 16, sse3
SAD_X_SSE2 3, 16,  8, sse3
SAD_X_SSE2 4, 16, 16, sse3
SAD_X_SSE2 4, 16,  8, sse3
%undef movdqu


;=============================================================================
; SAD cacheline split
;=============================================================================

; Core2 (Conroe) can load unaligned data just as quickly as aligned data...
; unless the unaligned data spans the border between 2 cachelines, in which
; case it's really slow. The exact numbers may differ, but all Intel cpus prior
; to Nehalem have a large penalty for cacheline splits.
; (8-byte alignment exactly half way between two cachelines is ok though.)
; LDDQU was supposed to fix this, but it only works on Pentium 4.
; So in the split case we load aligned data and explicitly perform the
; alignment between registers. Like on archs that have only aligned loads,
; except complicated by the fact that PALIGNR takes only an immediate, not
; a variable alignment.
; It is also possible to hoist the realignment to the macroblock level (keep
; 2 copies of the reference frame, offset by 32 bytes), but the extra memory
; needed for that method makes it often slower.

; sad 16x16 costs on Core2:
; good offsets: 49 cycles (50/64 of all mvs)
; cacheline split: 234 cycles (14/64 of all mvs. ammortized: +40 cycles)
; page split: 3600 cycles (14/4096 of all mvs. ammortized: +11.5 cycles)
; cache or page split with palignr: 57 cycles (ammortized: +2 cycles)

; computed jump assumes this loop is exactly 80 bytes
%macro SAD16_CACHELINE_LOOP_SSE2 1 ; alignment
ALIGN 16
sad_w16_align%1_sse2:
    movdqa  xmm1, [r2+16]
    movdqa  xmm2, [r2+r3+16]
    movdqa  xmm3, [r2]
    movdqa  xmm4, [r2+r3]
    pslldq  xmm1, 16-%1
    pslldq  xmm2, 16-%1
    psrldq  xmm3, %1
    psrldq  xmm4, %1
    por     xmm1, xmm3
    por     xmm2, xmm4
    psadbw  xmm1, [r0]
    psadbw  xmm2, [r0+r1]
    paddw   xmm0, xmm1
    paddw   xmm0, xmm2
    lea     r0,   [r0+2*r1]
    lea     r2,   [r2+2*r3]
    dec     r4
    jg sad_w16_align%1_sse2
    ret
%endmacro

; computed jump assumes this loop is exactly 64 bytes
%macro SAD16_CACHELINE_LOOP_SSSE3 1 ; alignment
ALIGN 16
sad_w16_align%1_ssse3:
    movdqa  xmm1, [r2+16]
    movdqa  xmm2, [r2+r3+16]
    palignr xmm1, [r2], %1
    palignr xmm2, [r2+r3], %1
    psadbw  xmm1, [r0]
    psadbw  xmm2, [r0+r1]
    paddw   xmm0, xmm1
    paddw   xmm0, xmm2
    lea     r0,   [r0+2*r1]
    lea     r2,   [r2+2*r3]
    dec     r4
    jg sad_w16_align%1_ssse3
    ret
%endmacro

%macro SAD16_CACHELINE_FUNC 2 ; cpu, height
cglobal x264_pixel_sad_16x%2_cache64_%1, 0,0
    mov     eax, r2m
    and     eax, 0x37
    cmp     eax, 0x30
    jle x264_pixel_sad_16x%2_sse2
    PROLOGUE 4,6
    mov     r4d, r2d
    and     r4d, 15
%ifidn %1, ssse3
    shl     r4d, 6  ; code size = 64
%else
    lea     r4, [r4*5]
    shl     r4d, 4  ; code size = 80
%endif
%define sad_w16_addr (sad_w16_align1_%1 + (sad_w16_align1_%1 - sad_w16_align2_%1))
%ifdef PIC
    lea     r5, [sad_w16_addr GLOBAL]
    add     r5, r4
%else
    lea     r5, [sad_w16_addr + r4 GLOBAL]
%endif
    and     r2, ~15
    mov     r4d, %2/2
    pxor    xmm0, xmm0
    call    r5
    movhlps xmm1, xmm0
    paddw   xmm0, xmm1
    movd    eax,  xmm0
    RET
%endmacro

%macro SAD_CACHELINE_START_MMX2 4 ; width, height, iterations, cacheline
    mov    eax, r2m
    and    eax, 0x17|%1|(%4>>1)
    cmp    eax, 0x10|%1|(%4>>1)
    jle x264_pixel_sad_%1x%2_mmxext
    and    eax, 7
    shl    eax, 3
    movd   mm6, [sw_64 GLOBAL]
    movd   mm7, eax
    psubw  mm6, mm7
    PROLOGUE 4,5
    and    r2, ~7
    mov    r4d, %3
    pxor   mm0, mm0
%endmacro

%macro SAD16_CACHELINE_FUNC_MMX2 2 ; height, cacheline
cglobal x264_pixel_sad_16x%1_cache%2_mmxext, 0,0
    SAD_CACHELINE_START_MMX2 16, %1, %1, %2
.loop:
    movq   mm1, [r2]
    movq   mm2, [r2+8]
    movq   mm3, [r2+16]
    movq   mm4, mm2
    psrlq  mm1, mm7
    psllq  mm2, mm6
    psllq  mm3, mm6
    psrlq  mm4, mm7
    por    mm1, mm2
    por    mm3, mm4
    psadbw mm1, [r0]
    psadbw mm3, [r0+8]
    paddw  mm0, mm1
    paddw  mm0, mm3
    add    r2, r3
    add    r0, r1
    dec    r4
    jg .loop
    movd   eax, mm0
    RET
%endmacro

%macro SAD8_CACHELINE_FUNC_MMX2 2 ; height, cacheline
cglobal x264_pixel_sad_8x%1_cache%2_mmxext, 0,0
    SAD_CACHELINE_START_MMX2 8, %1, %1/2, %2
.loop:
    movq   mm1, [r2+8]
    movq   mm2, [r2+r3+8]
    movq   mm3, [r2]
    movq   mm4, [r2+r3]
    psllq  mm1, mm6
    psllq  mm2, mm6
    psrlq  mm3, mm7
    psrlq  mm4, mm7
    por    mm1, mm3
    por    mm2, mm4
    psadbw mm1, [r0]
    psadbw mm2, [r0+r1]
    paddw  mm0, mm1
    paddw  mm0, mm2
    lea    r2, [r2+2*r3]
    lea    r0, [r0+2*r1]
    dec    r4
    jg .loop
    movd   eax, mm0
    RET
%endmacro

; sad_x3/x4_cache64: check each mv.
; if they're all within a cacheline, use normal sad_x3/x4.
; otherwise, send them individually to sad_cache64.
%macro CHECK_SPLIT 3 ; pix, width, cacheline
    mov  eax, %1
    and  eax, 0x17|%2|(%3>>1)
    cmp  eax, 0x10|%2|(%3>>1)
    jg .split
%endmacro

%macro SADX3_CACHELINE_FUNC 5 ; width, height, cacheline, normal_ver, split_ver
cglobal x264_pixel_sad_x3_%1x%2_cache%3_%5, 0,0
    CHECK_SPLIT r1m, %1, %3
    CHECK_SPLIT r2m, %1, %3
    CHECK_SPLIT r3m, %1, %3
    jmp x264_pixel_sad_x3_%1x%2_%4
.split:
%ifdef ARCH_X86_64
    push r3
    push r2
    mov  r2, r1
    mov  r1, FENC_STRIDE
    mov  r3, r4
    mov  r10, r0
    mov  r11, r5
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  [r11], eax
    pop  r2
    mov  r0, r10
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  [r11+4], eax
    pop  r2
    mov  r0, r10
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  [r11+8], eax
%else
    push edi
    mov  edi, [esp+28]
    push dword [esp+24]
    push dword [esp+16]
    push dword 16
    push dword [esp+20]
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  ecx, [esp+32]
    mov  [edi], eax
    mov  [esp+8], ecx
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  ecx, [esp+36]
    mov  [edi+4], eax
    mov  [esp+8], ecx
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  [edi+8], eax
    add  esp, 16
    pop  edi
%endif
    ret
%endmacro

%macro SADX4_CACHELINE_FUNC 5 ; width, height, cacheline, normal_ver, split_ver
cglobal x264_pixel_sad_x4_%1x%2_cache%3_%5, 0,0
    CHECK_SPLIT r1m, %1, %3
    CHECK_SPLIT r2m, %1, %3
    CHECK_SPLIT r3m, %1, %3
    CHECK_SPLIT r4m, %1, %3
    jmp x264_pixel_sad_x4_%1x%2_%4
.split:
%ifdef ARCH_X86_64
    mov  r11, r6m
    push r4
    push r3
    push r2
    mov  r2, r1
    mov  r1, FENC_STRIDE
    mov  r3, r5
    mov  r10, r0
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  [r11], eax
    pop  r2
    mov  r0, r10
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  [r11+4], eax
    pop  r2
    mov  r0, r10
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  [r11+8], eax
    pop  r2
    mov  r0, r10
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  [r11+12], eax
%else
    push edi
    mov  edi, [esp+32]
    push dword [esp+28]
    push dword [esp+16]
    push dword 16
    push dword [esp+20]
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  ecx, [esp+32]
    mov  [edi], eax
    mov  [esp+8], ecx
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  ecx, [esp+36]
    mov  [edi+4], eax
    mov  [esp+8], ecx
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  ecx, [esp+40]
    mov  [edi+8], eax
    mov  [esp+8], ecx
    call x264_pixel_sad_%1x%2_cache%3_%5
    mov  [edi+12], eax
    add  esp, 16
    pop  edi
%endif
    ret
%endmacro

%macro SADX34_CACHELINE_FUNC 5
    SADX3_CACHELINE_FUNC %1, %2, %3, %4, %5
    SADX4_CACHELINE_FUNC %1, %2, %3, %4, %5
%endmacro


; instantiate the aligned sads

%ifndef ARCH_X86_64
SAD16_CACHELINE_FUNC_MMX2  8, 32
SAD16_CACHELINE_FUNC_MMX2 16, 32
SAD8_CACHELINE_FUNC_MMX2   4, 32
SAD8_CACHELINE_FUNC_MMX2   8, 32
SAD8_CACHELINE_FUNC_MMX2  16, 32
SAD16_CACHELINE_FUNC_MMX2  8, 64
SAD16_CACHELINE_FUNC_MMX2 16, 64
%endif ; !ARCH_X86_64
SAD8_CACHELINE_FUNC_MMX2   4, 64
SAD8_CACHELINE_FUNC_MMX2   8, 64
SAD8_CACHELINE_FUNC_MMX2  16, 64

%ifndef ARCH_X86_64
SADX34_CACHELINE_FUNC 16, 16, 32, mmxext, mmxext
SADX34_CACHELINE_FUNC 16,  8, 32, mmxext, mmxext
SADX34_CACHELINE_FUNC  8, 16, 32, mmxext, mmxext
SADX34_CACHELINE_FUNC  8,  8, 32, mmxext, mmxext
SADX34_CACHELINE_FUNC 16, 16, 64, mmxext, mmxext
SADX34_CACHELINE_FUNC 16,  8, 64, mmxext, mmxext
%endif ; !ARCH_X86_64
SADX34_CACHELINE_FUNC  8, 16, 64, mmxext, mmxext
SADX34_CACHELINE_FUNC  8,  8, 64, mmxext, mmxext

%ifndef ARCH_X86_64
SAD16_CACHELINE_FUNC sse2, 8
SAD16_CACHELINE_FUNC sse2, 16
%assign i 1
%rep 15
SAD16_CACHELINE_LOOP_SSE2 i
%assign i i+1
%endrep
SADX34_CACHELINE_FUNC 16, 16, 64, sse2, sse2
SADX34_CACHELINE_FUNC 16,  8, 64, sse2, sse2
%endif ; !ARCH_X86_64

SAD16_CACHELINE_FUNC ssse3, 8
SAD16_CACHELINE_FUNC ssse3, 16
%assign i 1
%rep 15
SAD16_CACHELINE_LOOP_SSSE3 i
%assign i i+1
%endrep
SADX34_CACHELINE_FUNC 16, 16, 64, sse2, ssse3
SADX34_CACHELINE_FUNC 16,  8, 64, sse2, ssse3