;*****************************************************************************
;* pixel.asm: h264 encoder library
;*****************************************************************************
;* Copyright (C) 2003-2008 x264 project
;*
;* Authors: Loren Merritt <lorenm@u.washington.edu>
;*          Laurent Aimar <fenrir@via.ecp.fr>
;*          Alex Izvorski <aizvorksi@gmail.com>
;*          Jason Garrett-Glaser <darkshikari@gmail.com>
;*
;* This program is free software; you can redistribute it and/or modify
;* it under the terms of the GNU General Public License as published by
;* the Free Software Foundation; either version 2 of the License, or
;* (at your option) any later version.
;*
;* This program is distributed in the hope that it will be useful,
;* but WITHOUT ANY WARRANTY; without even the implied warranty of
;* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;* GNU General Public License for more details.
;*
;* You should have received a copy of the GNU General Public License
;* along with this program; if not, write to the Free Software
;* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02111, USA.
;*****************************************************************************

%include "x86inc.asm"
%include "x86util.asm"

SECTION_RODATA
pw_1:    times 8 dw 1
ssim_c1: times 4 dd 416    ; .01*.01*255*255*64
ssim_c2: times 4 dd 235963 ; .03*.03*255*255*64*63
mask_ff: times 16 db 0xff
         times 16 db 0
mask_ac4: dw 0,-1,-1,-1, 0,-1,-1,-1
mask_ac8: dw 0,-1,-1,-1,-1,-1,-1,-1

SECTION .text

%macro HADDD 2 ; sum junk
%if mmsize == 16
    movhlps %2, %1
    paddd   %1, %2
    pshuflw %2, %1, 0xE
    paddd   %1, %2
%else
    mova    %2, %1
    psrlq   %2, 32
    paddd   %1, %2
%endif
%endmacro

%macro HADDW 2
    pmaddwd %1, [pw_1 GLOBAL]
    HADDD   %1, %2
%endmacro

%macro HADDUW 2
    mova  %2, %1
    pslld %1, 16
    psrld %2, 16
    psrld %1, 16
    paddd %1, %2
    HADDD %1, %2
%endmacro

;=============================================================================
; SSD
;=============================================================================

%macro SSD_FULL 6
    mova      m1, [r0+%1]
    mova      m2, [r2+%2]
    mova      m3, [r0+%3]
    mova      m4, [r2+%4]

    mova      m5, m2
    mova      m6, m4
    psubusb   m2, m1
    psubusb   m4, m3
    psubusb   m1, m5
    psubusb   m3, m6
    por       m1, m2
    por       m3, m4

    mova      m2, m1
    mova      m4, m3
    punpcklbw m1, m7
    punpcklbw m3, m7
    punpckhbw m2, m7
    punpckhbw m4, m7
    pmaddwd   m1, m1
    pmaddwd   m2, m2
    pmaddwd   m3, m3
    pmaddwd   m4, m4

%if %6
    lea       r0, [r0+2*r1]
    lea       r2, [r2+2*r3]
%endif
    paddd     m1, m2
    paddd     m3, m4
%if %5
    paddd     m0, m1
%else
    SWAP      m0, m1
%endif
    paddd     m0, m3
%endmacro

%macro SSD_HALF 6
    movh      m1, [r0+%1]
    movh      m2, [r2+%2]
    movh      m3, [r0+%3]
    movh      m4, [r2+%4]

    punpcklbw m1, m7
    punpcklbw m2, m7
    punpcklbw m3, m7
    punpcklbw m4, m7
    psubw     m1, m2
    psubw     m3, m4
    pmaddwd   m1, m1
    pmaddwd   m3, m3

%if %6
    lea       r0, [r0+2*r1]
    lea       r2, [r2+2*r3]
%endif
%if %5
    paddd     m0, m1
%else
    SWAP      m0, m1
%endif
    paddd     m0, m3
%endmacro

%macro SSD_QUARTER 6
    movd      m1, [r0+%1]
    movd      m2, [r2+%2]
    movd      m3, [r0+%3]
    movd      m4, [r2+%4]
    lea       r0, [r0+2*r1]
    lea       r2, [r2+2*r3]
    pinsrd    m1, [r0+%1], 1
    pinsrd    m2, [r2+%2], 1
    pinsrd    m3, [r0+%3], 1
    pinsrd    m4, [r2+%4], 1
    punpcklbw m1, m7
    punpcklbw m2, m7
    punpcklbw m3, m7
    punpcklbw m4, m7
    psubw     m1, m2
    psubw     m3, m4
    pmaddwd   m1, m1
    pmaddwd   m3, m3

%if %6
    lea       r0, [r0+2*r1]
    lea       r2, [r2+2*r3]
%endif
%if %5
    paddd     m0, m1
%else
    SWAP      m0, m1
%endif
    paddd     m0, m3
%endmacro

;-----------------------------------------------------------------------------
; int x264_pixel_ssd_16x16_mmx( uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
%macro SSD 3
cglobal x264_pixel_ssd_%1x%2_%3, 4,4
%if %1 >= mmsize
    pxor    m7, m7
%endif
%assign i 0
%rep %2/2
%if %1 > mmsize
    SSD_FULL 0,  0,     mmsize,    mmsize, i, 0
    SSD_FULL r1, r3, r1+mmsize, r3+mmsize, 1, i<%2/2-1
%elif %1 == mmsize
    SSD_FULL 0, 0, r1, r3, i, i<%2/2-1
%else
    SSD_HALF 0, 0, r1, r3, i, i<%2/2-1
%endif
%assign i i+1
%endrep
    HADDD   m0, m1
    movd   eax, m0
    RET
%endmacro

INIT_MMX
SSD 16, 16, mmx
SSD 16,  8, mmx
SSD  8, 16, mmx
SSD  8,  8, mmx
SSD  8,  4, mmx
SSD  4,  8, mmx
SSD  4,  4, mmx
INIT_XMM
SSD 16, 16, sse2
SSD 16,  8, sse2
SSD  8, 16, sse2
SSD  8,  8, sse2
SSD  8,  4, sse2

cglobal x264_pixel_ssd_4x8_sse4, 4,4
    SSD_QUARTER 0, 0, r1, r3, 0, 1
    SSD_QUARTER 0, 0, r1, r3, 1, 0
    HADDD   m0, m1
    movd   eax, m0
    RET

cglobal x264_pixel_ssd_4x4_sse4, 4,4
    SSD_QUARTER 0, 0, r1, r3, 0, 0
    HADDD   m0, m1
    movd   eax, m0
    RET


;=============================================================================
; variance
;=============================================================================

%macro VAR_START 0
    pxor  m5, m5    ; sum
    pxor  m6, m6    ; sum squared
    pxor  m7, m7    ; zero
%ifdef ARCH_X86_64
    %define t3d r3d
%else
    %define t3d r2d
%endif
%endmacro

%macro VAR_END 1
%if mmsize == 16
    movhlps m0, m5
    paddw   m5, m0
%endif
    movifnidn r2d, r2m
    movd   r1d, m5
    movd  [r2], m5  ; return sum
    imul   r1d, r1d
    HADDD   m6, m1
    shr    r1d, %1
    movd   eax, m6
    sub    eax, r1d  ; sqr - (sum * sum >> shift)
    RET
%endmacro

%macro VAR_2ROW 2
    mov      t3d, %2
.loop:
    mova      m0, [r0]
    mova      m1, m0
    mova      m3, [r0+%1]
    mova      m2, m0
    punpcklbw m0, m7
    mova      m4, m3
    punpckhbw m1, m7
%ifidn %1, r1
    lea       r0, [r0+%1*2]
%else
    add       r0, r1
%endif
    punpckhbw m4, m7
    psadbw    m2, m7
    paddw     m5, m2
    mova      m2, m3
    punpcklbw m3, m7
    dec t3d
    psadbw    m2, m7
    pmaddwd   m0, m0
    paddw     m5, m2
    pmaddwd   m1, m1
    paddd     m6, m0
    pmaddwd   m3, m3
    paddd     m6, m1
    pmaddwd   m4, m4
    paddd     m6, m3
    paddd     m6, m4
    jg .loop
%endmacro

;-----------------------------------------------------------------------------
; int x264_pixel_var_wxh_mmxext( uint8_t *, int, int * )
;-----------------------------------------------------------------------------
INIT_MMX
cglobal x264_pixel_var_16x16_mmxext, 2,3
    VAR_START
    VAR_2ROW 8, 16
    VAR_END 8

cglobal x264_pixel_var_8x8_mmxext, 2,3
    VAR_START
    VAR_2ROW r1, 4
    VAR_END 6

INIT_XMM
cglobal x264_pixel_var_16x16_sse2, 2,3
    VAR_START
    VAR_2ROW r1, 8
    VAR_END 8

cglobal x264_pixel_var_8x8_sse2, 2,3
    VAR_START
    mov t3d, 4
.loop:
    movh      m0, [r0]
    movhps    m0, [r0+r1]
    lea       r0, [r0+r1*2]
    mova      m1, m0
    punpcklbw m0, m7
    mova      m2, m1
    punpckhbw m1, m7
    dec t3d
    pmaddwd   m0, m0
    pmaddwd   m1, m1
    psadbw    m2, m7
    paddw     m5, m2
    paddd     m6, m0
    paddd     m6, m1
    jnz .loop
    VAR_END 6


;=============================================================================
; SATD
;=============================================================================

; phaddw is used only in 4x4 hadamard, because in 8x8 it's slower:
; even on Penryn, phaddw has latency 3 while paddw and punpck* have 1.
; 4x4 is special in that 4x4 transpose in xmmregs takes extra munging,
; whereas phaddw-based transform doesn't care what order the coefs end up in.

%macro PHSUMSUB 3
    movdqa m%3, m%1
    phaddw m%1, m%2
    phsubw m%3, m%2
    SWAP %2, %3
%endmacro

%macro HADAMARD4_ROW_PHADD 5
    PHSUMSUB %1, %2, %5
    PHSUMSUB %3, %4, %5
    PHSUMSUB %1, %3, %5
    PHSUMSUB %2, %4, %5
    SWAP %3, %4
%endmacro

%macro HADAMARD4_1D 4
    SUMSUB_BADC %1, %2, %3, %4
    SUMSUB_BADC %1, %3, %2, %4
%endmacro

%macro HADAMARD4x4_SUM 1    ; %1 = dest (row sum of one block)
    %xdefine %%n n%1
    HADAMARD4_1D  m4, m5, m6, m7
    TRANSPOSE4x4W  4,  5,  6,  7, %%n
    HADAMARD4_1D  m4, m5, m6, m7
    ABS2          m4, m5, m3, m %+ %%n
    ABS2          m6, m7, m3, m %+ %%n
    paddw         m6, m4
    paddw         m7, m5
    pavgw         m6, m7
    SWAP %%n, 6
%endmacro

; in: r4=3*stride1, r5=3*stride2
; in: %2 = horizontal offset
; in: %3 = whether we need to increment pix1 and pix2
; clobber: m3..m7
; out: %1 = satd
%macro SATD_4x4_MMX 3
    LOAD_DIFF m4, m3, none, [r0+%2],      [r2+%2]
    LOAD_DIFF m5, m3, none, [r0+r1+%2],   [r2+r3+%2]
    LOAD_DIFF m6, m3, none, [r0+2*r1+%2], [r2+2*r3+%2]
    LOAD_DIFF m7, m3, none, [r0+r4+%2],   [r2+r5+%2]
%if %3
    lea  r0, [r0+4*r1]
    lea  r2, [r2+4*r3]
%endif
    HADAMARD4x4_SUM %1
%endmacro

%macro SATD_8x4_SSE2 1
    HADAMARD4_1D    m0, m1, m2, m3
%ifidn %1, ssse3_phadd
    HADAMARD4_ROW_PHADD 0, 1, 2, 3, 4
%else
    TRANSPOSE2x4x4W  0,  1,  2,  3,  4
    HADAMARD4_1D    m0, m1, m2, m3
%endif
    ABS4            m0, m1, m2, m3, m4, m5
    paddusw  m0, m1
    paddusw  m2, m3
    paddusw  m6, m0
    paddusw  m6, m2
%endmacro

%macro SATD_START_MMX 0
    lea  r4, [3*r1] ; 3*stride1
    lea  r5, [3*r3] ; 3*stride2
%endmacro

%macro SATD_END_MMX 0
    pshufw      m1, m0, 01001110b
    paddw       m0, m1
    pshufw      m1, m0, 10110001b
    paddw       m0, m1
    movd       eax, m0
    and        eax, 0xffff
    RET
%endmacro

; FIXME avoid the spilling of regs to hold 3*stride.
; for small blocks on x86_32, modify pixel pointer instead.

;-----------------------------------------------------------------------------
; int x264_pixel_satd_16x16_mmxext (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
INIT_MMX
cglobal x264_pixel_satd_16x4_internal_mmxext
    SATD_4x4_MMX m2,  0, 0
    SATD_4x4_MMX m1,  4, 0
    paddw        m0, m2
    SATD_4x4_MMX m2,  8, 0
    paddw        m0, m1
    SATD_4x4_MMX m1, 12, 0
    paddw        m0, m2
    paddw        m0, m1
    ret

cglobal x264_pixel_satd_8x8_internal_mmxext
    SATD_4x4_MMX m2,  0, 0
    SATD_4x4_MMX m1,  4, 1
    paddw        m0, m2
    paddw        m0, m1
x264_pixel_satd_8x4_internal_mmxext:
    SATD_4x4_MMX m2,  0, 0
    SATD_4x4_MMX m1,  4, 0
    paddw        m0, m2
    paddw        m0, m1
    ret

cglobal x264_pixel_satd_16x16_mmxext, 4,6
    SATD_START_MMX
    pxor   m0, m0
%rep 3
    call x264_pixel_satd_16x4_internal_mmxext
    lea  r0, [r0+4*r1]
    lea  r2, [r2+4*r3]
%endrep
    call x264_pixel_satd_16x4_internal_mmxext
    HADDUW m0, m1
    movd  eax, m0
    RET

cglobal x264_pixel_satd_16x8_mmxext, 4,6
    SATD_START_MMX
    pxor   m0, m0
    call x264_pixel_satd_16x4_internal_mmxext
    lea  r0, [r0+4*r1]
    lea  r2, [r2+4*r3]
    call x264_pixel_satd_16x4_internal_mmxext
    SATD_END_MMX

cglobal x264_pixel_satd_8x16_mmxext, 4,6
    SATD_START_MMX
    pxor   m0, m0
    call x264_pixel_satd_8x8_internal_mmxext
    lea  r0, [r0+4*r1]
    lea  r2, [r2+4*r3]
    call x264_pixel_satd_8x8_internal_mmxext
    SATD_END_MMX

cglobal x264_pixel_satd_8x8_mmxext, 4,6
    SATD_START_MMX
    pxor   m0, m0
    call x264_pixel_satd_8x8_internal_mmxext
    SATD_END_MMX

cglobal x264_pixel_satd_8x4_mmxext, 4,6
    SATD_START_MMX
    pxor   m0, m0
    call x264_pixel_satd_8x4_internal_mmxext
    SATD_END_MMX

cglobal x264_pixel_satd_4x8_mmxext, 4,6
    SATD_START_MMX
    SATD_4x4_MMX m0, 0, 1
    SATD_4x4_MMX m1, 0, 0
    paddw  m0, m1
    SATD_END_MMX

%macro SATD_W4 1
INIT_MMX
cglobal x264_pixel_satd_4x4_%1, 4,6
    SATD_START_MMX
    SATD_4x4_MMX m0, 0, 0
    SATD_END_MMX
%endmacro

SATD_W4 mmxext

%macro SATD_START_SSE2 0
    pxor    m6, m6
    lea     r4, [3*r1]
    lea     r5, [3*r3]
%endmacro

%macro SATD_END_SSE2 0
    psrlw   m6, 1
    HADDW   m6, m7
    movd   eax, m6
    RET
%endmacro

%macro BACKUP_POINTERS 0
%ifdef ARCH_X86_64
    mov    r10, r0
    mov    r11, r2
%endif
%endmacro

%macro RESTORE_AND_INC_POINTERS 0
%ifdef ARCH_X86_64
    lea     r0, [r10+8]
    lea     r2, [r11+8]
%else
    mov     r0, r0m
    mov     r2, r2m
    add     r0, 8
    add     r2, 8
%endif
%endmacro

;-----------------------------------------------------------------------------
; int x264_pixel_satd_8x4_sse2 (uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
%macro SATDS_SSE2 1
INIT_XMM
cglobal x264_pixel_satd_8x8_internal_%1
    LOAD_DIFF_8x4P  m0, m1, m2, m3, m4, m5
    SATD_8x4_SSE2 %1
    lea  r0, [r0+4*r1]
    lea  r2, [r2+4*r3]
x264_pixel_satd_8x4_internal_%1:
    LOAD_DIFF_8x4P  m0, m1, m2, m3, m4, m5
x264_pixel_satd_4x8_internal_%1:
    SAVE_MM_PERMUTATION satd_4x8_internal
    SATD_8x4_SSE2 %1
    ret

cglobal x264_pixel_satd_16x16_%1, 4,6
    SATD_START_SSE2
    BACKUP_POINTERS
    call x264_pixel_satd_8x8_internal_%1
    lea  r0, [r0+4*r1]
    lea  r2, [r2+4*r3]
    call x264_pixel_satd_8x8_internal_%1
    RESTORE_AND_INC_POINTERS
    call x264_pixel_satd_8x8_internal_%1
    lea  r0, [r0+4*r1]
    lea  r2, [r2+4*r3]
    call x264_pixel_satd_8x8_internal_%1
    SATD_END_SSE2

cglobal x264_pixel_satd_16x8_%1, 4,6
    SATD_START_SSE2
    BACKUP_POINTERS
    call x264_pixel_satd_8x8_internal_%1
    RESTORE_AND_INC_POINTERS
    call x264_pixel_satd_8x8_internal_%1
    SATD_END_SSE2

cglobal x264_pixel_satd_8x16_%1, 4,6
    SATD_START_SSE2
    call x264_pixel_satd_8x8_internal_%1
    lea  r0, [r0+4*r1]
    lea  r2, [r2+4*r3]
    call x264_pixel_satd_8x8_internal_%1
    SATD_END_SSE2

cglobal x264_pixel_satd_8x8_%1, 4,6
    SATD_START_SSE2
    call x264_pixel_satd_8x8_internal_%1
    SATD_END_SSE2

cglobal x264_pixel_satd_8x4_%1, 4,6
    SATD_START_SSE2
    call x264_pixel_satd_8x4_internal_%1
    SATD_END_SSE2

cglobal x264_pixel_satd_4x8_%1, 4,6
    INIT_XMM
    LOAD_MM_PERMUTATION satd_4x8_internal
    %define movh movd
    SATD_START_SSE2
    LOAD_DIFF  m0, m7, m6, [r0],      [r2]
    LOAD_DIFF  m1, m7, m6, [r0+r1],   [r2+r3]
    LOAD_DIFF  m2, m7, m6, [r0+2*r1], [r2+2*r3]
    LOAD_DIFF  m3, m7, m6, [r0+r4],   [r2+r5]
    lea        r0, [r0+4*r1]
    lea        r2, [r2+4*r3]
    LOAD_DIFF  m4, m7, m6, [r0],      [r2]
    LOAD_DIFF  m5, m7, m6, [r0+r1],   [r2+r3]
    punpcklqdq m0, m4
    punpcklqdq m1, m5
    LOAD_DIFF  m4, m7, m6, [r0+2*r1], [r2+2*r3]
    LOAD_DIFF  m5, m7, m6, [r0+r4],   [r2+r5]
    punpcklqdq m2, m4
    punpcklqdq m3, m5
    %define movh movq
    call x264_pixel_satd_4x8_internal_%1
    SATD_END_SSE2

%ifdef ARCH_X86_64
;-----------------------------------------------------------------------------
; int x264_pixel_sa8d_8x8_sse2( uint8_t *, int, uint8_t *, int )
;-----------------------------------------------------------------------------
cglobal x264_pixel_sa8d_8x8_internal_%1
    lea  r10, [r0+4*r1]
    lea  r11, [r2+4*r3]
    LOAD_DIFF_8x4P m0, m1, m2, m3, m8, m9, r0, r2
    LOAD_DIFF_8x4P m4, m5, m6, m7, m8, m9, r10, r11

    HADAMARD8_1D  m0, m1, m2, m3, m4, m5, m6, m7
    TRANSPOSE8x8W  0,  1,  2,  3,  4,  5,  6,  7,  8
    HADAMARD8_1D  m0, m1, m2, m3, m4, m5, m6, m7

    ABS4 m0, m1, m2, m3, m8, m9
    ABS4 m4, m5, m6, m7, m8, m9
    paddusw  m0, m1
    paddusw  m2, m3
    paddusw  m4, m5
    paddusw  m6, m7
    paddusw  m0, m2
    paddusw  m4, m6
    pavgw    m0, m4
    ret

cglobal x264_pixel_sa8d_8x8_%1, 4,6
    lea  r4, [3*r1]
    lea  r5, [3*r3]
    call x264_pixel_sa8d_8x8_internal_%1
    HADDW m0, m1
    movd eax, m0
    add eax, 1
    shr eax, 1
    ret

cglobal x264_pixel_sa8d_16x16_%1, 4,6
    lea  r4, [3*r1]
    lea  r5, [3*r3]
    call x264_pixel_sa8d_8x8_internal_%1 ; pix[0]
    add  r0, 8
    add  r2, 8
    mova m10, m0
    call x264_pixel_sa8d_8x8_internal_%1 ; pix[8]
    lea  r0, [r0+8*r1]
    lea  r2, [r2+8*r3]
    paddusw m10, m0
    call x264_pixel_sa8d_8x8_internal_%1 ; pix[8*stride+8]
    sub  r0, 8
    sub  r2, 8
    paddusw m10, m0
    call x264_pixel_sa8d_8x8_internal_%1 ; pix[8*stride]
    paddusw m0, m10
    HADDUW m0, m1
    movd eax, m0
    add  eax, 1
    shr  eax, 1
    ret

%else ; ARCH_X86_32
cglobal x264_pixel_sa8d_8x8_internal_%1
    LOAD_DIFF_8x4P m0, m1, m2, m3, m6, m7
    movdqa [esp+4], m2
    lea  r0, [r0+4*r1]
    lea  r2, [r2+4*r3]
    LOAD_DIFF_8x4P m4, m5, m6, m7, m2, m2
    movdqa m2, [esp+4]

    HADAMARD8_1D  m0, m1, m2, m3, m4, m5, m6, m7
    TRANSPOSE8x8W  0,  1,  2,  3,  4,  5,  6,  7, [esp+4], [esp+20]
    HADAMARD8_1D  m0, m1, m2, m3, m4, m5, m6, m7

%ifidn %1, sse2
    movdqa [esp+4], m4
    movdqa [esp+20], m2
%endif
    ABS2 m6, m3, m4, m2
    ABS2 m0, m7, m4, m2
    paddusw m0, m6
    paddusw m7, m3
%ifidn %1, sse2
    movdqa m4, [esp+4]
    movdqa m2, [esp+20]
%endif
    ABS2 m5, m1, m6, m3
    ABS2 m4, m2, m6, m3
    paddusw m5, m1
    paddusw m4, m2
    paddusw m0, m7
    paddusw m5, m4
    pavgw   m0, m5
    ret
%endif ; ARCH
%endmacro ; SATDS_SSE2

%macro SA8D_16x16_32 1
%ifndef ARCH_X86_64
cglobal x264_pixel_sa8d_8x8_%1, 4,7
    mov  r6, esp
    and  esp, ~15
    sub  esp, 32
    lea  r4, [3*r1]
    lea  r5, [3*r3]
    call x264_pixel_sa8d_8x8_internal_%1
    HADDW m0, m1
    movd eax, m0
    add  eax, 1
    shr  eax, 1
    mov  esp, r6
    RET

cglobal x264_pixel_sa8d_16x16_%1, 4,7
    mov  r6, esp
    and  esp, ~15
    sub  esp, 48
    lea  r4, [3*r1]
    lea  r5, [3*r3]
    call x264_pixel_sa8d_8x8_internal_%1
    lea  r0, [r0+4*r1]
    lea  r2, [r2+4*r3]
    mova [esp+32], m0
    call x264_pixel_sa8d_8x8_internal_%1
    mov  r0, [r6+20]
    mov  r2, [r6+28]
    add  r0, 8
    add  r2, 8
    paddusw m0, [esp+32]
    mova [esp+32], m0
    call x264_pixel_sa8d_8x8_internal_%1
    lea  r0, [r0+4*r1]
    lea  r2, [r2+4*r3]
%if mmsize == 16
    paddusw m0, [esp+32]
%endif
    mova [esp+48-mmsize], m0
    call x264_pixel_sa8d_8x8_internal_%1
    paddusw m0, [esp+48-mmsize]
%if mmsize == 16
    HADDUW m0, m1
%else
    mova m2, [esp+32]
    pxor m7, m7
    mova m1, m0
    mova m3, m2
    punpcklwd m0, m7
    punpckhwd m1, m7
    punpcklwd m2, m7
    punpckhwd m3, m7
    paddd m0, m1
    paddd m2, m3
    paddd m0, m2
    HADDD m0, m1
%endif
    movd eax, m0
    add  eax, 1
    shr  eax, 1
    mov  esp, r6
    RET
%endif ; !ARCH_X86_64
%endmacro ; SA8D_16x16_32


;=============================================================================
; INTRA SATD
;=============================================================================

%macro INTRA_SA8D_SSE2 1
%ifdef ARCH_X86_64
INIT_XMM
;-----------------------------------------------------------------------------
; void x264_intra_sa8d_x3_8x8_core_sse2( uint8_t *fenc, int16_t edges[2][8], int *res )
;-----------------------------------------------------------------------------
cglobal x264_intra_sa8d_x3_8x8_core_%1
    ; 8x8 hadamard
    pxor        m8, m8
    movq        m0, [r0+0*FENC_STRIDE]
    movq        m1, [r0+1*FENC_STRIDE]
    movq        m2, [r0+2*FENC_STRIDE]
    movq        m3, [r0+3*FENC_STRIDE]
    movq        m4, [r0+4*FENC_STRIDE]
    movq        m5, [r0+5*FENC_STRIDE]
    movq        m6, [r0+6*FENC_STRIDE]
    movq        m7, [r0+7*FENC_STRIDE]
    punpcklbw   m0, m8
    punpcklbw   m1, m8
    punpcklbw   m2, m8
    punpcklbw   m3, m8
    punpcklbw   m4, m8
    punpcklbw   m5, m8
    punpcklbw   m6, m8
    punpcklbw   m7, m8
    HADAMARD8_1D  m0, m1, m2, m3, m4, m5, m6, m7
    TRANSPOSE8x8W  0,  1,  2,  3,  4,  5,  6,  7,  8
    HADAMARD8_1D  m0, m1, m2, m3, m4, m5, m6, m7

    ; dc
    movzx       edi, word [r1+0]
    add          di, word [r1+16]
    add         edi, 8
    and         edi, -16
    shl         edi, 2

    pxor        m15, m15
    movdqa      m8,  m2
    movdqa      m9,  m3
    movdqa      m10, m4
    movdqa      m11, m5
    ABS4        m8, m9, m10, m11, m12, m13
    paddusw     m8,  m10
    paddusw     m9,  m11
%ifidn %1, ssse3
    pabsw       m10, m6
    pabsw       m11, m7
    pabsw       m15, m1
%else
    movdqa      m10, m6
    movdqa      m11, m7
    movdqa      m15, m1
    ABS2        m10, m11, m13, m14
    ABS1        m15, m13
%endif
    paddusw     m10, m11
    paddusw     m8,  m9
    paddusw     m15, m10
    paddusw     m15, m8
    movdqa      m14, m15 ; 7x8 sum

    movdqa      m8,  [r1+0] ; left edge
    movd        m9,  edi
    psllw       m8,  3
    psubw       m8,  m0
    psubw       m9,  m0
    ABS1        m8,  m10
    ABS1        m9,  m11 ; 1x8 sum
    paddusw     m14, m8
    paddusw     m15, m9
    punpcklwd   m0,  m1
    punpcklwd   m2,  m3
    punpcklwd   m4,  m5
    punpcklwd   m6,  m7
    punpckldq   m0,  m2
    punpckldq   m4,  m6
    punpcklqdq  m0,  m4 ; transpose
    movdqa      m1,  [r1+16] ; top edge
    movdqa      m2,  m15
    psllw       m1,  3
    psrldq      m2,  2     ; 8x7 sum
    psubw       m0,  m1  ; 8x1 sum
    ABS1        m0,  m1
    paddusw     m2,  m0

    ; 3x HADDW
    movdqa      m7,  [pw_1 GLOBAL]
    pmaddwd     m2,  m7
    pmaddwd     m14, m7
    pmaddwd     m15, m7
    movdqa      m3,  m2
    punpckldq   m2,  m14
    punpckhdq   m3,  m14
    pshufd      m5,  m15, 0xf5
    paddd       m2,  m3
    paddd       m5,  m15
    movdqa      m3,  m2
    punpcklqdq  m2,  m5
    punpckhqdq  m3,  m5
    pavgw       m3,  m2
    pxor        m0,  m0
    pavgw       m3,  m0
    movq      [r2],  m3 ; i8x8_v, i8x8_h
    psrldq      m3,  8
    movd    [r2+8],  m3 ; i8x8_dc
    ret
%endif ; ARCH_X86_64
%endmacro ; INTRA_SA8D_SSE2

; in: r0 = fenc
; out: m0..m3 = hadamard coefs
INIT_MMX
ALIGN 16
load_hadamard:
    pxor        m7, m7
    movd        m0, [r0+0*FENC_STRIDE]
    movd        m1, [r0+1*FENC_STRIDE]
    movd        m2, [r0+2*FENC_STRIDE]
    movd        m3, [r0+3*FENC_STRIDE]
    punpcklbw   m0, m7
    punpcklbw   m1, m7
    punpcklbw   m2, m7
    punpcklbw   m3, m7
    HADAMARD4_1D  m0, m1, m2, m3
    TRANSPOSE4x4W  0,  1,  2,  3,  4
    HADAMARD4_1D  m0, m1, m2, m3
    SAVE_MM_PERMUTATION load_hadamard
    ret

%macro SCALAR_SUMSUB 4
    add %1, %2
    add %3, %4
    add %2, %2
    add %4, %4
    sub %2, %1
    sub %4, %3
%endmacro

%macro SCALAR_HADAMARD_LEFT 5 ; y, 4x tmp
%ifnidn %1, 0
    shl         %1d, 5 ; log(FDEC_STRIDE)
%endif
    movzx       %2d, byte [r1+%1-1+0*FDEC_STRIDE]
    movzx       %3d, byte [r1+%1-1+1*FDEC_STRIDE]
    movzx       %4d, byte [r1+%1-1+2*FDEC_STRIDE]
    movzx       %5d, byte [r1+%1-1+3*FDEC_STRIDE]
%ifnidn %1, 0
    shr         %1d, 5
%endif
    SCALAR_SUMSUB %2d, %3d, %4d, %5d
    SCALAR_SUMSUB %2d, %4d, %3d, %5d
    mov         [left_1d+2*%1+0], %2w
    mov         [left_1d+2*%1+2], %3w
    mov         [left_1d+2*%1+4], %4w
    mov         [left_1d+2*%1+6], %5w
%endmacro

%macro SCALAR_HADAMARD_TOP 5 ; x, 4x tmp
    movzx       %2d, byte [r1+%1-FDEC_STRIDE+0]
    movzx       %3d, byte [r1+%1-FDEC_STRIDE+1]
    movzx       %4d, byte [r1+%1-FDEC_STRIDE+2]
    movzx       %5d, byte [r1+%1-FDEC_STRIDE+3]
    SCALAR_SUMSUB %2d, %3d, %4d, %5d
    SCALAR_SUMSUB %2d, %4d, %3d, %5d
    mov         [top_1d+2*%1+0], %2w
    mov         [top_1d+2*%1+2], %3w
    mov         [top_1d+2*%1+4], %4w
    mov         [top_1d+2*%1+6], %5w
%endmacro

%macro SUM_MM_X3 8 ; 3x sum, 4x tmp, op
    pxor        %7, %7
    pshufw      %4, %1, 01001110b
    pshufw      %5, %2, 01001110b
    pshufw      %6, %3, 01001110b
    paddw       %1, %4
    paddw       %2, %5
    paddw       %3, %6
    punpcklwd   %1, %7
    punpcklwd   %2, %7
    punpcklwd   %3, %7
    pshufw      %4, %1, 01001110b
    pshufw      %5, %2, 01001110b
    pshufw      %6, %3, 01001110b
    %8          %1, %4
    %8          %2, %5
    %8          %3, %6
%endmacro

%macro CLEAR_SUMS 0
%ifdef ARCH_X86_64
    mov   qword [sums+0], 0
    mov   qword [sums+8], 0
    mov   qword [sums+16], 0
%else
    pxor  m7, m7
    movq  [sums+0], m7
    movq  [sums+8], m7
    movq  [sums+16], m7
%endif
%endmacro

; in: m1..m3
; out: m7
; clobber: m4..m6
%macro SUM3x4 1
%ifidn %1, ssse3
    pabsw       m4, m1
    pabsw       m5, m2
    pabsw       m7, m3
    paddw       m4, m5
%else
    movq        m4, m1
    movq        m5, m2
    ABS2        m4, m5, m6, m7
    movq        m7, m3
    paddw       m4, m5
    ABS1        m7, m6
%endif
    paddw       m7, m4
%endmacro

; in: m0..m3 (4x4), m7 (3x4)
; out: m0 v, m4 h, m5 dc
; clobber: m6
%macro SUM4x3 3 ; dc, left, top
    movq        m4, %2
    movd        m5, %1
    psllw       m4, 2
    psubw       m4, m0
    psubw       m5, m0
    punpcklwd   m0, m1
    punpcklwd   m2, m3
    punpckldq   m0, m2 ; transpose
    movq        m1, %3
    psllw       m1, 2
    psubw       m0, m1
    ABS2        m4, m5, m2, m3 ; 1x4 sum
    ABS1        m0, m1 ; 4x1 sum
%endmacro

%macro INTRA_SATDS_MMX 1
INIT_MMX
;-----------------------------------------------------------------------------
; void x264_intra_satd_x3_4x4_mmxext( uint8_t *fenc, uint8_t *fdec, int *res )
;-----------------------------------------------------------------------------
cglobal x264_intra_satd_x3_4x4_%1, 2,6
%ifdef ARCH_X86_64
    ; stack is 16 byte aligned because abi says so
    %define  top_1d  rsp-8  ; size 8
    %define  left_1d rsp-16 ; size 8
    %define  t0  r10
    %define  t0d r10d
%else
    ; stack is 16 byte aligned at least in gcc, and we've pushed 3 regs + return address, so it's still aligned
    SUB         esp, 16
    %define  top_1d  esp+8
    %define  left_1d esp
    %define  t0  r2
    %define  t0d r2d
%endif

    call load_hadamard
    SCALAR_HADAMARD_LEFT 0, r0, r3, r4, r5
    mov         t0d, r0d
    SCALAR_HADAMARD_TOP  0, r0, r3, r4, r5
    lea         t0d, [t0d + r0d + 4]
    and         t0d, -8
    shl         t0d, 1 ; dc

    SUM3x4 %1
    SUM4x3 t0d, [left_1d], [top_1d]
    paddw       m4, m7
    paddw       m5, m7
    movq        m1, m5
    psrlq       m1, 16  ; 4x3 sum
    paddw       m0, m1

    SUM_MM_X3   m0, m4, m5, m1, m2, m3, m6, pavgw
%ifndef ARCH_X86_64
    mov         r2, r2m
%endif
    movd        [r2+0], m0 ; i4x4_v satd
    movd        [r2+4], m4 ; i4x4_h satd
    movd        [r2+8], m5 ; i4x4_dc satd
%ifndef ARCH_X86_64
    ADD         esp, 16
%endif
    RET

%ifdef ARCH_X86_64
    %define  t0  r10
    %define  t0d r10d
    %define  t2  r11
    %define  t2w r11w
    %define  t2d r11d
%else
    %define  t0  r0
    %define  t0d r0d
    %define  t2  r2
    %define  t2w r2w
    %define  t2d r2d
%endif

;-----------------------------------------------------------------------------
; void x264_intra_satd_x3_16x16_mmxext( uint8_t *fenc, uint8_t *fdec, int *res )
;-----------------------------------------------------------------------------
cglobal x264_intra_satd_x3_16x16_%1, 0,7
%ifdef ARCH_X86_64
    %assign  stack_pad  88
%else
    %assign  stack_pad  88 + ((stack_offset+88+4)&15)
%endif
    ; not really needed on x86_64, just shuts up valgrind about storing data below the stack across a function call
    SUB         rsp, stack_pad
%define sums    rsp+64 ; size 24
%define top_1d  rsp+32 ; size 32
%define left_1d rsp    ; size 32
    movifnidn   r1d, r1m
    CLEAR_SUMS

    ; 1D hadamards
    xor         t2d, t2d
    mov         t0d, 12
.loop_edge:
    SCALAR_HADAMARD_LEFT t0, r3, r4, r5, r6
    add         t2d, r3d
    SCALAR_HADAMARD_TOP  t0, r3, r4, r5, r6
    add         t2d, r3d
    sub         t0d, 4
    jge .loop_edge
    shr         t2d, 1
    add         t2d, 8
    and         t2d, -16 ; dc

    ; 2D hadamards
    movifnidn   r0d, r0m
    xor         r3d, r3d
.loop_y:
    xor         r4d, r4d
.loop_x:
    call load_hadamard

    SUM3x4 %1
    SUM4x3 t2d, [left_1d+8*r3], [top_1d+8*r4]
    pavgw       m4, m7
    pavgw       m5, m7
    paddw       m0, [sums+0]  ; i16x16_v satd
    paddw       m4, [sums+8]  ; i16x16_h satd
    paddw       m5, [sums+16] ; i16x16_dc satd
    movq        [sums+0], m0
    movq        [sums+8], m4
    movq        [sums+16], m5

    add         r0, 4
    inc         r4d
    cmp         r4d, 4
    jl  .loop_x
    add         r0, 4*FENC_STRIDE-16
    inc         r3d
    cmp         r3d, 4
    jl  .loop_y

; horizontal sum
    movifnidn   r2d, r2m
    movq        m2, [sums+16]
    movq        m1, [sums+8]
    movq        m0, [sums+0]
    movq        m7, m2
    SUM_MM_X3   m0, m1, m2, m3, m4, m5, m6, paddd
    psrld       m0, 1
    pslld       m7, 16
    psrld       m7, 16
    paddd       m0, m2
    psubd       m0, m7
    movd        [r2+8], m2 ; i16x16_dc satd
    movd        [r2+4], m1 ; i16x16_h satd
    movd        [r2+0], m0 ; i16x16_v satd
    ADD         rsp, stack_pad
    RET

;-----------------------------------------------------------------------------
; void x264_intra_satd_x3_8x8c_mmxext( uint8_t *fenc, uint8_t *fdec, int *res )
;-----------------------------------------------------------------------------
cglobal x264_intra_satd_x3_8x8c_%1, 0,6
    ; not really needed on x86_64, just shuts up valgrind about storing data below the stack across a function call
    SUB          rsp, 72
%define  sums    rsp+48 ; size 24
%define  dc_1d   rsp+32 ; size 16
%define  top_1d  rsp+16 ; size 16
%define  left_1d rsp    ; size 16
    movifnidn   r1d, r1m
    CLEAR_SUMS

    ; 1D hadamards
    mov         t0d, 4
.loop_edge:
    SCALAR_HADAMARD_LEFT t0, t2, r3, r4, r5
    SCALAR_HADAMARD_TOP  t0, t2, r3, r4, r5
    sub         t0d, 4
    jge .loop_edge

    ; dc
    movzx       t2d, word [left_1d+0]
    movzx       r3d, word [top_1d+0]
    movzx       r4d, word [left_1d+8]
    movzx       r5d, word [top_1d+8]
    add         t2d, r3d
    lea         r3, [r4 + r5]
    lea         t2, [2*t2 + 8]
    lea         r3, [2*r3 + 8]
    lea         r4, [4*r4 + 8]
    lea         r5, [4*r5 + 8]
    and         t2d, -16 ; tl
    and         r3d, -16 ; br
    and         r4d, -16 ; bl
    and         r5d, -16 ; tr
    mov         [dc_1d+ 0], t2d ; tl
    mov         [dc_1d+ 4], r5d ; tr
    mov         [dc_1d+ 8], r4d ; bl
    mov         [dc_1d+12], r3d ; br
    lea         r5, [dc_1d]

    ; 2D hadamards
    movifnidn   r0d, r0m
    movifnidn   r2d, r2m
    xor         r3d, r3d
.loop_y:
    xor         r4d, r4d
.loop_x:
    call load_hadamard

    SUM3x4 %1
    SUM4x3 [r5+4*r4], [left_1d+8*r3], [top_1d+8*r4]
    pavgw       m4, m7
    pavgw       m5, m7
    paddw       m0, [sums+16] ; i4x4_v satd
    paddw       m4, [sums+8]  ; i4x4_h satd
    paddw       m5, [sums+0]  ; i4x4_dc satd
    movq        [sums+16], m0
    movq        [sums+8], m4
    movq        [sums+0], m5

    add         r0, 4
    inc         r4d
    cmp         r4d, 2
    jl  .loop_x
    add         r0, 4*FENC_STRIDE-8
    add         r5, 8
    inc         r3d
    cmp         r3d, 2
    jl  .loop_y

; horizontal sum
    movq        m0, [sums+0]
    movq        m1, [sums+8]
    movq        m2, [sums+16]
    movq        m7, m0
    psrlq       m7, 15
    paddw       m2, m7
    SUM_MM_X3   m0, m1, m2, m3, m4, m5, m6, paddd
    psrld       m2, 1
    movd        [r2+0], m0 ; i8x8c_dc satd
    movd        [r2+4], m1 ; i8x8c_h satd
    movd        [r2+8], m2 ; i8x8c_v satd
    ADD         rsp, 72
    RET
%endmacro ; INTRA_SATDS_MMX


%macro ABS_MOV_SSSE3 2
    pabsw   %1, %2
%endmacro

%macro ABS_MOV_MMX 2
    pxor    %1, %1
    psubw   %1, %2
    pmaxsw  %1, %2
%endmacro

%define ABS_MOV ABS_MOV_MMX

; in:  r0=pix, r1=stride, r2=stride*3, r3=tmp, m6=mask_ac4, m7=0
; out: [tmp]=hadamard4, m0=satd
cglobal x264_hadamard_ac_4x4_mmxext
    movh      m0, [r0]
    movh      m1, [r0+r1]
    movh      m2, [r0+r1*2]
    movh      m3, [r0+r2]
    punpcklbw m0, m7
    punpcklbw m1, m7
    punpcklbw m2, m7
    punpcklbw m3, m7
    HADAMARD4_1D m0, m1, m2, m3
    TRANSPOSE4x4W 0, 1, 2, 3, 4
    HADAMARD4_1D m0, m1, m2, m3
    mova [r3],    m0
    mova [r3+8],  m1
    mova [r3+16], m2
    mova [r3+24], m3
    ABS1      m0, m4
    ABS1      m1, m4
    pand      m0, m6
    ABS1      m2, m4
    ABS1      m3, m4
    paddw     m0, m1
    paddw     m2, m3
    paddw     m0, m2
    SAVE_MM_PERMUTATION x264_hadamard_ac_4x4_mmxext
    ret

cglobal x264_hadamard_ac_2x2_mmxext
    mova      m0, [r3+0x00]
    mova      m1, [r3+0x20]
    mova      m2, [r3+0x40]
    mova      m3, [r3+0x60]
    HADAMARD4_1D m0, m1, m2, m3
    ABS2      m0, m1, m4, m5
    ABS2      m2, m3, m4, m5
    SAVE_MM_PERMUTATION x264_hadamard_ac_2x2_mmxext
    ret

cglobal x264_hadamard_ac_8x8_mmxext
    mova      m6, [mask_ac4 GLOBAL]
    pxor      m7, m7
    call x264_hadamard_ac_4x4_mmxext
    add       r0, 4
    add       r3, 32
    mova      m5, m0
    call x264_hadamard_ac_4x4_mmxext
    lea       r0, [r0+4*r1]
    add       r3, 64
    paddw     m5, m0
    call x264_hadamard_ac_4x4_mmxext
    sub       r0, 4
    sub       r3, 32
    paddw     m5, m0
    call x264_hadamard_ac_4x4_mmxext
    paddw     m5, m0
    sub       r3, 64
    mova [rsp+gprsize+8], m5 ; save satd
    call x264_hadamard_ac_2x2_mmxext
    add       r3, 8
    pand      m6, m0
    mova      m7, m1
    paddw     m6, m2
    paddw     m7, m3
%rep 2
    call x264_hadamard_ac_2x2_mmxext
    add       r3, 8
    paddw     m6, m0
    paddw     m7, m1
    paddw     m6, m2
    paddw     m7, m3
%endrep
    call x264_hadamard_ac_2x2_mmxext
    sub       r3, 24
    paddw     m6, m0
    paddw     m7, m1
    paddw     m6, m2
    paddw     m7, m3
    paddw     m6, m7
    mova [rsp+gprsize], m6 ; save sa8d
    SWAP      m0, m6
    SAVE_MM_PERMUTATION x264_hadamard_ac_8x8_mmxext
    ret

%macro HADAMARD_AC_WXH_MMX 2
cglobal x264_pixel_hadamard_ac_%1x%2_mmxext, 2,4
    %assign pad 16-gprsize-(stack_offset&15)
    %define ysub r1
    sub  rsp, 16+128+pad
    lea  r2, [r1*3]
    lea  r3, [rsp+16]
    call x264_hadamard_ac_8x8_mmxext
%if %2==16
    %define ysub r2
    lea  r0, [r0+r1*4]
    sub  rsp, 16
    call x264_hadamard_ac_8x8_mmxext
%endif
%if %1==16
    neg  ysub
    sub  rsp, 16
    lea  r0, [r0+ysub*4+8]
    neg  ysub
    call x264_hadamard_ac_8x8_mmxext
%if %2==16
    lea  r0, [r0+r1*4]
    sub  rsp, 16
    call x264_hadamard_ac_8x8_mmxext
%endif
%endif
    mova    m1, [rsp+0x08]
%if %1*%2 >= 128
    paddusw m0, [rsp+0x10]
    paddusw m1, [rsp+0x18]
%endif
%if %1*%2 == 256
    mova    m2, [rsp+0x20]
    paddusw m1, [rsp+0x28]
    paddusw m2, [rsp+0x30]
    mova    m3, m0
    paddusw m1, [rsp+0x38]
    pxor    m3, m2
    pand    m3, [pw_1 GLOBAL]
    pavgw   m0, m2
    psubusw m0, m3
    HADDUW  m0, m2
%else
    psrlw m0, 1
    HADDW m0, m2
%endif
    psrlw m1, 1
    HADDW m1, m3
    movd edx, m0
    movd eax, m1
    shr  edx, 1
%ifdef ARCH_X86_64
    shl  rdx, 32
    add  rax, rdx
%endif
    add  rsp, 128+%1*%2/4+pad
    RET
%endmacro ; HADAMARD_AC_WXH_MMX

HADAMARD_AC_WXH_MMX 16, 16
HADAMARD_AC_WXH_MMX  8, 16
HADAMARD_AC_WXH_MMX 16,  8
HADAMARD_AC_WXH_MMX  8,  8

%macro HADAMARD_AC_SSE2 1
INIT_XMM
; in:  r0=pix, r1=stride, r2=stride*3
; out: [esp+16]=sa8d, [esp+32]=satd, r0+=stride*4
cglobal x264_hadamard_ac_8x8_%1
%ifdef ARCH_X86_64
    %define spill0 m8
    %define spill1 m9
    %define spill2 m10
%else
    %define spill0 [rsp+gprsize]
    %define spill1 [rsp+gprsize+16]
    %define spill2 [rsp+gprsize+32]
%endif
    pxor      m7, m7
    movh      m0, [r0]
    movh      m1, [r0+r1]
    movh      m2, [r0+r1*2]
    movh      m3, [r0+r2]
    lea       r0, [r0+r1*4]
    punpcklbw m0, m7
    punpcklbw m1, m7
    punpcklbw m2, m7
    punpcklbw m3, m7
    HADAMARD4_1D m0, m1, m2, m3
    mova  spill0, m3
    SWAP      m3, m7
    movh      m4, [r0]
    movh      m5, [r0+r1]
    movh      m6, [r0+r1*2]
    movh      m7, [r0+r2]
    punpcklbw m4, m3
    punpcklbw m5, m3
    punpcklbw m6, m3
    punpcklbw m7, m3
    HADAMARD4_1D m4, m5, m6, m7
    mova      m3, spill0
%ifdef ARCH_X86_64
    TRANSPOSE8x8W 0,1,2,3,4,5,6,7,8
%else
    TRANSPOSE8x8W 0,1,2,3,4,5,6,7,spill0,spill1
%endif
    HADAMARD4_1D m0, m1, m2, m3
    HADAMARD4_1D m4, m5, m6, m7
    mova  spill0, m1
    mova  spill1, m2
    mova  spill2, m3
    ABS_MOV   m1, m0
    ABS_MOV   m2, m4
    ABS_MOV   m3, m5
    paddw     m1, m2
    SUMSUB_BA m0, m4
    pand      m1, [mask_ac4 GLOBAL]
    ABS_MOV   m2, spill0
    paddw     m1, m3
    ABS_MOV   m3, spill1
    paddw     m1, m2
    ABS_MOV   m2, spill2
    paddw     m1, m3
    ABS_MOV   m3, m6
    paddw     m1, m2
    ABS_MOV   m2, m7
    paddw     m1, m3
    mova      m3, m7
    paddw     m1, m2
    mova      m2, m6
    psubw     m7, spill2
    paddw     m3, spill2
    mova  [rsp+gprsize+32], m1 ; save satd
    mova      m1, m5
    psubw     m6, spill1
    paddw     m2, spill1
    psubw     m5, spill0
    paddw     m1, spill0
    mova  spill1, m7
    SBUTTERFLY qdq, 0, 4, 7
    SBUTTERFLY qdq, 1, 5, 7
    SBUTTERFLY qdq, 2, 6, 7
    SUMSUB_BADC m0, m4, m1, m5
    SUMSUB_BA m2, m6
    ABS1      m0, m7
    ABS1      m1, m7
    pand      m0, [mask_ac8 GLOBAL]
    ABS1      m2, m7
    ABS1      m4, m7
    ABS1      m5, m7
    ABS1      m6, m7
    mova      m7, spill1
    paddw     m0, m4
    SBUTTERFLY qdq, 3, 7, 4
    SUMSUB_BA m3, m7
    paddw     m1, m5
    ABS1      m3, m4
    ABS1      m7, m4
    paddw     m2, m6
    paddw     m3, m7
    paddw     m0, m1
    paddw     m2, m3
    paddw     m0, m2
    mova  [rsp+gprsize+16], m0 ; save sa8d
    SAVE_MM_PERMUTATION x264_hadamard_ac_8x8_%1
    ret

HADAMARD_AC_WXH_SSE2 16, 16, %1
HADAMARD_AC_WXH_SSE2  8, 16, %1
HADAMARD_AC_WXH_SSE2 16,  8, %1
HADAMARD_AC_WXH_SSE2  8,  8, %1
%endmacro ; HADAMARD_AC_SSE2

; struct { int satd, int sa8d; } x264_pixel_hadamard_ac_16x16( uint8_t *pix, int stride )
%macro HADAMARD_AC_WXH_SSE2 3
cglobal x264_pixel_hadamard_ac_%1x%2_%3, 2,3
    %assign pad 16-gprsize-(stack_offset&15)
    %define ysub r1
    sub  rsp, 48+pad
    lea  r2, [r1*3]
    call x264_hadamard_ac_8x8_%3
%if %2==16
    %define ysub r2
    lea  r0, [r0+r1*4]
    sub  rsp, 32
    call x264_hadamard_ac_8x8_%3
%endif
%if %1==16
    neg  ysub
    sub  rsp, 32
    lea  r0, [r0+ysub*4+8]
    neg  ysub
    call x264_hadamard_ac_8x8_%3
%if %2==16
    lea  r0, [r0+r1*4]
    sub  rsp, 32
    call x264_hadamard_ac_8x8_%3
%endif
%endif
    mova    m1, [rsp+0x20]
%if %1*%2 >= 128
    paddusw m0, [rsp+0x30]
    paddusw m1, [rsp+0x40]
%endif
%if %1*%2 == 256
    paddusw m0, [rsp+0x50]
    paddusw m1, [rsp+0x60]
    paddusw m0, [rsp+0x70]
    paddusw m1, [rsp+0x80]
    psrlw m0, 1
%endif
    HADDW m0, m2
    HADDW m1, m3
    movd edx, m0
    movd eax, m1
    shr  edx, 2 - (%1*%2 >> 8)
    shr  eax, 1
%ifdef ARCH_X86_64
    shl  rdx, 32
    add  rax, rdx
%endif
    add  rsp, 16+%1*%2/2+pad
    RET
%endmacro ; HADAMARD_AC_WXH_SSE2

; instantiate satds

%ifndef ARCH_X86_64
cextern x264_pixel_sa8d_8x8_internal_mmxext
SA8D_16x16_32 mmxext
%endif

%define ABS1 ABS1_MMX
%define ABS2 ABS2_MMX
SATDS_SSE2 sse2
SA8D_16x16_32 sse2
INTRA_SA8D_SSE2 sse2
INTRA_SATDS_MMX mmxext
HADAMARD_AC_SSE2 sse2
%define ABS1 ABS1_SSSE3
%define ABS2 ABS2_SSSE3
%define ABS_MOV ABS_MOV_SSSE3
SATD_W4 ssse3 ; mmx, but uses pabsw from ssse3.
SATDS_SSE2 ssse3
SA8D_16x16_32 ssse3
INTRA_SA8D_SSE2 ssse3
INTRA_SATDS_MMX ssse3
HADAMARD_AC_SSE2 ssse3
SATDS_SSE2 ssse3_phadd


;=============================================================================
; SSIM
;=============================================================================

;-----------------------------------------------------------------------------
; void x264_pixel_ssim_4x4x2_core_sse2( const uint8_t *pix1, int stride1,
;                                       const uint8_t *pix2, int stride2, int sums[2][4] )
;-----------------------------------------------------------------------------
cglobal x264_pixel_ssim_4x4x2_core_sse2, 4,4
    pxor      m0, m0
    pxor      m1, m1
    pxor      m2, m2
    pxor      m3, m3
    pxor      m4, m4
%rep 4
    movq      m5, [r0]
    movq      m6, [r2]
    punpcklbw m5, m0
    punpcklbw m6, m0
    paddw     m1, m5
    paddw     m2, m6
    movdqa    m7, m5
    pmaddwd   m5, m5
    pmaddwd   m7, m6
    pmaddwd   m6, m6
    paddd     m3, m5
    paddd     m4, m7
    paddd     m3, m6
    add       r0, r1
    add       r2, r3
%endrep
    ; PHADDW m1, m2
    ; PHADDD m3, m4
    movdqa    m7, [pw_1 GLOBAL]
    pshufd    m5, m3, 0xb1
    pmaddwd   m1, m7
    pmaddwd   m2, m7
    pshufd    m6, m4, 0xb1
    packssdw  m1, m2
    paddd     m3, m5
    pshufd    m1, m1, 0xd8
    paddd     m4, m6
    pmaddwd   m1, m7
    movdqa    m5, m3
    punpckldq m3, m4
    punpckhdq m5, m4

%ifdef ARCH_X86_64
    %define t0 r4
%else
    %define t0 eax
    mov t0, r4m
%endif

    movq      [t0+ 0], m1
    movq      [t0+ 8], m3
    psrldq    m1, 8
    movq      [t0+16], m1
    movq      [t0+24], m5
    RET

;-----------------------------------------------------------------------------
; float x264_pixel_ssim_end_sse2( int sum0[5][4], int sum1[5][4], int width )
;-----------------------------------------------------------------------------
cglobal x264_pixel_ssim_end4_sse2, 3,3
    movdqa    m0, [r0+ 0]
    movdqa    m1, [r0+16]
    movdqa    m2, [r0+32]
    movdqa    m3, [r0+48]
    movdqa    m4, [r0+64]
    paddd     m0, [r1+ 0]
    paddd     m1, [r1+16]
    paddd     m2, [r1+32]
    paddd     m3, [r1+48]
    paddd     m4, [r1+64]
    paddd     m0, m1
    paddd     m1, m2
    paddd     m2, m3
    paddd     m3, m4
    movdqa    m5, [ssim_c1 GLOBAL]
    movdqa    m6, [ssim_c2 GLOBAL]
    TRANSPOSE4x4D  0, 1, 2, 3, 4

;   s1=m0, s2=m1, ss=m2, s12=m3
    movdqa    m4, m1
    pslld     m1, 16
    pmaddwd   m4, m0  ; s1*s2
    por       m0, m1
    pmaddwd   m0, m0  ; s1*s1 + s2*s2
    pslld     m4, 1
    pslld     m3, 7
    pslld     m2, 6
    psubd     m3, m4  ; covar*2
    psubd     m2, m0  ; vars
    paddd     m0, m5
    paddd     m4, m5
    paddd     m3, m6
    paddd     m2, m6
    cvtdq2ps  m0, m0  ; (float)(s1*s1 + s2*s2 + ssim_c1)
    cvtdq2ps  m4, m4  ; (float)(s1*s2*2 + ssim_c1)
    cvtdq2ps  m3, m3  ; (float)(covar*2 + ssim_c2)
    cvtdq2ps  m2, m2  ; (float)(vars + ssim_c2)
    mulps     m4, m3
    mulps     m0, m2
    divps     m4, m0  ; ssim

    cmp       r2d, 4
    je .skip ; faster only if this is the common case; remove branch if we use ssim on a macroblock level
    neg       r2
%ifdef PIC
    lea       r3, [mask_ff + 16 GLOBAL]
    movdqu    m1, [r3 + r2*4]
%else
    movdqu    m1, [mask_ff + r2*4 + 16 GLOBAL]
%endif
    pand      m4, m1
.skip:
    movhlps   m0, m4
    addps     m0, m4
    pshuflw   m4, m0, 0xE
    addss     m0, m4
%ifndef ARCH_X86_64
    movd     r0m, m0
    fld     dword r0m
%endif
    RET


;=============================================================================
; Successive Elimination ADS
;=============================================================================

%macro ADS_START 1 ; unroll_size
%ifdef ARCH_X86_64
    %define t0  r6
    mov     r10, rsp
%else
    %define t0  r4
    mov     rbp, rsp
%endif
    mov     r0d, r5m
    sub     rsp, r0
    sub     rsp, %1*4-1
    and     rsp, ~15
    mov     t0,  rsp
    shl     r2d,  1
%endmacro

%macro ADS_END 1
    add     r1, 8*%1
    add     r3, 8*%1
    add     t0, 4*%1
    sub     r0d, 4*%1
    jg .loop
    jmp ads_mvs
%endmacro

%define ABS1 ABS1_MMX

;-----------------------------------------------------------------------------
; int x264_pixel_ads4_mmxext( int enc_dc[4], uint16_t *sums, int delta,
;                             uint16_t *cost_mvx, int16_t *mvs, int width, int thresh )
;-----------------------------------------------------------------------------
cglobal x264_pixel_ads4_mmxext, 4,7
    movq    mm6, [r0]
    movq    mm4, [r0+8]
    pshufw  mm7, mm6, 0
    pshufw  mm6, mm6, 0xAA
    pshufw  mm5, mm4, 0
    pshufw  mm4, mm4, 0xAA
    ADS_START 1
.loop:
    movq    mm0, [r1]
    movq    mm1, [r1+16]
    psubw   mm0, mm7
    psubw   mm1, mm6
    ABS1    mm0, mm2
    ABS1    mm1, mm3
    movq    mm2, [r1+r2]
    movq    mm3, [r1+r2+16]
    psubw   mm2, mm5
    psubw   mm3, mm4
    paddw   mm0, mm1
    ABS1    mm2, mm1
    ABS1    mm3, mm1
    paddw   mm0, mm2
    paddw   mm0, mm3
%ifdef ARCH_X86_64
    pshufw  mm1, [r10+8], 0
%else
    pshufw  mm1, [ebp+stack_offset+28], 0
%endif
    paddusw mm0, [r3]
    psubusw mm1, mm0
    packsswb mm1, mm1
    movd    [t0], mm1
    ADS_END 1

cglobal x264_pixel_ads2_mmxext, 4,7
    movq    mm6, [r0]
    pshufw  mm5, r6m, 0
    pshufw  mm7, mm6, 0
    pshufw  mm6, mm6, 0xAA
    ADS_START 1
.loop:
    movq    mm0, [r1]
    movq    mm1, [r1+r2]
    psubw   mm0, mm7
    psubw   mm1, mm6
    ABS1    mm0, mm2
    ABS1    mm1, mm3
    paddw   mm0, mm1
    paddusw mm0, [r3]
    movq    mm4, mm5
    psubusw mm4, mm0
    packsswb mm4, mm4
    movd    [t0], mm4
    ADS_END 1

cglobal x264_pixel_ads1_mmxext, 4,7
    pshufw  mm7, [r0], 0
    pshufw  mm6, r6m, 0
    ADS_START 2
.loop:
    movq    mm0, [r1]
    movq    mm1, [r1+8]
    psubw   mm0, mm7
    psubw   mm1, mm7
    ABS1    mm0, mm2
    ABS1    mm1, mm3
    paddusw mm0, [r3]
    paddusw mm1, [r3+8]
    movq    mm4, mm6
    movq    mm5, mm6
    psubusw mm4, mm0
    psubusw mm5, mm1
    packsswb mm4, mm5
    movq    [t0], mm4
    ADS_END 2

%macro ADS_SSE2 1
cglobal x264_pixel_ads4_%1, 4,7
    movdqa  xmm4, [r0]
    pshuflw xmm7, xmm4, 0
    pshuflw xmm6, xmm4, 0xAA
    pshufhw xmm5, xmm4, 0
    pshufhw xmm4, xmm4, 0xAA
    punpcklqdq xmm7, xmm7
    punpcklqdq xmm6, xmm6
    punpckhqdq xmm5, xmm5
    punpckhqdq xmm4, xmm4
%ifdef ARCH_X86_64
    pshuflw xmm8, r6m, 0
    punpcklqdq xmm8, xmm8
    ADS_START 2
    movdqu  xmm10, [r1]
    movdqu  xmm11, [r1+r2]
.loop:
    movdqa  xmm0, xmm10
    movdqu  xmm1, [r1+16]
    movdqa  xmm10, xmm1
    psubw   xmm0, xmm7
    psubw   xmm1, xmm6
    ABS1    xmm0, xmm2
    ABS1    xmm1, xmm3
    movdqa  xmm2, xmm11
    movdqu  xmm3, [r1+r2+16]
    movdqa  xmm11, xmm3
    psubw   xmm2, xmm5
    psubw   xmm3, xmm4
    paddw   xmm0, xmm1
    movdqu  xmm9, [r3]
    ABS1    xmm2, xmm1
    ABS1    xmm3, xmm1
    paddw   xmm0, xmm2
    paddw   xmm0, xmm3
    paddusw xmm0, xmm9
    movdqa  xmm1, xmm8
    psubusw xmm1, xmm0
    packsswb xmm1, xmm1
    movq    [t0], xmm1
%else
    ADS_START 2
.loop:
    movdqu  xmm0, [r1]
    movdqu  xmm1, [r1+16]
    psubw   xmm0, xmm7
    psubw   xmm1, xmm6
    ABS1    xmm0, xmm2
    ABS1    xmm1, xmm3
    movdqu  xmm2, [r1+r2]
    movdqu  xmm3, [r1+r2+16]
    psubw   xmm2, xmm5
    psubw   xmm3, xmm4
    paddw   xmm0, xmm1
    ABS1    xmm2, xmm1
    ABS1    xmm3, xmm1
    paddw   xmm0, xmm2
    paddw   xmm0, xmm3
    movd    xmm1, [ebp+stack_offset+28]
    movdqu  xmm2, [r3]
    pshuflw xmm1, xmm1, 0
    punpcklqdq xmm1, xmm1
    paddusw xmm0, xmm2
    psubusw xmm1, xmm0
    packsswb xmm1, xmm1
    movq    [t0], xmm1
%endif ; ARCH
    ADS_END 2

cglobal x264_pixel_ads2_%1, 4,7
    movq    xmm6, [r0]
    movd    xmm5, r6m
    pshuflw xmm7, xmm6, 0
    pshuflw xmm6, xmm6, 0xAA
    pshuflw xmm5, xmm5, 0
    punpcklqdq xmm7, xmm7
    punpcklqdq xmm6, xmm6
    punpcklqdq xmm5, xmm5
    ADS_START 2
.loop:
    movdqu  xmm0, [r1]
    movdqu  xmm1, [r1+r2]
    psubw   xmm0, xmm7
    psubw   xmm1, xmm6
    movdqu  xmm4, [r3]
    ABS1    xmm0, xmm2
    ABS1    xmm1, xmm3
    paddw   xmm0, xmm1
    paddusw xmm0, xmm4
    movdqa  xmm1, xmm5
    psubusw xmm1, xmm0
    packsswb xmm1, xmm1
    movq    [t0], xmm1
    ADS_END 2

cglobal x264_pixel_ads1_%1, 4,7
    movd    xmm7, [r0]
    movd    xmm6, r6m
    pshuflw xmm7, xmm7, 0
    pshuflw xmm6, xmm6, 0
    punpcklqdq xmm7, xmm7
    punpcklqdq xmm6, xmm6
    ADS_START 4
.loop:
    movdqu  xmm0, [r1]
    movdqu  xmm1, [r1+16]
    psubw   xmm0, xmm7
    psubw   xmm1, xmm7
    movdqu  xmm2, [r3]
    movdqu  xmm3, [r3+16]
    ABS1    xmm0, xmm4
    ABS1    xmm1, xmm5
    paddusw xmm0, xmm2
    paddusw xmm1, xmm3
    movdqa  xmm4, xmm6
    movdqa  xmm5, xmm6
    psubusw xmm4, xmm0
    psubusw xmm5, xmm1
    packsswb xmm4, xmm5
    movdqa  [t0], xmm4
    ADS_END 4
%endmacro

ADS_SSE2 sse2
%define ABS1 ABS1_SSSE3
ADS_SSE2 ssse3

; int x264_pixel_ads_mvs( int16_t *mvs, uint8_t *masks, int width )
; {
;     int nmv=0, i, j;
;     *(uint32_t*)(masks+width) = 0;
;     for( i=0; i<width; i+=8 )
;     {
;         uint64_t mask = *(uint64_t*)(masks+i);
;         if( !mask ) continue;
;         for( j=0; j<8; j++ )
;             if( mask & (255<<j*8) )
;                 mvs[nmv++] = i+j;
;     }
;     return nmv;
; }
cglobal x264_pixel_ads_mvs
ads_mvs:
    xor     eax, eax
    xor     esi, esi
%ifdef ARCH_X86_64
    ; mvs = r4
    ; masks = rsp
    ; width = r5
    ; clear last block in case width isn't divisible by 8. (assume divisible by 4, so clearing 4 bytes is enough.)
    mov     dword [rsp+r5], 0
    jmp .loopi
.loopi0:
    add     esi, 8
    cmp     esi, r5d
    jge .end
.loopi:
    mov     rdi, [rsp+rsi]
    test    rdi, rdi
    jz .loopi0
    xor     ecx, ecx
%macro TEST 1
    mov     [r4+rax*2], si
    test    edi, 0xff<<(%1*8)
    setne   cl
    add     eax, ecx
    inc     esi
%endmacro
    TEST 0
    TEST 1
    TEST 2
    TEST 3
    shr     rdi, 32
    TEST 0
    TEST 1
    TEST 2
    TEST 3
    cmp     esi, r5d
    jl .loopi
.end:
    mov     rsp, r10
    ret

%else
    ; no PROLOGUE, inherit from x264_pixel_ads1
    mov     ebx, [ebp+stack_offset+20] ; mvs
    mov     edi, [ebp+stack_offset+24] ; width
    mov     dword [esp+edi], 0
    push    ebp
    jmp .loopi
.loopi0:
    add     esi, 8
    cmp     esi, edi
    jge .end
.loopi:
    mov     ebp, [esp+esi+4]
    mov     edx, [esp+esi+8]
    mov     ecx, ebp
    or      ecx, edx
    jz .loopi0
    xor     ecx, ecx
%macro TEST 2
    mov     [ebx+eax*2], si
    test    %2, 0xff<<(%1*8)
    setne   cl
    add     eax, ecx
    inc     esi
%endmacro
    TEST 0, ebp
    TEST 1, ebp
    TEST 2, ebp
    TEST 3, ebp
    TEST 0, edx
    TEST 1, edx
    TEST 2, edx
    TEST 3, edx
    cmp     esi, edi
    jl .loopi
.end:
    pop     esp
    RET
%endif ; ARCH