// Copyright 2005-2010 Intel Corporation.  All Rights Reserved.
//
// This file is part of Threading Building Blocks.
//
// Threading Building Blocks is free software; you can redistribute it
// and/or modify it under the terms of the GNU General Public License
// version 2 as published by the Free Software Foundation.
//
// Threading Building Blocks is distributed in the hope that it will be
// useful, but WITHOUT ANY WARRANTY; without even the implied warranty
// of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with Threading Building Blocks; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
//
// As a special exception, you may use this file as part of a free software
// library without restriction.  Specifically, if other files instantiate
// templates or use macros or inline functions from this file, or you compile
// this file and link it with other files to produce an executable, this
// file does not by itself cause the resulting executable to be covered by
// the GNU General Public License.  This exception does not however
// invalidate any other reasons why the executable file might be covered by
// the GNU General Public License.

// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh
# 1 "<stdin>"
# 1 "<built-in>"
# 1 "<command line>"
# 1 "<stdin>"


        .section .text
        .align 16


        .proc __TBB_machine_fetchadd1__TBB_full_fence#
        .global __TBB_machine_fetchadd1__TBB_full_fence#
__TBB_machine_fetchadd1__TBB_full_fence:
{
        mf
        br __TBB_machine_fetchadd1acquire
}
        .endp __TBB_machine_fetchadd1__TBB_full_fence#

        .proc __TBB_machine_fetchadd1acquire#
        .global __TBB_machine_fetchadd1acquire#
__TBB_machine_fetchadd1acquire:


        ld1 r9=[r32]
;;
Retry_1acquire:
        mov ar.ccv=r9
        mov r8=r9;
        add r10=r9,r33
;;
        cmpxchg1.acq r9=[r32],r10,ar.ccv
;;
        cmp.ne p7,p0=r8,r9
  (p7) br.cond.dpnt Retry_1acquire
        br.ret.sptk.many b0
# 49 "<stdin>"
        .endp __TBB_machine_fetchadd1acquire#
# 62 "<stdin>"
        .section .text
        .align 16
        .proc __TBB_machine_fetchstore1__TBB_full_fence#
        .global __TBB_machine_fetchstore1__TBB_full_fence#
__TBB_machine_fetchstore1__TBB_full_fence:
        mf
;;
        xchg1 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore1__TBB_full_fence#


        .proc __TBB_machine_fetchstore1acquire#
        .global __TBB_machine_fetchstore1acquire#
__TBB_machine_fetchstore1acquire:
        xchg1 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore1acquire#
# 88 "<stdin>"
        .section .text
        .align 16


        .proc __TBB_machine_cmpswp1__TBB_full_fence#
        .global __TBB_machine_cmpswp1__TBB_full_fence#
__TBB_machine_cmpswp1__TBB_full_fence:
{
        mf
        br __TBB_machine_cmpswp1acquire
}
        .endp __TBB_machine_cmpswp1__TBB_full_fence#

        .proc __TBB_machine_cmpswp1acquire#
        .global __TBB_machine_cmpswp1acquire#
__TBB_machine_cmpswp1acquire:

        zxt1 r34=r34
;;

        mov ar.ccv=r34
;;
        cmpxchg1.acq r8=[r32],r33,ar.ccv
        br.ret.sptk.many b0
        .endp __TBB_machine_cmpswp1acquire#
// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh
# 1 "<stdin>"
# 1 "<built-in>"
# 1 "<command line>"
# 1 "<stdin>"


        .section .text
        .align 16


        .proc __TBB_machine_fetchadd2__TBB_full_fence#
        .global __TBB_machine_fetchadd2__TBB_full_fence#
__TBB_machine_fetchadd2__TBB_full_fence:
{
        mf
        br __TBB_machine_fetchadd2acquire
}
        .endp __TBB_machine_fetchadd2__TBB_full_fence#

        .proc __TBB_machine_fetchadd2acquire#
        .global __TBB_machine_fetchadd2acquire#
__TBB_machine_fetchadd2acquire:


        ld2 r9=[r32]
;;
Retry_2acquire:
        mov ar.ccv=r9
        mov r8=r9;
        add r10=r9,r33
;;
        cmpxchg2.acq r9=[r32],r10,ar.ccv
;;
        cmp.ne p7,p0=r8,r9
  (p7) br.cond.dpnt Retry_2acquire
        br.ret.sptk.many b0
# 49 "<stdin>"
        .endp __TBB_machine_fetchadd2acquire#
# 62 "<stdin>"
        .section .text
        .align 16
        .proc __TBB_machine_fetchstore2__TBB_full_fence#
        .global __TBB_machine_fetchstore2__TBB_full_fence#
__TBB_machine_fetchstore2__TBB_full_fence:
        mf
;;
        xchg2 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore2__TBB_full_fence#


        .proc __TBB_machine_fetchstore2acquire#
        .global __TBB_machine_fetchstore2acquire#
__TBB_machine_fetchstore2acquire:
        xchg2 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore2acquire#
# 88 "<stdin>"
        .section .text
        .align 16


        .proc __TBB_machine_cmpswp2__TBB_full_fence#
        .global __TBB_machine_cmpswp2__TBB_full_fence#
__TBB_machine_cmpswp2__TBB_full_fence:
{
        mf
        br __TBB_machine_cmpswp2acquire
}
        .endp __TBB_machine_cmpswp2__TBB_full_fence#

        .proc __TBB_machine_cmpswp2acquire#
        .global __TBB_machine_cmpswp2acquire#
__TBB_machine_cmpswp2acquire:

        zxt2 r34=r34
;;

        mov ar.ccv=r34
;;
        cmpxchg2.acq r8=[r32],r33,ar.ccv
        br.ret.sptk.many b0
        .endp __TBB_machine_cmpswp2acquire#
// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh
# 1 "<stdin>"
# 1 "<built-in>"
# 1 "<command line>"
# 1 "<stdin>"


        .section .text
        .align 16


        .proc __TBB_machine_fetchadd4__TBB_full_fence#
        .global __TBB_machine_fetchadd4__TBB_full_fence#
__TBB_machine_fetchadd4__TBB_full_fence:
{
        mf
        br __TBB_machine_fetchadd4acquire
}
        .endp __TBB_machine_fetchadd4__TBB_full_fence#

        .proc __TBB_machine_fetchadd4acquire#
        .global __TBB_machine_fetchadd4acquire#
__TBB_machine_fetchadd4acquire:

        cmp.eq p6,p0=1,r33
        cmp.eq p8,p0=-1,r33
  (p6) br.cond.dptk Inc_4acquire
  (p8) br.cond.dpnt Dec_4acquire
;;

        ld4 r9=[r32]
;;
Retry_4acquire:
        mov ar.ccv=r9
        mov r8=r9;
        add r10=r9,r33
;;
        cmpxchg4.acq r9=[r32],r10,ar.ccv
;;
        cmp.ne p7,p0=r8,r9
  (p7) br.cond.dpnt Retry_4acquire
        br.ret.sptk.many b0

Inc_4acquire:
        fetchadd4.acq r8=[r32],1
        br.ret.sptk.many b0
Dec_4acquire:
        fetchadd4.acq r8=[r32],-1
        br.ret.sptk.many b0

        .endp __TBB_machine_fetchadd4acquire#
# 62 "<stdin>"
        .section .text
        .align 16
        .proc __TBB_machine_fetchstore4__TBB_full_fence#
        .global __TBB_machine_fetchstore4__TBB_full_fence#
__TBB_machine_fetchstore4__TBB_full_fence:
        mf
;;
        xchg4 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore4__TBB_full_fence#


        .proc __TBB_machine_fetchstore4acquire#
        .global __TBB_machine_fetchstore4acquire#
__TBB_machine_fetchstore4acquire:
        xchg4 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore4acquire#
# 88 "<stdin>"
        .section .text
        .align 16


        .proc __TBB_machine_cmpswp4__TBB_full_fence#
        .global __TBB_machine_cmpswp4__TBB_full_fence#
__TBB_machine_cmpswp4__TBB_full_fence:
{
        mf
        br __TBB_machine_cmpswp4acquire
}
        .endp __TBB_machine_cmpswp4__TBB_full_fence#

        .proc __TBB_machine_cmpswp4acquire#
        .global __TBB_machine_cmpswp4acquire#
__TBB_machine_cmpswp4acquire:

        zxt4 r34=r34
;;

        mov ar.ccv=r34
;;
        cmpxchg4.acq r8=[r32],r33,ar.ccv
        br.ret.sptk.many b0
        .endp __TBB_machine_cmpswp4acquire#
// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh
# 1 "<stdin>"
# 1 "<built-in>"
# 1 "<command line>"
# 1 "<stdin>"


        .section .text
        .align 16


        .proc __TBB_machine_fetchadd8__TBB_full_fence#
        .global __TBB_machine_fetchadd8__TBB_full_fence#
__TBB_machine_fetchadd8__TBB_full_fence:
{
        mf
        br __TBB_machine_fetchadd8acquire
}
        .endp __TBB_machine_fetchadd8__TBB_full_fence#

        .proc __TBB_machine_fetchadd8acquire#
        .global __TBB_machine_fetchadd8acquire#
__TBB_machine_fetchadd8acquire:

        cmp.eq p6,p0=1,r33
        cmp.eq p8,p0=-1,r33
  (p6) br.cond.dptk Inc_8acquire
  (p8) br.cond.dpnt Dec_8acquire
;;

        ld8 r9=[r32]
;;
Retry_8acquire:
        mov ar.ccv=r9
        mov r8=r9;
        add r10=r9,r33
;;
        cmpxchg8.acq r9=[r32],r10,ar.ccv
;;
        cmp.ne p7,p0=r8,r9
  (p7) br.cond.dpnt Retry_8acquire
        br.ret.sptk.many b0

Inc_8acquire:
        fetchadd8.acq r8=[r32],1
        br.ret.sptk.many b0
Dec_8acquire:
        fetchadd8.acq r8=[r32],-1
        br.ret.sptk.many b0

        .endp __TBB_machine_fetchadd8acquire#
# 62 "<stdin>"
        .section .text
        .align 16
        .proc __TBB_machine_fetchstore8__TBB_full_fence#
        .global __TBB_machine_fetchstore8__TBB_full_fence#
__TBB_machine_fetchstore8__TBB_full_fence:
        mf
;;
        xchg8 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore8__TBB_full_fence#


        .proc __TBB_machine_fetchstore8acquire#
        .global __TBB_machine_fetchstore8acquire#
__TBB_machine_fetchstore8acquire:
        xchg8 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore8acquire#
# 88 "<stdin>"
        .section .text
        .align 16


        .proc __TBB_machine_cmpswp8__TBB_full_fence#
        .global __TBB_machine_cmpswp8__TBB_full_fence#
__TBB_machine_cmpswp8__TBB_full_fence:
{
        mf
        br __TBB_machine_cmpswp8acquire
}
        .endp __TBB_machine_cmpswp8__TBB_full_fence#

        .proc __TBB_machine_cmpswp8acquire#
        .global __TBB_machine_cmpswp8acquire#
__TBB_machine_cmpswp8acquire:


        mov ar.ccv=r34
;;
        cmpxchg8.acq r8=[r32],r33,ar.ccv
        br.ret.sptk.many b0
        .endp __TBB_machine_cmpswp8acquire#
// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh
# 1 "<stdin>"
# 1 "<built-in>"
# 1 "<command line>"
# 1 "<stdin>"


        .section .text
        .align 16
# 19 "<stdin>"
        .proc __TBB_machine_fetchadd1release#
        .global __TBB_machine_fetchadd1release#
__TBB_machine_fetchadd1release:


        ld1 r9=[r32]
;;
Retry_1release:
        mov ar.ccv=r9
        mov r8=r9;
        add r10=r9,r33
;;
        cmpxchg1.rel r9=[r32],r10,ar.ccv
;;
        cmp.ne p7,p0=r8,r9
  (p7) br.cond.dpnt Retry_1release
        br.ret.sptk.many b0
# 49 "<stdin>"
        .endp __TBB_machine_fetchadd1release#
# 62 "<stdin>"
        .section .text
        .align 16
        .proc __TBB_machine_fetchstore1release#
        .global __TBB_machine_fetchstore1release#
__TBB_machine_fetchstore1release:
        mf
;;
        xchg1 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore1release#
# 88 "<stdin>"
        .section .text
        .align 16
# 101 "<stdin>"
        .proc __TBB_machine_cmpswp1release#
        .global __TBB_machine_cmpswp1release#
__TBB_machine_cmpswp1release:

        zxt1 r34=r34
;;

        mov ar.ccv=r34
;;
        cmpxchg1.rel r8=[r32],r33,ar.ccv
        br.ret.sptk.many b0
        .endp __TBB_machine_cmpswp1release#
// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh
# 1 "<stdin>"
# 1 "<built-in>"
# 1 "<command line>"
# 1 "<stdin>"


        .section .text
        .align 16
# 19 "<stdin>"
        .proc __TBB_machine_fetchadd2release#
        .global __TBB_machine_fetchadd2release#
__TBB_machine_fetchadd2release:


        ld2 r9=[r32]
;;
Retry_2release:
        mov ar.ccv=r9
        mov r8=r9;
        add r10=r9,r33
;;
        cmpxchg2.rel r9=[r32],r10,ar.ccv
;;
        cmp.ne p7,p0=r8,r9
  (p7) br.cond.dpnt Retry_2release
        br.ret.sptk.many b0
# 49 "<stdin>"
        .endp __TBB_machine_fetchadd2release#
# 62 "<stdin>"
        .section .text
        .align 16
        .proc __TBB_machine_fetchstore2release#
        .global __TBB_machine_fetchstore2release#
__TBB_machine_fetchstore2release:
        mf
;;
        xchg2 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore2release#
# 88 "<stdin>"
        .section .text
        .align 16
# 101 "<stdin>"
        .proc __TBB_machine_cmpswp2release#
        .global __TBB_machine_cmpswp2release#
__TBB_machine_cmpswp2release:

        zxt2 r34=r34
;;

        mov ar.ccv=r34
;;
        cmpxchg2.rel r8=[r32],r33,ar.ccv
        br.ret.sptk.many b0
        .endp __TBB_machine_cmpswp2release#
// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh
# 1 "<stdin>"
# 1 "<built-in>"
# 1 "<command line>"
# 1 "<stdin>"


        .section .text
        .align 16
# 19 "<stdin>"
        .proc __TBB_machine_fetchadd4release#
        .global __TBB_machine_fetchadd4release#
__TBB_machine_fetchadd4release:

        cmp.eq p6,p0=1,r33
        cmp.eq p8,p0=-1,r33
  (p6) br.cond.dptk Inc_4release
  (p8) br.cond.dpnt Dec_4release
;;

        ld4 r9=[r32]
;;
Retry_4release:
        mov ar.ccv=r9
        mov r8=r9;
        add r10=r9,r33
;;
        cmpxchg4.rel r9=[r32],r10,ar.ccv
;;
        cmp.ne p7,p0=r8,r9
  (p7) br.cond.dpnt Retry_4release
        br.ret.sptk.many b0

Inc_4release:
        fetchadd4.rel r8=[r32],1
        br.ret.sptk.many b0
Dec_4release:
        fetchadd4.rel r8=[r32],-1
        br.ret.sptk.many b0

        .endp __TBB_machine_fetchadd4release#
# 62 "<stdin>"
        .section .text
        .align 16
        .proc __TBB_machine_fetchstore4release#
        .global __TBB_machine_fetchstore4release#
__TBB_machine_fetchstore4release:
        mf
;;
        xchg4 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore4release#
# 88 "<stdin>"
        .section .text
        .align 16
# 101 "<stdin>"
        .proc __TBB_machine_cmpswp4release#
        .global __TBB_machine_cmpswp4release#
__TBB_machine_cmpswp4release:

        zxt4 r34=r34
;;

        mov ar.ccv=r34
;;
        cmpxchg4.rel r8=[r32],r33,ar.ccv
        br.ret.sptk.many b0
        .endp __TBB_machine_cmpswp4release#
// DO NOT EDIT - AUTOMATICALLY GENERATED FROM tools/generate_atomic/ipf_generate.sh
# 1 "<stdin>"
# 1 "<built-in>"
# 1 "<command line>"
# 1 "<stdin>"


        .section .text
        .align 16
# 19 "<stdin>"
        .proc __TBB_machine_fetchadd8release#
        .global __TBB_machine_fetchadd8release#
__TBB_machine_fetchadd8release:

        cmp.eq p6,p0=1,r33
        cmp.eq p8,p0=-1,r33
  (p6) br.cond.dptk Inc_8release
  (p8) br.cond.dpnt Dec_8release
;;

        ld8 r9=[r32]
;;
Retry_8release:
        mov ar.ccv=r9
        mov r8=r9;
        add r10=r9,r33
;;
        cmpxchg8.rel r9=[r32],r10,ar.ccv
;;
        cmp.ne p7,p0=r8,r9
  (p7) br.cond.dpnt Retry_8release
        br.ret.sptk.many b0

Inc_8release:
        fetchadd8.rel r8=[r32],1
        br.ret.sptk.many b0
Dec_8release:
        fetchadd8.rel r8=[r32],-1
        br.ret.sptk.many b0

        .endp __TBB_machine_fetchadd8release#
# 62 "<stdin>"
        .section .text
        .align 16
        .proc __TBB_machine_fetchstore8release#
        .global __TBB_machine_fetchstore8release#
__TBB_machine_fetchstore8release:
        mf
;;
        xchg8 r8=[r32],r33
        br.ret.sptk.many b0
        .endp __TBB_machine_fetchstore8release#
# 88 "<stdin>"
        .section .text
        .align 16
# 101 "<stdin>"
        .proc __TBB_machine_cmpswp8release#
        .global __TBB_machine_cmpswp8release#
__TBB_machine_cmpswp8release:


        mov ar.ccv=r34
;;
        cmpxchg8.rel r8=[r32],r33,ar.ccv
        br.ret.sptk.many b0
        .endp __TBB_machine_cmpswp8release#