<|start_filename|>src/openpa/src/primitives/opa_gcc_intel_32_64_p3barrier.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef OPA_GCC_INTEL_32_64_P3BARRIER_H_INCLUDED
#define OPA_GCC_INTEL_32_64_P3BARRIER_H_INCLUDED

#define OPA_compiler_barrier() __asm__ __volatile__ ( "" ::: "memory" )

/* For all regular memory (write-back cacheable, not driver/graphics
 * memory), there is only one general ordering relaxation permitted by
 * x86/x86_64 processors: earlier stores may be retired after later
 * stores.  The "clflush" and "movnt*" instructions also don't follow
 * general ordering constraints, although any code using these
 * instructions should be responsible for ensuring proper ordering
 * itself.  So our read and write barriers may be implemented as simple
 * compiler barriers. */
#define OPA_write_barrier() OPA_compiler_barrier()
#define OPA_read_barrier()  OPA_compiler_barrier()

/* Some Pentium III and earlier processors have store barriers
   (sfence), but no full barrier or load barriers.  Some other very
   recent x86-like processors don't seem to have sfence either.  A
   lock-prefixed atomic instruction (operating on any memory) behaves as
   a full memory barrier, so we use that here. */
static inline void OPA_read_write_barrier(void)
{
    int a = 0;
    __asm__ __volatile__ ("lock orl $0, %0" : "+m" (a) : /*no outputs*/ : "memory");
}

#endif /* OPA_GCC_INTEL_32_64_P3BARRIER_H_INCLUDED */


<|start_filename|>src/openpa/src/primitives/opa_gcc_sicortex.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

/* Atomic primitives for SiCortex machines.
 * Originally contributed by <NAME> at SiCortex.
 */

#ifndef OPA_GCC_SICORTEX_H
#define OPA_GCC_SICORTEX_H

/* FIXME do these need alignment? */
typedef struct { volatile int v;  } OPA_int_t;
typedef struct { int * volatile v; } OPA_ptr_t;

#define OPA_INT_T_INITIALIZER(val_) { (val_) }
#define OPA_PTR_T_INITIALIZER(val_) { (val_) }

#define OPA_write_barrier()      __asm__ __volatile__  ("sync" ::: "memory" )
#define OPA_read_barrier()       __asm__ __volatile__  ("sync" ::: "memory" )
#define OPA_read_write_barrier() __asm__ __volatile__  ("sync" ::: "memory" )
#define OPA_compiler_barrier()   __asm__ __volatile__  ( ""    ::: "memory" )

/* Aligned loads and stores are atomic. */
static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr)
{
    return ptr->v;
}

/* Aligned loads and stores are atomic. */
static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val)
{
    ptr->v = val;
}

/* Aligned loads and stores are atomic. */
static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr)
{
    return ptr->v;
}

/* Aligned loads and stores are atomic. */
static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val)
{
    ptr->v = val;
}

/* NOTE: these acquire/release operations have not been optimized, I just threw
 * down a full memory barrier.  Spending much time working on the SiCortex platform
 * doesn't really make a lot of sense since there are so few machines in
 * existence and no more will ever be built. */
static _opa_inline int OPA_load_acquire_int(_opa_const OPA_int_t *ptr)
{
    int tmp;
    tmp = ptr->v;
    OPA_read_write_barrier();
    return tmp;
}

static _opa_inline void OPA_store_release_int(OPA_int_t *ptr, int val)
{
    OPA_read_write_barrier();
    ptr->v = val;
}

static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr)
{
    void *tmp;
    tmp = ptr->v;
    OPA_read_write_barrier();
    return tmp;
}

static _opa_inline void OPA_store_release_ptr(OPA_ptr_t *ptr, void *val)
{
    OPA_read_write_barrier();
    ptr->v = val;
}

#include <stdint.h>

/* ICE9 rev A1 chips have a low-frequency bug that causes LL to
   fail. The workaround is to do the LL twice to make sure the data
   is in L1

   very few systems are affected

   FIXME We should either remove the workaround entirely or make it
   configurable/autodetected somehow. [goodell@ 2008/01/06]
 */
#define ICE9A_LLSC_WAR 0

/* For a description of the _opa_inline assembly constraints, see the MIPS section of:
   http://gcc.gnu.org/onlinedocs/gcc-4.3.2/gcc/Machine-Constraints.html#Machine-Constraints

   relevant excerpt:
       I - A signed 16-bit constant (for arithmetic instructions).
       J - Integer zero.

   Other _opa_inline asm knowledge worth remembering:
       r - a general register operand is allowed
       m - a memory address operand
       & - earlyclobber; operand is modified before instruction is finished using the input operands
       = - this operand is write-only

   general format:
       asm volatile ("instructions" : [outputs] : [inputs] : [clobbered_regs]);
       "memory" should be included as a clobbered reg for most of these operations.
 */

/* Atomic increment of a 32 bit value, returning the old value */
static _opa_inline int OPA_shmemi_fetch_add_4(volatile int * v, int inc)
{
        unsigned long result;
        if (ICE9A_LLSC_WAR) {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     ll      %0, %2          # fetch_add_4           \n"
                "       ll      %0, %2          # fetch_add_4           \n"
                "       addu    %1, %0, %3                              \n"
                "       sc      %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "Ir" (inc)
                : "memory");
        } else {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     ll      %0, %2          # fetch_add_4           \n"
                "       addu    %1, %0, %3                              \n"
                "       sc      %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "Ir" (inc)
                : "memory");
        }

        return result;
}

/* Atomic increment of a 64 bit value, returning the old value */
static _opa_inline long int OPA_shmemi_fetch_add_8(volatile long int * v, long int inc)
{
        unsigned long result;
        if (ICE9A_LLSC_WAR) {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     lld     %0, %2          # fetch_add_8           \n"
                "       lld     %0, %2          # fetch_add_8           \n"
                "       daddu   %1, %0, %3                              \n"
                "       scd     %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "Ir" (inc)
                : "memory");
        } else {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     lld     %0, %2          # fetch_add_8           \n"
                "       daddu   %1, %0, %3                              \n"
                "       scd     %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "Ir" (inc)
                : "memory");
        }

        return result;
}

/* Atomic swap of a 32 bit value, returning the old contents */
static _opa_inline int OPA_shmemi_swap_4(volatile int * v, int val)
{
        unsigned long result;
        if (ICE9A_LLSC_WAR) {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     ll      %0, %2          # swap_4                \n"
                "       ll      %0, %2          # swap_4                \n"
                "       move    %1, %3                                  \n"
                "       sc      %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "r" (val)
                : "memory");
        } else {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     ll      %0, %2          # swap_4                \n"
                "       move    %1, %3                                  \n"
                "       sc      %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "r" (val)
                : "memory");
        }

        return result;
}

/* Atomic swap of a 64 bit value, returning the old contents */
static _opa_inline long int OPA_shmemi_swap_8(volatile long int * v, long int val)
{
        unsigned long result;
        if (ICE9A_LLSC_WAR) {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     lld     %0, %2          # swap_8                \n"
                "       lld     %0, %2          # swap_8                \n"
                "       move    %1, %3                                  \n"
                "       scd     %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "r" (val)
                : "memory");
        } else {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     lld     %0, %2          # swap_8                \n"
                "       move    %1, %3                                  \n"
                "       scd     %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "r" (val)
                : "memory");
        }

        return result;
}

/* Atomic compare and swap of a 32 bit value, returns the old value
 * but only does the store of the val value if the old value == expect */
static _opa_inline int OPA_shmemi_cswap_4(volatile int * v, int expect, int val)
{
        unsigned long result;
        if (ICE9A_LLSC_WAR) {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     ll      %0, %2          # cswap_4               \n"
                "       ll      %0, %2          # cswap_4               \n"
                "       bne     %0, %4, 1f                              \n"
                "       move    %1, %3                                  \n"
                "       sc      %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "1:                                                     \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "r" (val), "Jr" (expect)
                : "memory");
        } else {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     ll      %0, %2          # cswap_4               \n"
                "       bne     %0, %4, 1f                              \n"
                "       move    %1, %3                                  \n"
                "       sc      %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "1:                                                     \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "r" (val), "Jr" (expect)
                : "memory");
        }

        return result;
}

/* Atomic compare and swap of a 64 bit value, returns the old value
 * but only does the store of the val value if the old value == expect */
static _opa_inline long int OPA_shmemi_cswap_8(volatile long int * v, long int expect, long int val)
{
        unsigned long result;
        if (ICE9A_LLSC_WAR) {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     lld     %0, %2          # cswap_8               \n"
                "       lld     %0, %2          # cswap_8               \n"
                "       bne     %0, %4, 1f                              \n"
                "       move    %1, %3                                  \n"
                "       scd     %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "1:                                                     \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "r" (val), "Jr" (expect)
                : "memory");
        } else {
                unsigned long temp;

                __asm__ __volatile__(
                "       .set    mips3                                   \n"
                "       .set    noreorder                               \n"
                "1:     lld     %0, %2          # cswap_8               \n"
                "       bne     %0, %4, 1f                              \n"
                "       move    %1, %3                                  \n"
                "       scd     %1, %2                                  \n"
                "       beqz    %1, 1b                                  \n"
                "        nop                                            \n"
                "       .set    reorder                                 \n"
                "1:                                                     \n"
                "       .set    mips0                                   \n"
                : "=&r" (result), "=&r" (temp), "=m" (*v)
                : "r" (val), "Jr" (expect)
                : "memory");
        }

        return result;
}

static _opa_inline void OPA_add_int(OPA_int_t *ptr, int val)
{
    OPA_shmemi_fetch_add_4(&ptr->v, val);
}

static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv)
{
#if (OPA_SIZEOF_VOID_P == 8)
    return((int *) OPA_shmemi_cswap_8((volatile long int *) &ptr->v, (uintptr_t) oldv, (uintptr_t) newv));
#elif (OPA_SIZEOF_VOID_P == 4)
    return((int *) OPA_shmemi_cswap_4((volatile int *) &ptr->v, (uintptr_t) oldv, (uintptr_t) newv));
#else
#error "OPA_SIZEOF_VOID_P has an unexpected value :" OPA_QUOTE(OPA_SIZEOF_VOID_P);
#endif
}

static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv)
{
    return(OPA_shmemi_cswap_4(&ptr->v, oldv, newv));
}

static _opa_inline void OPA_decr_int(OPA_int_t *ptr)
{
    OPA_shmemi_fetch_add_4(&ptr->v, -1);
}

static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr)
{
    int old = OPA_shmemi_fetch_add_4(&ptr->v, -1);
    return (old == 1);
}

static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val)
{
    return(OPA_shmemi_fetch_add_4(&ptr->v, val));
}

static _opa_inline int OPA_fetch_and_decr_int(OPA_int_t *ptr)
{
    return(OPA_shmemi_fetch_add_4(&ptr->v, -1));
}

static _opa_inline int OPA_fetch_and_incr_int(OPA_int_t *ptr)
{
    return(OPA_shmemi_fetch_add_4(&ptr->v, 1));
}

static _opa_inline void OPA_incr_int(OPA_int_t *ptr)
{
    OPA_shmemi_fetch_add_4(&ptr->v, 1);
}

static _opa_inline int *OPA_swap_ptr(OPA_ptr_t *ptr, int *val)
{
#if (OPA_SIZEOF_VOID_P == 8)
    return((int *) OPA_shmemi_swap_8((volatile long int *) &ptr->v, (uintptr_t) val));
#elif (OPA_SIZEOF_VOID_P == 4)
    return((int *) OPA_shmemi_swap_4((volatile int *) &ptr->v, (uintptr_t) val));
#else
#error "OPA_SIZEOF_VOID_P has an unexpected value :" OPA_QUOTE(OPA_SIZEOF_VOID_P);
#endif
}

static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val)
{
    return(OPA_shmemi_swap_4(&ptr->v, val));
}

#endif /* OPA_GCC_SICORTEX_H */


<|start_filename|>src/openpa/src/primitives/opa_gcc_intel_32_64_p3.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef OPA_GCC_INTEL_32_64_P3_H_INCLUDED
#define OPA_GCC_INTEL_32_64_P3_H_INCLUDED

#include "opa_gcc_intel_32_64_p3barrier.h"
#include "opa_gcc_intel_32_64_ops.h"

#include "opa_emulated.h"

#endif /* OPA_GCC_INTEL_32_64_P3_H_INCLUDED */


<|start_filename|>src/openpa/src/opa_primitives.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*  
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef OPA_PRIMITIVES_H_INCLUDED
#define OPA_PRIMITIVES_H_INCLUDED

#include "opa_config.h"
#include "opa_util.h"

/* Clean up some of the opa_config.h definitions.  This is a consequence
   of using the AX_PREFIX_CONFIG_H macro. Autoconf won't define inline
   or _opa_inline when a real "inline" works.  Since we are
   unconditionally using _opa_inline we must define it ourselves in this
   case. */
#ifndef _opa_inline
#define _opa_inline inline
#endif
#ifndef _opa_restrict
#define _opa_restrict restrict
#endif
#ifndef _opa_const
#define _opa_const const
#endif

/*
   Primitive atomic functions
   --------------------------

   The included file is responsible for defining the types of OPA_int_t and
   OPA_ptr_t as well as a set of functions for operating on these
   types.  If you have the following declaration:

       OPA_int_t atomic_var;

   Then in order for the emulation functions to compile, the underlying value of
   atomic_var should be accessible via:

       atomic_var.v;

   The same goes for OPA_ptr_t.

   The atomic functions that must be ported for each architecture: 

   static _opa_inline int   OPA_load_int(_opa_const OPA_int_t *ptr);
   static _opa_inline void  OPA_store_int(OPA_int_t *ptr, int val);
   static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr);
   static _opa_inline void  OPA_store_ptr(OPA_ptr_t *ptr, void *val);

   static _opa_inline void OPA_add_int(OPA_int_t *ptr, int val);
   static _opa_inline void OPA_incr_int(OPA_int_t *ptr);
   static _opa_inline void OPA_decr_int(OPA_int_t *ptr);

   static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr);
   static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val);
   static _opa_inline int OPA_fetch_and_incr_int(OPA_int_t *ptr);
   static _opa_inline int OPA_fetch_and_decr_int(OPA_int_t *ptr);

   static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv);
   static _opa_inline int   OPA_cas_int(OPA_int_t *ptr, int oldv, int newv);

   static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val);
   static _opa_inline int   OPA_swap_int(OPA_int_t *ptr, int val);

   // (the memory barriers may be macros instead of inline functions)
   static _opa_inline void OPA_write_barrier();
   static _opa_inline void OPA_read_barrier();
   static _opa_inline void OPA_read_write_barrier();

   // Loads and stores with memory ordering guarantees (also may be macros):
   static _opa_inline int   OPA_load_acquire_int(_opa_const OPA_int_t *ptr);
   static _opa_inline void  OPA_store_release_int(OPA_int_t *ptr, int val);
   static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr);
   static _opa_inline void  OPA_store_release_ptr(OPA_ptr_t *ptr, void *val);

   // Compiler barrier, only preventing compiler reordering, *not* CPU
   // reordering (may be a macro):
   static _opa_inline void OPA_compiler_barrier();

   // The following need to be ported only for architectures supporting LL/SC:
   static _opa_inline int OPA_LL_int(OPA_int_t *ptr);
   static _opa_inline int OPA_SC_int(OPA_int_t *ptr, int val);
   static _opa_inline void *OPA_LL_ptr(OPA_ptr_t *ptr);
   static _opa_inline int OPA_SC_ptr(OPA_ptr_t *ptr, void *val);

   // Additionally, the following initializer macros must be defined:
   #define OPA_INT_T_INITIALIZER(val_) ...
   #define OPA_PTR_T_INITIALIZER(val_) ...

   // They should be useable as C89 static initializers like so:

   struct { int x; OPA_int_t y; OPA_ptr_t z; } foo = { 35, OPA_INT_T_INITIALIZER(1), OPA_PTR_T_INITIALIZER(NULL) };
*/

/* Include the appropriate header for the architecture */
#if defined(OPA_USE_UNSAFE_PRIMITIVES)
/* comes first to permit user overrides in the style of NDEBUG */
#include "primitives/opa_unsafe.h"
#elif   defined(OPA_HAVE_GCC_AND_POWERPC_ASM)
#include "primitives/opa_gcc_ppc.h"
#elif   defined(OPA_HAVE_GCC_AND_ARM_ASM)
#include "primitives/opa_gcc_arm.h"
#elif defined(OPA_HAVE_GCC_X86_32_64)
#include "primitives/opa_gcc_intel_32_64.h"
#elif defined(OPA_HAVE_GCC_X86_32_64_P3)
#include "primitives/opa_gcc_intel_32_64_p3.h"
#elif defined(OPA_HAVE_GCC_AND_IA64_ASM)
#include "primitives/opa_gcc_ia64.h"
#elif defined(OPA_HAVE_GCC_AND_SICORTEX_ASM)
#include "primitives/opa_gcc_sicortex.h"
#elif defined(OPA_HAVE_GCC_INTRINSIC_ATOMICS)
#include "primitives/opa_gcc_intrinsics.h"
#elif defined(OPA_HAVE_SUN_ATOMIC_OPS)
#include "primitives/opa_sun_atomic_ops.h"
#elif defined(OPA_HAVE_NT_INTRINSICS)
#include "primitives/opa_nt_intrinsics.h"
#elif defined(OPA_USE_LOCK_BASED_PRIMITIVES)
#include "primitives/opa_by_lock.h"
#else
#error no primitives implementation specified
#endif

/*
    This routine is needed because the MPIU_THREAD_XXX_CS_{ENTER,EXIT} macros do
    not provide synchronization across multiple processes, only across multiple
    threads within a process.  In order to safely emulate atomic operations on a
    shared memory region, we need a shared memory backed lock mechanism.

    This routine must be called by any subsystem that intends to use the atomic
    abstractions if the cpp directive OPA_USE_LOCK_BASED_PRIMITIVES is defined.  It must
    be called exactly once by _all_ processes, not just a single leader.  This
    function will initialize the contents of the lock variable if the caller
    specifies (isLeader==true).  Note that multiple initialization is forbidden
    by several lock implementations, especially pthreads.

    Inputs:
      shm_lock - A pointer to an allocated piece of shared memory that can hold
                 a mutex (e.g., pthread_mutex_t).  This is not portable to
                 non-pthreads systems at this time.
      isLeader - This boolean value should be set to true for exactly one
                 thread/process of the group that calls this function.
*/
#if defined(OPA_HAVE_PTHREAD_H)
#  include <pthread.h>
typedef pthread_mutex_t OPA_emulation_ipl_t;
int OPA_Interprocess_lock_init(OPA_emulation_ipl_t *shm_lock, int isLeader);
#endif


/* FIXME This should probably be pushed down into the platform-specific headers. */
#if defined(OPA_HAVE_SCHED_YIELD)
#  include <sched.h>
#  define OPA_busy_wait() sched_yield()
#else
#  define OPA_busy_wait() do { } while (0)
#endif

#endif /* defined(OPA_PRIMITIVES_H_INCLUDED) */


<|start_filename|>src/openpa/test/opa_test.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef OPA_TEST_NAIVE
#  include "opa_primitives.h"
#else /* OPA_TEST_NAIVE */
#  define OPA_PRIMITIVES_H_INCLUDED
#  include "opa_config.h"
#  include "opa_util.h"
#  ifndef _opa_inline
#    define _opa_inline inline
#  endif
#endif /* OPA_TEST_NAIVE */
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
#include <limits.h>
#include <time.h>
#if defined(OPA_HAVE_PTHREAD_H)
#  include <pthread.h>
#endif /* HAVE_PTHREAD_H */

/* Define the macro to use for yielding the current thread (to others) */
#if defined(OPA_HAVE_PTHREAD_YIELD)
#  define OPA_TEST_YIELD() pthread_yield()
#elif defined(OPA_HAVE_SCHED_YIELD)
#  include <sched.h>
#  define OPA_TEST_YIELD() (void) sched_yield()
#else
#  define OPA_TEST_YIELD() (void) 0
#endif

/*
 * Naive redefinition of OPA functions as simple C operations.  These should
 * pass all the "simple" tests, but fail on the threaded tests (with >1 thread),
 * unless the tested operation is natively atomic.
 */
#ifdef OPA_TEST_NAIVE

#define OPA_UNIVERSAL_PRIMITIVE OPA_CAS
#define OPA_LL_SC_SUPPORTED 1

typedef volatile int OPA_int_t;
typedef void * volatile OPA_ptr_t;

#define OPA_load_int(A) (*(A))
#define OPA_store_int(A, B) ((void) (*(A) = (B)))
#define OPA_load_ptr(A) (*(A))
#define OPA_store_ptr(A, B) ((void) (*(A) = (B)))

#define OPA_add_int(A, B) ((void) (*(A) += (B)))
#define OPA_incr_int(A) ((void) (++(*(A))))
#define OPA_decr_int(A) ((void) (--(*(A))))

#define OPA_decr_and_test_int(A) (0 == --(*(A)))
static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val)
{
    int prev = *ptr;
    *ptr += val;
    return prev;
}
#define OPA_fetch_and_incr_int(A) ((*(A))++)
#define OPA_fetch_and_decr_int(A) ((*(A))--)

#define OPA_cas_ptr(A, B, C) (*(A) == (B) ? (*(A) = (C), (B)) : *(A))
#define OPA_cas_int(A, B, C) (*(A) == (B) ? (*(A) = (C), (B)) : *(A))

static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val)
{
    void *prev;
    prev = *ptr;
    *ptr = val;
    return prev;
}

static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val)
{
    int prev;
    prev = *ptr;
    *ptr = val;
    return prev;
}

#define OPA_write_barrier() ((void) 0)
#define OPA_read_barrier() ((void) 0)
#define OPA_read_write_barrier() ((void) 0)

/* For LL/SC only use load/store.  This is more naive than the above
 * implementation for CAS */
#define OPA_LL_int OPA_load_int
#define OPA_SC_int(A, B) (OPA_store_int(A, B), 1)
#define OPA_LL_ptr OPA_load_ptr
#define OPA_SC_ptr(A, B) (OPA_store_ptr(A, B), 1)

#if defined(OPA_HAVE_SCHED_YIELD)
#  include <sched.h>
#  define OPA_busy_wait() sched_yield()
#else
#  define OPA_busy_wait() do { } while (0)
#endif

#endif /* OPA_TEST_NAIVE */

/*
 * Cache line padding.  See note in opa_queue.h.
 */
#define OPA_TEST_CACHELINE_PADDING 128

/*
 * Print the current location on the standard output stream.
 */
#define AT()            printf ("        at %s:%d in %s()...\n",              \
                                __FILE__, __LINE__, __FUNCTION__)

/*
 * The name of the test is printed by saying TESTING("something") which will
 * result in the string `Testing something' being flushed to standard output.
 * If a test passes, fails, or is skipped then the PASSED(), FAILED(), or
 * SKIPPED() macro should be called.  After HFAILED() or SKIPPED() the caller
 * should print additional information to stdout indented by at least four
 * spaces.
 */
#define TESTING(WHAT, NTHREADS)                                                \
do {                                                                           \
    if(NTHREADS) {                                                             \
        int nwritten_chars;                                                    \
        printf("Testing %s with %d thread%s %n",                               \
                WHAT,                                                          \
                (int) NTHREADS,                                                \
                (NTHREADS) > 1 ? "s" : "",                                     \
                &nwritten_chars);                                              \
        printf("%*s", nwritten_chars > 70 ? 0 : 70 - nwritten_chars, "");      \
    } /* end if */                                                             \
    else                                                                       \
        printf("Testing %-62s", WHAT);                                         \
    fflush(stdout);                                                            \
} while(0)
#define PASSED()        do {puts(" PASSED");fflush(stdout);} while(0)
#define FAILED()        do {puts("*FAILED*");fflush(stdout);} while(0)
#define WARNING()       do {puts("*WARNING*");fflush(stdout);} while(0)
#define SKIPPED()       do {puts(" -SKIP-");fflush(stdout);} while(0)
#define TEST_ERROR      do {FAILED(); AT(); goto error;} while(0)
#define FAIL_OP_ERROR(OP) do {FAILED(); AT(); OP; goto error;} while(0)
#define OP_SUPPRESS(OP, COUNTER, LIMIT) do {                                   \
    if((COUNTER) <= (LIMIT)) {                                                 \
        OP;                                                                    \
        if((COUNTER) == (LIMIT))                                               \
            puts("    Suppressing further output...");                         \
        fflush(stdout);                                                        \
    } /* end if */                                                             \
} while(0)

/*
 * Array of number of threads.  Each threaded test is run once for each entry in
 * this array.  Remove the last test if OPA_LIMIT_THREADS is defined.
 */
static const unsigned num_threads[] = {1, 2, 4, 10, 100};
static const unsigned num_thread_tests = sizeof(num_threads) / sizeof(num_threads[0])
#if OPA_MAX_NTHREADS < 10
- 2
#elif OPA_MAX_NTHREADS < 100
- 1
#endif /* OPA_LIMIT_THREADS */
;

/*
 * Factor to reduce the number of iterations by for each test.  Must be the same
 * size as num_threads.
 */
static const unsigned iter_reduction[] = {1, 1, 1, 1, 10};

/*
 * Other global variables.
 */
static unsigned curr_test;   /* Current test number bein run */



<|start_filename|>src/openpa/src/primitives/opa_gcc_intel_32_64.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef OPA_GCC_INTEL_32_64_H_INCLUDED
#define OPA_GCC_INTEL_32_64_H_INCLUDED

#include "opa_gcc_intel_32_64_barrier.h"
#include "opa_gcc_intel_32_64_ops.h"

#include "opa_emulated.h"

#endif /* OPA_GCC_INTEL_32_64_H_INCLUDED */


<|start_filename|>src/openpa/src/opa_queue.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*  
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

/* Implements a fast, lockfree, multi-producer, single-consumer queue.  It's
 * important to note the *single-consumer* piece of this, since multithreaded
 * consumption will surely lead to data corruption and/or other problems. */

#ifndef OPA_QUEUE_H_INCLUDED
#define OPA_QUEUE_H_INCLUDED

#include "opa_primitives.h"
#ifdef OPA_HAVE_STDDEF_H
#include <stddef.h>
#endif /* OPA_HAVE_STDDEF_H */

/* This value is used to indicate NULL in the OPA_Shm_asymm_base_addr
   variable.  It is non-zero because one of the likely base addresses is zero
   (indicating memory is actually symmetrically mapped).  The value 64 was used
   because it is unlikely that mmap will choose an address this low for a
   mapping.  */
/* XXX DJG TODO put some conditionally compiled error checking in that uses this */
#define OPA_SHM_ASYMM_NULL_VAL    64

extern char *OPA_Shm_asymm_base_addr;

/* Used to initialize the base address for relative pointers.  This interface
   assumes that there is only one shared memory segment.  If this turns out to
   not be the case in the future, we should probably add support for multiple
   shm segments.
   
   This function will return an error if it has already been called. */
int OPA_Shm_asymm_init(char *base);

/* Relative addressing macros.  These are for manipulating addresses relative
   to the start of a shared memory region. */
#define OPA_SHM_REL_NULL (0x0)
#define OPA_SHM_IS_REL_NULL(rel_ptr) (OPA_load_ptr(&(rel_ptr).offset) == OPA_SHM_REL_NULL)
#define OPA_SHM_SET_REL_NULL(rel_ptr) (OPA_store_ptr(&(rel_ptr).offset, OPA_SHM_REL_NULL))
#define OPA_SHM_REL_ARE_EQUAL(rel_ptr1, rel_ptr2) \
    (OPA_load_ptr(&(rel_ptr1).offset) == OPA_load_ptr(&(rel_ptr2).offset))

/* This structure exists such that it is possible to expand the expressiveness
   of a relative address at some point in the future.  It also provides a
   modicum of type safety to help prevent certain flavors of errors.
   
   For example, instead of referencing an offset from a global base address, it
   might make sense for there to be multiple base addresses.  These base
   addresses could correspond to the start of a segment or region of shared
   memory.  This structure could store the segment number that is used to lookup
   a base address in a non-shared table.  Note that you would have to be very
   careful about all of this because if you add the segment number as a separate
   field you can no longer (compare and) swap a relative address atomically.  So
   you'll either have to shave bits from the pointer or make some sort of
   requirement that relative addresses can only be swapped within the same
   segment.  */
typedef struct OPA_Shm_rel_addr_t {
    OPA_ptr_t offset;
} OPA_Shm_rel_addr_t;

/* converts a relative pointer to an absolute pointer */
static _opa_inline
void *OPA_Shm_rel_to_abs(OPA_Shm_rel_addr_t r)
{
    void *offset = OPA_load_ptr(&r.offset);
    OPA_assert((size_t)OPA_Shm_asymm_base_addr != OPA_SHM_ASYMM_NULL_VAL);
    return (void*)(OPA_Shm_asymm_base_addr + (size_t)offset);
}

/* converts an absolute pointer to a relative pointer */
static _opa_inline
OPA_Shm_rel_addr_t OPA_Shm_abs_to_rel(void *a)
{
    OPA_Shm_rel_addr_t ret;

    OPA_assert((size_t)OPA_Shm_asymm_base_addr != OPA_SHM_ASYMM_NULL_VAL);
    OPA_store_ptr(&ret.offset, (void*)((size_t)a - (size_t)OPA_Shm_asymm_base_addr));
    return ret;
}

static _opa_inline
OPA_Shm_rel_addr_t OPA_Shm_swap_rel(OPA_Shm_rel_addr_t *addr, OPA_Shm_rel_addr_t newv) {
    OPA_Shm_rel_addr_t oldv;
    OPA_store_ptr(&oldv.offset, OPA_swap_ptr(&addr->offset, OPA_load_ptr(&newv.offset)));
    return oldv;
}

/* Compare the relative pointer to (relative) null and swap if equal.  Prevents
   the guts of the _rel_addr_t abstraction from bleeding up into the
   enqueue/dequeue operations. */
static _opa_inline
OPA_Shm_rel_addr_t OPA_Shm_cas_rel_null(OPA_Shm_rel_addr_t *addr, OPA_Shm_rel_addr_t oldv) {
    OPA_Shm_rel_addr_t prev;
    OPA_store_ptr(&prev.offset, OPA_cas_ptr(&(addr->offset), OPA_load_ptr(&oldv.offset), (void*)OPA_SHM_REL_NULL));
    return prev;
}

/* The shadow head pointer makes this queue implementation perform much better
   than a standard queue.  Unfortunately, it also makes it a bit non-intuitive
   when read the code.  The following is an excerpt from "Design and Evaluation
   of Nemesis,  a Scalable, Low-Latency, Message-Passing Communication
   Subsystem" by <NAME>, <NAME>, and <NAME> that gives an
   explanation:

      A process must access both the head and tail of the queue when it is
      enqueuing an element on an empty queue or when it is dequeuing an element
      that is the last element in the queue. In these cases, if the head and
      tail were in the same cache line, only one L2 cache miss would be
      encountered. If the queue has more elements in it, however, then the
      enqueuer only needs to access the tail, and the dequeuer only needs to
      access the head. If the head and tail were in the same cache line, then
      there would be L2 misses encountered as a result of false sharing each
      time a process enqueues an element after another has been dequeued from
      the same queue, and vice versa. In this case it would be better if the
      head and tail were in separate cache lines.

      Our solution is to put the head and tail in the same cache line and have a
      shadow head pointer in a separate cache line. The shadow head is
      initialized to NULL. The dequeuer uses the shadow head in place of the
      real head except when the shadow head is NULL, meaning that the queue has
      become empty. If the shadow head is NULL when the dequeuer tries to
      dequeue, it checks the value of the real head. If the real head is not
      NULL, meaning that an element has been enqueued on the queue since the
      last time the queue became empty, the dequeuer initializes its shadow head
      to the value of the real head and sets the real head to NULL. In this way,
      only one L2 cache miss is encountered when enqueuing onto an empty queue
      or dequeuing from a queue with one element. And because the tail and
      shadow head are in separate cache lines, there are no L2 cache misses from
      false sharing. 

      We found that using a shadow head pointer reduced one-way latency by about
      200 ns on a dual 2 GHz Xeon node.
*/

/* Pick an arbitrary cacheline size for now, we can setup a mechanism to detect
   it at build time later on.  This should work well on most intel systems at
   the very least. */
#define OPA_QUEUE_CACHELINE_PADDING 128

/* All absolute and relative pointers point to the start of the enclosing element. */
typedef struct OPA_Queue_info_t {
    OPA_Shm_rel_addr_t head; /* holds the offset pointer, not the abs ptr */
    OPA_Shm_rel_addr_t tail; /* holds the offset pointer, not the abs ptr */
    char padding1[OPA_QUEUE_CACHELINE_PADDING-2*sizeof(OPA_Shm_rel_addr_t)];
    OPA_Shm_rel_addr_t  shadow_head; /* holds the offset pointer, not the abs ptr */
    char padding2[OPA_QUEUE_CACHELINE_PADDING-sizeof(OPA_Shm_rel_addr_t)];
} OPA_Queue_info_t;

/* Using this header struct even though it's just one element gives us the
   opportunity to vary the implementation more easily in the future without
   updating all callers. */
typedef struct OPA_Queue_element_hdr_t {
    OPA_Shm_rel_addr_t next; /* holds the offset pointer, not the abs ptr */
} OPA_Queue_element_hdr_t;


/* Used to initialize a queue structure. */
void OPA_Queue_init(OPA_Queue_info_t *qhead);

/* Used to initialize a queue element header. */
void OPA_Queue_header_init(OPA_Queue_element_hdr_t *hdr);

static _opa_inline
int OPA_Queue_is_empty(OPA_Queue_info_t *qhead)
{
    int __ret = 0;
    if (OPA_SHM_IS_REL_NULL (qhead->shadow_head)) {
        if (OPA_SHM_IS_REL_NULL(qhead->head)) {
            __ret = 1;
        }
        else {
            qhead->shadow_head = qhead->head;
            OPA_SHM_SET_REL_NULL(qhead->head); /* reset it for next time */
        }
    }
    return __ret;
}

/* Returns a pointer to the element at the head of the queue.  The current
   implementation of these queue algorithms imposes several notable
   restrictions on the use of this function:
    - The caller must currently hold the critical section, just as if you were
      calling OPA_Queue_dequeue.  Failure to do could easily produce incorrect
      results.
    - OPA_Queue_is_empty must be called on this queue prior to calling this
      function.  Furthermore, there must be no intervening calls to any
      OPA_Queue_* functions (for this queue) between _is_empty and _peek_head.
      Failure to do so will produce incorrect results.

   This operation is effectively the same as the dequeue operation (insofar as
   it shares the same calling restrictions) but it does not disturb the actual
   contents of the queue. */
static _opa_inline
void *OPA_Queue_peek_head(OPA_Queue_info_t *qhead_ptr)
{
    OPA_assert(qhead_ptr != NULL);

    if (OPA_SHM_IS_REL_NULL(qhead_ptr->shadow_head)) {
        return NULL;
    }
    return OPA_Shm_rel_to_abs(qhead_ptr->shadow_head);
}

/* This macro atomically enqueues an element (elt for short) into the queue
   indicated by qhead_ptr.  You need to pass several arguments:
     qhead_ptr - a pointer to a OPA_Queue_info_t structure to which the
                 element should be enqueued
     elt_ptr - a pointer to an element structure type that is to be enqueued
     elt_type - The base type of elt_ptr.  That is, if elt_ptr is a
                '(struct example_t *)' then elt_type is 'struct example_t'.
     elt_hdr_field - This is the member name of elt_type that is a
                     OPA_Queue_element_hdr_t.

    This queue implementation is loosely modeled after the linked lists used in
    the linux kernel.  You put the list header structure inside of the client
    structure, rather than the other way around.
   */
#define OPA_Queue_enqueue(qhead_ptr, elt_ptr, elt_type, elt_hdr_field)    \
do {                                                                      \
    OPA_Shm_rel_addr_t r_prev;                                            \
    OPA_Shm_rel_addr_t r_elt = OPA_Shm_abs_to_rel(elt_ptr);               \
                                                                          \
    OPA_SHM_SET_REL_NULL((elt_ptr)->elt_hdr_field.next);                  \
                                                                          \
    OPA_write_barrier();                                                  \
                                                                          \
    r_prev = OPA_Shm_swap_rel(&((qhead_ptr)->tail), r_elt);               \
                                                                          \
    if (OPA_SHM_IS_REL_NULL(r_prev)) {                                    \
        (qhead_ptr)->head = r_elt;                                        \
    }                                                                     \
    else {                                                                \
        elt_type *abs_prev = (elt_type *)OPA_Shm_rel_to_abs(r_prev);      \
        abs_prev->elt_hdr_field.next = r_elt;                             \
    }                                                                     \
} while (0)

/* This macro atomically dequeues an element (elt for short) from the queue
   indicated by qhead_ptr.  You need to pass several arguments:
     qhead_ptr - a pointer to a OPA_Queue_info_t structure to which the
                 element should be dequeued
     elt_ptr - A pointer to an element structure type that should be populated
               with the dequeued value.  Must be an lvalue.
     elt_type - The base type of elt_ptr.  That is, if elt_ptr is a
                '(struct example_t *)' then elt_type is 'struct example_t'.
     elt_hdr_field - This is the member name of elt_type that is a
                     OPA_Queue_element_hdr_t.

    This queue implementation is loosely modeled after the linked lists used in
    the linux kernel.  You put the list header structure inside of the client
    structure, rather than the other way around.

    NOTE: you must *always* call _is_empty() prior to this function */
#define OPA_Queue_dequeue(qhead_ptr, elt_ptr, elt_type, elt_hdr_field)        \
do {                                                                          \
    elt_type *_e;                                                             \
    OPA_Shm_rel_addr_t _r_e;                                                  \
                                                                              \
    _r_e = (qhead_ptr)->shadow_head;                                          \
    _e = OPA_Shm_rel_to_abs(_r_e);                                            \
                                                                              \
    if (!OPA_SHM_IS_REL_NULL(_e->elt_hdr_field.next)) {                       \
        (qhead_ptr)->shadow_head = _e->elt_hdr_field.next;                    \
    }                                                                         \
    else {                                                                    \
        OPA_Shm_rel_addr_t old_tail;                                          \
                                                                              \
        OPA_SHM_SET_REL_NULL((qhead_ptr)->shadow_head);                       \
                                                                              \
        old_tail = OPA_Shm_cas_rel_null(&((qhead_ptr)->tail), _r_e);          \
                                                                              \
        if (!OPA_SHM_REL_ARE_EQUAL(old_tail, _r_e)) {                         \
            while (OPA_SHM_IS_REL_NULL(_e->elt_hdr_field.next)) {             \
                OPA_busy_wait();                                              \
            }                                                                 \
            (qhead_ptr)->shadow_head = _e->elt_hdr_field.next;                \
        }                                                                     \
    }                                                                         \
    OPA_SHM_SET_REL_NULL(_e->elt_hdr_field.next);                             \
    OPA_read_barrier();                                                       \
    elt_ptr = _e;                                                             \
} while (0)

#endif /* OPA_QUEUE_H_INCLUDED */


<|start_filename|>src/openpa/src/primitives/opa_gcc_intel_32_64_barrier.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef OPA_GCC_INTEL_32_64_BARRIER_H_INCLUDED
#define OPA_GCC_INTEL_32_64_BARRIER_H_INCLUDED

#define OPA_compiler_barrier()   __asm__ __volatile__  ( "" ::: "memory" )

/* For all regular memory (write-back cacheable, not driver/graphics
 * memory), there is only one general ordering relaxation permitted by
 * x86/x86_64 processors: earlier stores may be retired after later
 * stores.  The "clflush" and "movnt*" instructions also don't follow
 * general ordering constraints, although any code using these
 * instructions should be responsible for ensuring proper ordering
 * itself.  So our read and write barriers may be implemented as simple
 * compiler barriers. */
#define OPA_write_barrier() OPA_compiler_barrier()
#define OPA_read_barrier()  OPA_compiler_barrier()

#define OPA_read_write_barrier() __asm__ __volatile__  ( "mfence" ::: "memory" )

#endif /* OPA_GCC_INTEL_32_64_BARRIER_H_INCLUDED */


<|start_filename|>src/openpa/src/primitives/opa_gcc_ppc.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*  
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef OPA_GCC_PPC_H_INCLUDED
#define OPA_GCC_PPC_H_INCLUDED

/* these need to be aligned on an 8-byte boundary to work on a BG/P */
typedef struct { volatile int v    OPA_ATTRIBUTE((aligned (8))); } OPA_int_t;
typedef struct { void * volatile v OPA_ATTRIBUTE((aligned (8))); } OPA_ptr_t;

#define OPA_INT_T_INITIALIZER(val_) { (val_) }
#define OPA_PTR_T_INITIALIZER(val_) { (val_) }

/* Aligned loads and stores are atomic. */
static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr)
{
    return ptr->v;
}

/* Aligned loads and stores are atomic. */
static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val)
{
    ptr->v = val;
}

/* Aligned loads and stores are atomic. */
static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr)
{
    return ptr->v;
}

/* Aligned loads and stores are atomic. */
static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val)
{
    ptr->v = val;
}

/* a useful link for PPC memory ordering issues:
 *   http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2009.02.22a.html
 *
 * lwsync: orders L-L, S-S, L-S, but *not* S-L (i.e. gives x86-ish ordering)
 * eieio: orders S-S (but only for cacheable memory, not for MMIO)
 * sync: totally orders memops
 * isync: force all preceeding insns to appear complete before starting
 *        subsequent insns, but w/o cumulativity (very confusing)
 */
/* helper macros */
#define OPA_ppc_lwsync_() __asm__ __volatile__  ( "lwsync" ::: "memory" )
#define OPA_ppc_hwsync_() __asm__ __volatile__  ( "sync"   ::: "memory" )
#define OPA_ppc_eieio_()  __asm__ __volatile__  ( "eieio"  ::: "memory" )

#define OPA_write_barrier()      OPA_ppc_eieio_()
#define OPA_read_barrier()       OPA_ppc_lwsync_()
#define OPA_read_write_barrier() OPA_ppc_hwsync_()
#define OPA_compiler_barrier()   __asm__ __volatile__  ( "" ::: "memory" )

/* NOTE-PPC-1 we use lwsync, although I think we might be able to use
 * conditional-branch+isync in some cases (load_acquire?) once we understand it
 * better */

static _opa_inline int   OPA_load_acquire_int(_opa_const OPA_int_t *ptr)
{
    int tmp;
    tmp = ptr->v;
    OPA_ppc_lwsync_(); /* NOTE-PPC-1 */
    return tmp;
}
static _opa_inline void  OPA_store_release_int(OPA_int_t *ptr, int val)
{
    OPA_ppc_lwsync_();
    ptr->v = val;
}
static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr)
{
    void *tmp;
    tmp = ptr->v;
    OPA_ppc_lwsync_(); /* NOTE-PPC-1 */
    return tmp;
}
static _opa_inline void  OPA_store_release_ptr(OPA_ptr_t *ptr, void *val)
{
    OPA_ppc_lwsync_();
    ptr->v = val;
}


/*
   load-link/store-conditional (LL/SC) primitives.  We LL/SC implement
   these here, which are arch-specific, then use the generic
   implementations from opa_emulated.h */

static _opa_inline int OPA_LL_int(OPA_int_t *ptr)
{
    int val;
    __asm__ __volatile__ ("lwarx %[val],0,%[ptr]"
                          : [val] "=r" (val)
                          : [ptr] "r" (&ptr->v)
                          : "cc");

    return val;
}

/* Returns non-zero if the store was successful, zero otherwise. */
static _opa_inline int OPA_SC_int(OPA_int_t *ptr, int val)
{
    int ret = 1; /* init to non-zero, will be reset to 0 if SC was unsuccessful */
    __asm__ __volatile__ ("stwcx. %[val],0,%[ptr];\n"
                          "beq 1f;\n"
                          "li %[ret], 0;\n"
                          "1: ;\n"
                          : [ret] "=r" (ret)
                          : [ptr] "r" (&ptr->v), [val] "r" (val), "0" (ret)
                          : "cc", "memory");
    return ret;
}


/* Pointer versions of LL/SC. */

/* Set OPA_SS (Size Suffix) which is used to choose between lwarx/stwcx and
 * ldarx/stdcx when using 4 or 8 byte pointer operands */
#if OPA_SIZEOF_VOID_P == 4
#define OPA_SS "w"
#elif OPA_SIZEOF_VOID_P == 8
#define OPA_SS "d"
#else
#error OPA_SIZEOF_VOID_P is not 4 or 8
#endif


static _opa_inline void *OPA_LL_ptr(OPA_ptr_t *ptr)
{
    void *val;
    __asm__ __volatile__ ("l"OPA_SS"arx %[val],0,%[ptr]"
                          : [val] "=r" (val)
                          : [ptr] "r" (&ptr->v)
                          : "cc");

    return val;
}

/* Returns non-zero if the store was successful, zero otherwise. */
static _opa_inline int OPA_SC_ptr(OPA_ptr_t *ptr, void *val)
{
    int ret = 1; /* init to non-zero, will be reset to 0 if SC was unsuccessful */
    __asm__ __volatile__ ("st"OPA_SS"cx. %[val],0,%[ptr];\n"
                          "beq 1f;\n"
                          "li %[ret], 0;\n"
                          "1: ;\n"
                          : [ret] "=r" (ret)
                          : [ptr] "r" (&ptr->v), [val] "r" (val), "0" (ret)
                          : "cc", "memory");
    return ret;
}

#undef OPA_SS

/* necessary to enable LL/SC emulation support */
#define OPA_LL_SC_SUPPORTED 1

/* Implement all function using LL/SC */
#define OPA_add_int_by_llsc            OPA_add_int
#define OPA_incr_int_by_llsc           OPA_incr_int
#define OPA_decr_int_by_llsc           OPA_decr_int
#define OPA_decr_and_test_int_by_llsc  OPA_decr_and_test_int
#define OPA_fetch_and_add_int_by_llsc  OPA_fetch_and_add_int
#define OPA_fetch_and_decr_int_by_llsc OPA_fetch_and_decr_int
#define OPA_fetch_and_incr_int_by_llsc OPA_fetch_and_incr_int
#define OPA_cas_ptr_by_llsc        OPA_cas_ptr
#define OPA_cas_int_by_llsc        OPA_cas_int
#define OPA_swap_ptr_by_llsc       OPA_swap_ptr
#define OPA_swap_int_by_llsc       OPA_swap_int


#include "opa_emulated.h"

#endif /* OPA_GCC_PPC_H_INCLUDED */


<|start_filename|>src/openpa/src/opa_util.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef OPA_UTIL_H_INCLUDED
#define OPA_UTIL_H_INCLUDED

#define OPA_QUOTE(x_) OPA_QUOTE2(x_)
#define OPA_QUOTE2(x_) #x_

#if defined(OPA_HAVE_GCC_ATTRIBUTE)
#  define OPA_ATTRIBUTE(x_) __attribute__ (x_)
#else
#  define OPA_ATTRIBUTE(x_)
#endif

/* FIXME this just needs a total rework in general with an OPA_NDEBUG or similar. */
#define OPA_assert(expr_) do {} while (0)
#define OPA_assertp(expr_) do { if (!(expr_)) ++((int *)NULL) } while (0) /* SEGV intentionally */

/* A compile-time assertion macro.  It should cause a compilation error if (expr_) is false. */
#define OPA_COMPILE_TIME_ASSERT(expr_) \
    do { switch(0) { case 0: case (expr_): default: break; } } while (0)

#endif /* OPA_UTIL_H_INCLUDED */


<|start_filename|>src/openpa/src/opa_primitives.c<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*  
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#include "opa_config.h"

/* FIXME For now we rely on pthreads for our IPC locks.  This is fairly
   portable, although it is obviously not 100% portable.  We need to
   figure out how to support other threading packages and lock
   implementations, such as the BG/P lockbox. */
#if defined(OPA_HAVE_PTHREAD_H)
#include <pthread.h>
#include <opa_primitives.h>

pthread_mutex_t *OPA_emulation_lock = NULL;

int OPA_Interprocess_lock_init(OPA_emulation_ipl_t *shm_lock, int isLeader)
{
    int mpi_errno = 0; /*MPI_SUCCESS*/
    pthread_mutexattr_t attr;
    OPA_emulation_lock = shm_lock;

    if (isLeader) {
        /* Set the mutex attributes to work correctly on inter-process
         * shared memory as well. This is required for some compilers
         * (such as SUN Studio) that don't enable it by default. */
        if (pthread_mutexattr_init(&attr) ||
            pthread_mutexattr_setpshared(&attr, PTHREAD_PROCESS_SHARED) ||
            pthread_mutex_init(OPA_emulation_lock, &attr))
            mpi_errno = 16; /* MPI_ERR_INTERN */
    }

    return mpi_errno;
}
#endif /* defined(OPA_HAVE_PTHREAD_H) */



<|start_filename|>src/openpa/src/primitives/opa_gcc_ia64.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*  
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

/* NOTE: We (@ANL) do not have easy access to any ia64 hosts, so it is difficult
 * to develop and test updates to this file.  Any help in this regard is greatly
 * appreciated. */

#ifndef OPA_GCC_IA64_H_INCLUDED
#define OPA_GCC_IA64_H_INCLUDED

/* FIXME do we need to align these? */
typedef struct { volatile int v;    } OPA_int_t;
typedef struct { void * volatile v; } OPA_ptr_t;

#define OPA_INT_T_INITIALIZER(val_) { (val_) }
#define OPA_PTR_T_INITIALIZER(val_) { (val_) }

/* Aligned loads and stores are atomic on ia64. */
static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr)
{
    return ptr->v;
}

/* Aligned loads and stores are atomic on ia64. */
static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val)
{
    ptr->v = val;
}

/* Aligned loads and stores are atomic on ia64. */
static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr)
{
    return ptr->v;
}

/* Aligned loads and stores are atomic on ia64. */
static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val)
{
    ptr->v = val;
}


/* NOTE-IA64-1 ia64 suports half-fence suffixes to ld/st instructions.  It seems
 * that compilers also treat any C-language load/store from/to a volatile
 * variable as acquire/release and append the corresponding suffix accordingly.
 * But we don't have extensive testing to back this up, so we'll explicitly
 * force these instructions for now.  When using the C-level approach, it's not
 * clear what instructions the compiler is allowed to reorder. */

#if 0
/* acquire means operations after the acquire will see any memory opeations
 * performed before the corresponding paired release operation */
static _opa_inline int   OPA_load_acquire_int(_opa_const OPA_int_t *ptr)
{
    int tmp;
    __asm__ __volatile__ ("ld.acq %0=[%1]"
                          : "=r" (tmp)
                          : "r"  (ptr->v)
                          : "memory");
    return tmp;
}
static _opa_inline void  OPA_store_release_int(OPA_int_t *ptr, int val)
{
    __asm__ __volatile__ ("st.rel [%0]=%1"
                          : "=m"  (ptr->v)
                          : "r" (val)
                          : "memory");
}
static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr)
{
    int tmp;
    __asm__ __volatile__ ("ld.acq %0=[%1]"
                          : "=r" (tmp)
                          : "r"  (ptr->v)
                          : "memory");
    return tmp;
}
static _opa_inline void  OPA_store_release_ptr(OPA_ptr_t *ptr, void *val)
{
    __asm__ __volatile__ ("st.rel [%0]=%1"
                          : "=m"  (ptr->v)
                          : "r" (val)
                          : "memory");
}
#else
/* FIXME because we can't test these implementations, they are currently
 * disabled.  The above impls are rough starting points, but probably won't
 * compile/assemble correctly as-is.  Patches are welcome :) */
#define OPA_load_acquire_int(ptr_)       ::"choke me"
#define OPA_store_release_int(ptr_,val_) ::"choke me"
#define OPA_load_acquire_ptr(ptr_)       ::"choke me"
#define OPA_store_release_ptr(ptr_,val_) ::"choke me"
#endif


#define OPA_add_int_by_faa OPA_add_int
#define OPA_incr_int_by_faa OPA_incr_int
#define OPA_decr_int_by_faa OPA_decr_int
#define OPA_fetch_and_decr_int_by_faa OPA_fetch_and_decr_int
#define OPA_fetch_and_incr_int_by_faa OPA_fetch_and_incr_int

static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr)
{
    int val;
    __asm__ __volatile__ ("fetchadd4.rel %0=[%2],%3"
                          : "=r"(val), "=m"(ptr->v)
                          : "r"(&ptr->v), "i"(-1));
    return val == 1;
}

static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv)
{
    int prev;

#if OPA_SIZEOF_INT == 8
    __asm__ __volatile__ ("mov ar.ccv=%1;;"
                          "cmpxchg8.rel %0=[%3],%4,ar.ccv"
                          : "=r"(prev), "=m"(ptr->v)
                          : "rO"(oldv), "r"(&ptr->v), "r"(newv)
                          : "memory");
    break;
#elif OPA_SIZEOF_INT == 4
    __asm__ __volatile__ ("zxt4 %1=%1;;" /* don't want oldv sign-extended to 64 bits */
                          "mov ar.ccv=%1;;"
                          "cmpxchg4.rel %0=[%3],%4,ar.ccv"
                          : "=r"(prev), "=m"(ptr->v)
                          : "r0"(oldv), "r"(&ptr->v), "r"(newv)
                          : "memory");
#else
#error OPA_SIZEOF_INT is not 4 or 8
#endif

    return prev;
}

/* IA64 has a fetch-and-add instruction that only accepts immediate
   values of -16, -8, -4, -1, 1, 4, 8, and 16.  So we check for these
   values before falling back to the CAS implementation. */
#define OPA_IA64_FAA_CASE_MACRO(ptr, val) case val: {   \
    int prev;                                           \
    __asm__ __volatile__ ("fetchadd4.rel %0=[%2],%3"    \
                          : "=r"(prev), "=m"(ptr->v)    \
                          : "r"(&ptr->v), "i"(val));    \
    return prev;                                        \
    }                                                   \
    break


static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val)
{
    switch (val)
    {
        OPA_IA64_FAA_CASE_MACRO(ptr, -16);
        OPA_IA64_FAA_CASE_MACRO(ptr,  -8);
        OPA_IA64_FAA_CASE_MACRO(ptr,  -4);
        OPA_IA64_FAA_CASE_MACRO(ptr,  -1);
        OPA_IA64_FAA_CASE_MACRO(ptr,   1);
        OPA_IA64_FAA_CASE_MACRO(ptr,   4);
        OPA_IA64_FAA_CASE_MACRO(ptr,   8);
        OPA_IA64_FAA_CASE_MACRO(ptr,  16);
    default:
        {
            int cmp;
            int prev = OPA_load_int(ptr);

            do {
                cmp = prev;
                prev = OPA_cas_int(ptr, cmp, val);
            } while (prev != cmp);

            return prev;
        }
    }
}
#undef OPA_IA64_FAA_CASE_MACRO


static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv)
{
    void *prev;
    __asm__ __volatile__ ("mov ar.ccv=%1;;"
                          "cmpxchg8.rel %0=[%3],%4,ar.ccv"
                          : "=r"(prev), "=m"(ptr->v)
                          : "rO"(oldv), "r"(&ptr->v), "r"(newv));
    return prev;
}

static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val)
{
    __asm__ __volatile__ ("xchg8 %0=[%2],%3"
                          : "=r" (val), "=m" (ptr->v)
                          : "r" (&ptr->v), "0" (val));
    return val;
}


static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val)
{
    __asm__ __volatile__ ("xchg8 %0=[%2],%3"
                          : "=r" (val), "=m" (ptr->v)
                          : "r" (&ptr->v), "0" (val));
    return val;
}


#define OPA_write_barrier()      __asm__ __volatile__  ("mf" ::: "memory" )
#define OPA_read_barrier()       __asm__ __volatile__  ("mf" ::: "memory" )
#define OPA_read_write_barrier() __asm__ __volatile__  ("mf" ::: "memory" )
#define OPA_compiler_barrier()   __asm__ __volatile__  (""   ::: "memory" )

#include "opa_emulated.h"

#endif /* OPA_GCC_IA64_H_INCLUDED */


<|start_filename|>src/openpa/src/primitives/opa_sun_atomic_ops.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*  
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */


#ifndef OPA_SUN_ATOMIC_OPS_H_INCLUDED
#define OPA_SUN_ATOMIC_OPS_H_INCLUDED

#include <atomic.h>

typedef struct { volatile uint_t v;  } OPA_int_t;
typedef struct { void * volatile v; } OPA_ptr_t;

#define OPA_INT_T_INITIALIZER(val_) { (val_) }
#define OPA_PTR_T_INITIALIZER(val_) { (val_) }

static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr)
{
    return (int)ptr->v;
}

static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val)
{
    ptr->v = (uint)val;
}

static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr)
{
    return ptr->v;
}

static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val)
{
    ptr->v = val;
}

static _opa_inline int OPA_load_acquire_int(_opa_const OPA_int_t *ptr)
{
    int tmp;
    tmp = ptr->v;
    membar_enter();
    return tmp;
}

static _opa_inline void OPA_store_release_int(OPA_int_t *ptr, int val)
{
    membar_exit();
    ptr->v = val;
}

static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr)
{
    void *tmp;
    tmp = ptr->v;
    membar_enter();
    return tmp;
}

static _opa_inline void OPA_store_release_ptr(OPA_ptr_t *ptr, void *val)
{
    membar_exit();
    ptr->v = val;
}


static _opa_inline void OPA_add_int(OPA_int_t *ptr, int val)
{
    atomic_add_int(&ptr->v, val);
}

static _opa_inline void OPA_incr_int(OPA_int_t *ptr)
{
    atomic_inc_uint(&ptr->v);
}

static _opa_inline void OPA_decr_int(OPA_int_t *ptr)
{
    atomic_dec_uint(&ptr->v);
}


static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr)
{
    return atomic_dec_uint_nv(&ptr->v) == 0;    
}


static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val)
{
    return (int)atomic_add_int_nv(&ptr->v, val) - val;
}

static _opa_inline int OPA_fetch_and_decr_int(OPA_int_t *ptr)
{
    return (int)atomic_dec_uint_nv(&ptr->v) + 1;
}

static _opa_inline int OPA_fetch_and_incr_int(OPA_int_t *ptr)
{
    return (int)atomic_inc_uint_nv(&ptr->v) - 1;
}


static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv)
{
    return atomic_cas_ptr(ptr, oldv, newv);
}

static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv)
{
    return (int)atomic_cas_uint(&ptr->v, (uint_t)oldv, (uint_t)newv);
}


static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val)
{
    return atomic_swap_ptr(ptr, val);
}

static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val)
{
    return (int)atomic_swap_uint(&ptr->v, (uint_t) val);
}


#define OPA_write_barrier()      membar_producer()
#define OPA_read_barrier()       membar_consumer()
#define OPA_read_write_barrier() do { membar_consumer(); membar_producer(); } while (0)

/* is this portable enough? */
#define OPA_compiler_barrier()   __asm__ __volatile__  ( ""  ::: "memory" )

#endif /* OPA_SUN_ATOMIC_OPS_H_INCLUDED */


<|start_filename|>src/openpa/src/primitives/opa_gcc_intel_32_64_ops.h<|end_filename|>
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
/*
 *  (C) 2008 by Argonne National Laboratory.
 *      See COPYRIGHT in top-level directory.
 */

#ifndef OPA_GCC_INTEL_32_64_OPS_H_INCLUDED
#define OPA_GCC_INTEL_32_64_OPS_H_INCLUDED

#ifndef OPA_SIZEOF_INT
#error OPA_SIZEOF_INT is not defined
#endif

/* Set OPA_SS (Size Suffix) which is to be appended to asm ops for
   specifying 4 or 8 byte operands */
#if OPA_SIZEOF_INT == 4
#define OPA_SS "l"
#elif OPA_SIZEOF_INT == 8
#define OPA_SS "q"
#else
#error OPA_SIZEOF_INT is not 4 or 8
#endif

/* XXX DJG FIXME do we need to align these? */
typedef struct { volatile int v;    } OPA_int_t;
typedef struct { void * volatile v; } OPA_ptr_t;

#define OPA_INT_T_INITIALIZER(val_) { (val_) }
#define OPA_PTR_T_INITIALIZER(val_) { (val_) }

/* Aligned loads and stores are atomic on x86(-64). */
static _opa_inline int OPA_load_int(_opa_const OPA_int_t *ptr)
{
    return ptr->v;
}

/* Aligned loads and stores are atomic on x86(-64). */
static _opa_inline void OPA_store_int(OPA_int_t *ptr, int val)
{
    ptr->v = val;
}

/* Aligned loads and stores are atomic on x86(-64). */
static _opa_inline void *OPA_load_ptr(_opa_const OPA_ptr_t *ptr)
{
    return ptr->v;
}

/* Aligned loads and stores are atomic on x86(-64). */
static _opa_inline void OPA_store_ptr(OPA_ptr_t *ptr, void *val)
{
    ptr->v = val;
}

/* NOTE-X86-1 x86 and x86_64 processors execute instructions such that their
 * effects are visible in issue order, with one exception:  earlier stores may
 * pass later loads.  Also, non-temporal stores (with the "movnt*" instructions)
 * do not follow these ordering constraints.  So all normal load/store
 * instructions on x86(_64) already have acquire/release semantics.  We just
 * have to make sure that the compiler does not reorder the instructions. */

static _opa_inline int OPA_load_acquire_int(_opa_const OPA_int_t *ptr)
{
    int tmp;
    tmp = ptr->v;
    OPA_compiler_barrier(); /* NOTE-X86-1 */
    return tmp;
}

static _opa_inline void OPA_store_release_int(OPA_int_t *ptr, int val)
{
    OPA_compiler_barrier(); /* NOTE-X86-1 */
    ptr->v = val;
}

static _opa_inline void *OPA_load_acquire_ptr(_opa_const OPA_ptr_t *ptr)
{
    void *tmp;
    tmp = ptr->v;
    OPA_compiler_barrier(); /* NOTE-X86-1 */
    return tmp;
}

static _opa_inline void OPA_store_release_ptr(OPA_ptr_t *ptr, void *val)
{
    OPA_compiler_barrier(); /* NOTE-X86-1 */
    ptr->v = val;
}


static _opa_inline void OPA_add_int(OPA_int_t *ptr, int val)
{
    __asm__ __volatile__ ("lock ; add"OPA_SS" %1,%0"
                          :"=m" (ptr->v)
                          :"ir" (val), "m" (ptr->v));
    return;
}

static _opa_inline void OPA_incr_int(OPA_int_t *ptr)
{
    __asm__ __volatile__ ("lock ; inc"OPA_SS" %0"
                          :"=m" (ptr->v)
                          :"m" (ptr->v));
    return;
}

static _opa_inline void OPA_decr_int(OPA_int_t *ptr)
{
    __asm__ __volatile__ ("lock ; dec"OPA_SS" %0"
                          :"=m" (ptr->v)
                          :"m" (ptr->v));
    return;
}


static _opa_inline int OPA_decr_and_test_int(OPA_int_t *ptr)
{
    char result;
    __asm__ __volatile__ ("lock ; dec"OPA_SS" %0; setz %1"
                          :"=m" (ptr->v), "=q" (result)
                          :"m" (ptr->v));
    return result;
}

static _opa_inline int OPA_fetch_and_add_int(OPA_int_t *ptr, int val)
{
    __asm__ __volatile__ ("lock ; xadd %0,%1"
                          : "=r" (val), "=m" (ptr->v)
                          :  "0" (val),  "m" (ptr->v));
    return val;
}

#define OPA_fetch_and_incr_int_by_faa OPA_fetch_and_incr_int
#define OPA_fetch_and_decr_int_by_faa OPA_fetch_and_decr_int


static _opa_inline void *OPA_cas_ptr(OPA_ptr_t *ptr, void *oldv, void *newv)
{
    void *prev;
    __asm__ __volatile__ ("lock ; cmpxchg %3,%4"
                          : "=a" (prev), "=m" (ptr->v)
                          : "0" (oldv), "q" (newv), "m" (ptr->v));
    return prev;
}

static _opa_inline int OPA_cas_int(OPA_int_t *ptr, int oldv, int newv)
{
    int prev;
    __asm__ __volatile__ ("lock ; cmpxchg %3,%4"
                          : "=a" (prev), "=m" (ptr->v)
                          : "0" (oldv), "q" (newv), "m" (ptr->v));
    return prev;
}

static _opa_inline void *OPA_swap_ptr(OPA_ptr_t *ptr, void *val)
{
    __asm__ __volatile__ ("xchg %0,%1"
                          :"=r" (val), "=m" (ptr->v)
                          : "0" (val),  "m" (ptr->v));
    return val;
}

static _opa_inline int OPA_swap_int(OPA_int_t *ptr, int val)
{
    __asm__ __volatile__ ("xchg %0,%1"
                          :"=r" (val), "=m" (ptr->v)
                          : "0" (val),  "m" (ptr->v));
    return val;
}

#undef OPA_SS

#endif /* OPA_GCC_INTEL_32_64_OPS_H_INCLUDED */
