#if defined (_MSC_VER)

#	include "Windows.h"
#	include <intrin.h>

#else

#	define ASM_DMB_ISH         "dmb    ish\n\t"

#	if defined(__ARM_ARCH_7S__)
	// this is sufficient for Swift processors
#		define ASM_REL         "dmb    ishst\n\t"
#	else
#		define ASM_REL         "dmb    ish\n\t"
#	endif

#	define ASM_CLREX		"clrex\n\t"

#	define ASM_ISB         "isb\n\t"
#	define ASM_LABEL(i)    #i ":\n\t"

#endif

static inline void atomic_pause ()
{
}

static inline void atomic_thread_fence (memory_order_relaxed_t)
{
}

static inline void atomic_thread_fence (memory_order_release_t)
{
#if defined(_MSC_VER)
	__dmb (_ARM_BARRIER_ISH);
#else
    __asm__ __volatile__ (ASM_REL : : : "memory");
#endif
}

static inline void atomic_thread_fence (memory_order_acquire_t)
{
#if defined(_MSC_VER)
	__dmb (_ARM_BARRIER_ISH);
#else
    __asm__ __volatile__ (ASM_DMB_ISH : : : "memory");
#endif
}

static inline void atomic_thread_fence (memory_order_acq_rel_t)
{
#if defined(_MSC_VER)
	__dmb (_ARM_BARRIER_ISH);
#else
    __asm__ __volatile__ (ASM_DMB_ISH : : : "memory");
#endif
}

static inline void atomic_thread_fence (int /* memory_order_seq_cst_t */)
{
#if defined(_MSC_VER)
	__dmb (_ARM_BARRIER_ISH);
#else
    __asm__ __volatile__ (ASM_DMB_ISH : : : "memory");
#endif
}

#define ATOMIC_LOAD(PRE, POST) \
	atomic_word res; \
	__asm__ __volatile__ \
	( \
        PRE \
		"ldr	%0, %1\n\t" \
        POST \
		: "=r" (res) \
		: "m" (*p) \
        : "memory" \
	); \
	return res;

static inline atomic_word atomic_load_explicit (const volatile atomic_word* p, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	return (atomic_word) __iso_volatile_load32 ((const volatile __int32*) p);
#else
    ATOMIC_LOAD ("", "")
#endif
}

static inline atomic_word atomic_load_explicit (const volatile atomic_word* p, memory_order_acquire_t)
{
#if defined(_MSC_VER)
	atomic_word res = (atomic_word) __iso_volatile_load32 ((const volatile __int32*) p);
	__dmb (_ARM_BARRIER_ISH);
	return res;
#else
    ATOMIC_LOAD ("", ASM_DMB_ISH)
#endif
}

static inline atomic_word atomic_load_explicit (const volatile atomic_word* p, int /* memory_order_seq_cst_t */)
{
#if defined(_MSC_VER)
	atomic_word res = (atomic_word) __iso_volatile_load32 ((const volatile __int32*) p);
	__dmb (_ARM_BARRIER_ISH);
	return res;
#else
    ATOMIC_LOAD ("", ASM_DMB_ISH)
#endif
}

#define ATOMIC_STORE(PRE, POST) \
	__asm__ __volatile__ \
	( \
        PRE \
		"str	%1, %0\n\t" \
        POST \
		: "=m" (*p) \
		: "r" (v) \
		: "memory" \
	);

static inline void atomic_store_explicit (volatile atomic_word* p, atomic_word v, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	__iso_volatile_store32 ((volatile __int32*) p, (__int32) v);
#else
    ATOMIC_STORE ("", "")
#endif
}

static inline void atomic_store_explicit (volatile atomic_word* p, atomic_word v, memory_order_release_t)
{
#if defined(_MSC_VER)
	__dmb(_ARM_BARRIER_ISH);
	__iso_volatile_store32 ((volatile __int32*) p, (__int32) v);
#else
    ATOMIC_STORE (ASM_REL, "")
#endif
}

static inline void atomic_store_explicit (volatile atomic_word* p, atomic_word v, int /* memory_order_seq_cst_t */)
{
#if defined(_MSC_VER)
	__dmb(_ARM_BARRIER_ISH);
	__iso_volatile_store32 ((volatile __int32*) p, (__int32) v);
	__dmb(_ARM_BARRIER_ISH);
#else
    ATOMIC_STORE (ASM_REL, ASM_DMB_ISH)
#endif
}

#define ATOMIC_XCHG(PRE, POST) \
	atomic_word res; \
	atomic_word success; \
	__asm__ __volatile__ \
	( \
        PRE \
	ASM_LABEL (0) \
		"ldrex	%2, [%4]\n\t" \
		"strex	%0, %3, [%4]\n\t" \
		"teq	%0, #0\n\t" \
		"bne	0b\n\t" \
        POST \
		: "=&r" (success), "+m" (*p), "=&r" (res) \
		: "r" (v), "r" (p) \
		: "cc", "memory" \
	); \
	return res;

static inline atomic_word atomic_exchange_explicit (volatile atomic_word* p, atomic_word v, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	return (atomic_word) _InterlockedExchange_nf ((long volatile*) p, (long) v);
#else
    ATOMIC_XCHG ("", "")
#endif
}

static inline atomic_word atomic_exchange_explicit (volatile atomic_word* p, atomic_word v, memory_order_release_t)
{
#if defined(_MSC_VER)
	// _InterlockedExchange_rel is documented by Microsoft, but it doesn't seem to be defined
	__dmb(_ARM_BARRIER_ISH);
	return (atomic_word) _InterlockedExchange_nf ((long volatile*) p, (long) v);
#else
    ATOMIC_XCHG (ASM_REL, "")
#endif
}

static inline atomic_word atomic_exchange_explicit (volatile atomic_word* p, atomic_word v, memory_order_acquire_t)
{
#if defined(_MSC_VER)
	return (atomic_word) _InterlockedExchange_acq ((long volatile*) p, (long) v);
#else
    ATOMIC_XCHG ("", ASM_ISB)
#endif
}

static inline atomic_word atomic_exchange_explicit (volatile atomic_word* p, atomic_word v, memory_order_acq_rel_t)
{
#if defined(_MSC_VER)
	__dmb(_ARM_BARRIER_ISH);
	return (atomic_word) _InterlockedExchange_acq ((long volatile*) p, (long) v);
#else
    ATOMIC_XCHG (ASM_REL, ASM_ISB)
#endif
}

static inline atomic_word atomic_exchange_explicit (volatile atomic_word* p, atomic_word v, int /* memory_order_seq_cst_t */)
{
#if defined(_MSC_VER)
	// _InterlockedExchange_rel is documented by Microsoft, but it doesn't seem to be defined
	__dmb(_ARM_BARRIER_ISH);
	atomic_word res = (atomic_word) _InterlockedExchange_nf ((long volatile*) p, (long) v);
	__dmb (_ARM_BARRIER_ISH);
	return res;
#else
    ATOMIC_XCHG (ASM_REL, ASM_DMB_ISH)
#endif
}

#if !defined(_MSC_VER)

// atomic_compare_exchange_weak_explicit: can fail spuriously even if *p == *oldval

#define ATOMIC_CMP_XCHG(PRE, POST) \
	atomic_word res; \
	atomic_word success = 0; \
	__asm__ __volatile__ \
	( \
        PRE \
		"ldrex	%2, [%4]\n\t" \
		"teq	%2, %5\n\t" \
		"it     eq\n\t" \
		"strexeq %0, %3, [%4]\n\t" \
        POST \
		: "+r" (success), "+m" (*p), "=&r" (res) \
		: "r" (newval), "r" (p), "r" (*oldval) \
		: "cc", "memory" \
	); \
    *oldval = res; \
	return success;

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_relaxed_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG ("", "")
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_release_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG (ASM_REL, "")
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acquire_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG ("", "teq %0, #0\n\tbne 1f\n\t" ASM_ISB ASM_LABEL (1))
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acq_rel_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG (ASM_REL, "teq %0, #0\n\tbne 1f\n\t" ASM_ISB ASM_LABEL (1))
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, int /* memory_order_seq_cst_t */, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG (ASM_REL, "teq %0, #0\n\tbne 1f\n\t" ASM_DMB_ISH ASM_LABEL (1))
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_release_t, memory_order_release_t)
{
    ATOMIC_CMP_XCHG (ASM_REL, "")
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acquire_t, memory_order_acquire_t)
{
    ATOMIC_CMP_XCHG ("", ASM_ISB)
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acq_rel_t, memory_order_acq_rel_t)
{
    ATOMIC_CMP_XCHG (ASM_REL, ASM_ISB)
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, int /* memory_order_seq_cst_t */, int /* memory_order_seq_cst_t */)
{
    ATOMIC_CMP_XCHG (ASM_REL, ASM_DMB_ISH)
}

#endif

// atomic_compare_exchange_strong_explicit: does loop and only returns false if *p != *oldval

#undef ATOMIC_CMP_XCHG
#define ATOMIC_CMP_XCHG(PRE, POST) \
	atomic_word res; \
	atomic_word success = 0; \
	__asm__ __volatile__ \
	( \
        PRE \
	ASM_LABEL (0) \
		"ldrex	%2, [%4]\n\t" \
		"teq	%2, %5\n\t" \
        "bne    1f\n\t" \
		"strex  %0, %3, [%4]\n\t" \
        "teq    %0, #0\n\t" \
        "bne    0b\n\t" \
        POST \
		: "=&r" (success), "+m" (*p), "=&r" (res) \
		: "r" (newval), "r" (p), "r" (*oldval) \
		: "cc", "memory" \
	); \
    *oldval = res; \
	return success;

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_relaxed_t, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	long tmp = _InterlockedCompareExchange_nf ((long volatile*) p, (long) newval, (long) *oldval);
	return *oldval==tmp ? true : (*oldval=tmp, false);
#else
    ATOMIC_CMP_XCHG ("", ASM_LABEL (1) ASM_CLREX)
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_release_t, memory_order_release_t)
{
#if defined(_MSC_VER)
	long tmp = _InterlockedCompareExchange_rel ((long volatile*) p, (long) newval, (long) *oldval);
	return *oldval==tmp ? true : (*oldval=tmp, false);
#else
    ATOMIC_CMP_XCHG (ASM_REL, ASM_LABEL (1) ASM_CLREX)
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acquire_t, memory_order_acquire_t)
{
#if defined(_MSC_VER)
	long tmp = _InterlockedCompareExchange_acq ((long volatile*) p, (long) newval, (long) *oldval);
	return *oldval==tmp ? true : (*oldval=tmp, false);
#else
    ATOMIC_CMP_XCHG ("", ASM_LABEL (1) ASM_CLREX ASM_ISB)
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acq_rel_t, memory_order_acq_rel_t)
{
#if defined(_MSC_VER)
	__dmb (_ARM_BARRIER_ISH);
	long tmp = _InterlockedCompareExchange_acq ((long volatile*) p, (long) newval, (long) *oldval);
	return *oldval==tmp ? true : (*oldval=tmp, false);
#else
    ATOMIC_CMP_XCHG (ASM_REL, ASM_LABEL (1) ASM_CLREX ASM_ISB)
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, int /* memory_order_seq_cst_t */, int /* memory_order_seq_cst_t */)
{
#if defined(_MSC_VER)
	long tmp = _InterlockedCompareExchange_rel ((long volatile*) p, (long) newval, (long) *oldval);
	__dmb (_ARM_BARRIER_ISH);
	return *oldval==tmp ? true : (*oldval=tmp, false);
#else
    ATOMIC_CMP_XCHG (ASM_REL, ASM_LABEL (1) ASM_CLREX ASM_DMB_ISH)
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_release_t, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_release, memory_order_release);
#else
    ATOMIC_CMP_XCHG (ASM_REL, ASM_LABEL (1) ASM_CLREX)
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acquire_t, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_acquire, memory_order_acquire);
#else
    ATOMIC_CMP_XCHG ("", ASM_ISB ASM_LABEL (1) ASM_CLREX)
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acq_rel_t, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_acq_rel, memory_order_acq_rel);
#else
    ATOMIC_CMP_XCHG (ASM_REL, ASM_ISB ASM_LABEL (1) ASM_CLREX)
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, int /* memory_order_seq_cst_t */, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_seq_cst, memory_order_seq_cst);
#else
    ATOMIC_CMP_XCHG (ASM_REL, ASM_DMB_ISH ASM_LABEL (1) ASM_CLREX)
#endif
}

#if defined(_MSC_VER)

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_relaxed_t, memory_order_relaxed_t)
{
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_relaxed, memory_order_relaxed);
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_release_t, memory_order_relaxed_t)
{
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_release, memory_order_relaxed);
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acquire_t, memory_order_relaxed_t)
{
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_acquire, memory_order_relaxed);
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acq_rel_t, memory_order_relaxed_t)
{
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_acq_rel, memory_order_relaxed);
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, int /* memory_order_seq_cst_t */, memory_order_relaxed_t)
{
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_seq_cst, memory_order_relaxed);
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_release_t, memory_order_release_t)
{
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_release, memory_order_release);
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acquire_t, memory_order_acquire_t)
{
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_acquire, memory_order_acquire);
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acq_rel_t, memory_order_acq_rel_t)
{
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_acq_rel, memory_order_acq_rel);
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, int /* memory_order_seq_cst_t */, int /* memory_order_seq_cst_t */)
{
	return atomic_compare_exchange_strong_explicit (p, oldval, newval, memory_order_seq_cst, memory_order_seq_cst);
}

#endif

#define ATOMIC_OP(PRE, OP, POST) \
	atomic_word res, tmp; \
	atomic_word success; \
	__asm__ __volatile__ \
	( \
        PRE \
	ASM_LABEL (0) \
		"ldrex	%2, [%5]\n\t" \
		OP "	%3, %2, %4\n\t" \
		"strex	%0, %3, [%5]\n\t" \
		"teq	%0, #0\n\t" \
		"bne	0b\n\t" \
        POST \
		: "=&r" (success), "+m" (*p), "=&r" (res), "=&r" (tmp) \
		: "Ir" (v), "r" (p) \
		: "cc", "memory" \
	); \
	return res;

static inline atomic_word atomic_fetch_add_explicit (volatile atomic_word* p, atomic_word v, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	return (atomic_word) _InterlockedExchangeAdd_nf ((volatile long*) p, (long) v);
#else
    ATOMIC_OP ("", "add", "")
#endif
}

static inline atomic_word atomic_fetch_add_explicit (volatile atomic_word* p, atomic_word v, memory_order_release_t)
{
#if defined(_MSC_VER)
	return (atomic_word) _InterlockedExchangeAdd_rel ((volatile long*) p, (long) v);
#else
    ATOMIC_OP (ASM_REL, "add", "")
#endif
}

static inline atomic_word atomic_fetch_add_explicit (volatile atomic_word* p, atomic_word v, memory_order_acquire_t)
{
#if defined(_MSC_VER)
	return (atomic_word) _InterlockedExchangeAdd_acq ((volatile long*) p, (long) v);
#else
    ATOMIC_OP ("", "add", ASM_ISB)
#endif
}

static inline atomic_word atomic_fetch_add_explicit (volatile atomic_word* p, atomic_word v, memory_order_acq_rel_t)
{
#if defined(_MSC_VER)
	__dmb (_ARM_BARRIER_ISH);
	return (atomic_word) _InterlockedExchangeAdd_acq ((volatile long*) p, (long) v);
#else
    ATOMIC_OP (ASM_REL, "add", ASM_ISB)
#endif
}

static inline atomic_word atomic_fetch_add_explicit (volatile atomic_word* p, atomic_word v, int /* memory_order_seq_cst_t */)
{
#if defined(_MSC_VER)
	long oldval = _InterlockedExchangeAdd_rel ((volatile long*) p, (long) v);
	__dmb (_ARM_BARRIER_ISH);
	return (atomic_word) oldval;
#else
    ATOMIC_OP (ASM_REL, "add", ASM_DMB_ISH)
#endif
}

static inline atomic_word atomic_fetch_sub_explicit (volatile atomic_word* p, atomic_word v, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	return atomic_fetch_add_explicit (p, -v, memory_order_relaxed);
#else
    ATOMIC_OP ("", "sub", "")
#endif
}

static inline atomic_word atomic_fetch_sub_explicit (volatile atomic_word* p, atomic_word v, memory_order_release_t)
{
#if defined(_MSC_VER)
	return atomic_fetch_add_explicit (p, -v, memory_order_release);
#else
    ATOMIC_OP (ASM_REL, "sub", "")
#endif
}

static inline atomic_word atomic_fetch_sub_explicit (volatile atomic_word* p, atomic_word v, memory_order_acquire_t)
{
#if defined(_MSC_VER)
	return atomic_fetch_add_explicit (p, -v, memory_order_acquire);
#else
    ATOMIC_OP ("", "sub", ASM_ISB)
#endif
}

static inline atomic_word atomic_fetch_sub_explicit (volatile atomic_word* p, atomic_word v, memory_order_acq_rel_t)
{
#if defined(_MSC_VER)
	return atomic_fetch_add_explicit (p, -v, memory_order_acq_rel);
#else
    ATOMIC_OP (ASM_REL, "sub", ASM_ISB)
#endif
}

static inline atomic_word atomic_fetch_sub_explicit (volatile atomic_word* p, atomic_word v, int /* memory_order_seq_cst_t */)
{
#if defined(_MSC_VER)
	return atomic_fetch_add_explicit (p, -v, memory_order_seq_cst);
#else
    ATOMIC_OP (ASM_REL, "sub", ASM_DMB_ISH)
#endif
}

/*
 *  extensions
 */

static inline void atomic_retain (volatile int* p)
{
#if defined(_MSC_VER)
	_InterlockedIncrement_nf((volatile long*) p);
#else
    atomic_fetch_add_explicit (p, 1, memory_order_relaxed);
#endif
}

static inline bool atomic_release (volatile int* p)
{
#if defined(_MSC_VER)
	// _interlockedDecrement returns the resulting decremented value
	bool res = _InterlockedDecrement_rel((volatile long*) p) == 0;
    if (res)
	{
		__dmb (_ARM_BARRIER_ISH);
	}
#else
	bool res = atomic_fetch_sub_explicit (p, 1, memory_order_release) == 1;
    if (res)
	{
		atomic_thread_fence (memory_order_acquire);
	}
#endif
	return res;
}

/*
 *	double word
 */

// Note: the only way to get atomic 64-bit memory accesses on ARM is to use ldrexd/strexd with a loop
// (ldrd and strd instructions are not guaranteed to appear atomic)

static inline atomic_word2 atomic_load_explicit (const volatile atomic_word2* p, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
//	atomic_word2 r; r.v = __iso_volatile_load64 ((volatile __int64*) p); return r;
	atomic_word2 r;
	r.v = _InterlockedCompareExchange64_nf ((volatile __int64*) p, (__int64) 0, (__int64) 0);
	return r;
#else
	register atomic_word lo __asm__ ("r2");
	register atomic_word hi __asm__ ("r3");
	atomic_word success;
	
	__asm__ __volatile__
	(
	ASM_LABEL (0)
		"ldrexd	%1, %2, [%3]\n\t"
		"strexd	%0, %1, %2, [%3]\n\t"
		"teq	%0, #0\n\t"
		"bne	0b\n\t"
	
		: "=&r" (success), "=&r" (lo), "=&r" (hi)
		: "r" (p)
		: "cc", "r2", "r3"
	);
	atomic_word2 w;
	w.lo = lo;
	w.hi = hi;
	return w;
#endif
}

static inline atomic_word2 atomic_load_explicit (const volatile atomic_word2* p, memory_order_acquire_t)
{
#if defined(_MSC_VER)
	atomic_word2 r;
	r.v = _InterlockedCompareExchange64_acq ((volatile __int64*) p, (__int64) 0, (__int64) 0);
	return r;
#else
	register atomic_word lo __asm__ ("r2");
	register atomic_word hi __asm__ ("r3");
	atomic_word success;
	
	__asm__ __volatile__
	(
	ASM_LABEL (0)
		"ldrexd	%1, %2, [%3]\n\t"
		"strexd	%0, %1, %2, [%3]\n\t"
		"teq	%0, #0\n\t"
		"bne	0b\n\t"
		ASM_ISB
	
		: "=&r" (success), "=&r" (lo), "=&r" (hi)
		: "r" (p)
		: "cc", "memory", "r2", "r3"
	);
	atomic_word2 w;
	w.lo = lo;
	w.hi = hi;
	return w;
#endif
}

static inline void atomic_store_explicit (volatile atomic_word2* p, atomic_word2 v, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	atomic_word2 w, x;
	w = v;
	x = v;
	do
	{
		w.v = _InterlockedCompareExchange64_nf((volatile __int64 *)p, x.v, w.v);
	} while (w.v != x.v);
#else
	register atomic_word l __asm__ ("r2");
	register atomic_word h __asm__ ("r3");
	register atomic_word lo __asm__ ("r0") = v.lo;
	register atomic_word hi __asm__ ("r1") = v.hi;
	atomic_word success;
	
	__asm__ __volatile__
	(
	ASM_LABEL (0)
		"ldrexd	%2, %3, [%6]\n\t"
		"strexd	%0, %4, %5, [%6]\n\t"
		"teq	%0, #0\n\t"
		"bne	0b\n\t"
	
		: "=&r" (success), "=m" (*p), "=&r" (l), "=&r" (h)
		: "r" (lo), "r" (hi), "r" (p)
		: "cc", "memory", "r2", "r3"
	);
#endif
}

static inline void atomic_store_explicit (volatile atomic_word2* p, atomic_word2 v, memory_order_release_t)
{
#if defined(_MSC_VER)
	atomic_word2 w, x;
	w = v;
	x = v;
	do 
	{
		w.v = _InterlockedCompareExchange64_rel((volatile __int64 *)p, x.v, w.v);
	} while (w.v != x.v);
#else
	register atomic_word l __asm__ ("r2");
	register atomic_word h __asm__ ("r3");
	register atomic_word lo __asm__ ("r0") = v.lo;
	register atomic_word hi __asm__ ("r1") = v.hi;
	atomic_word success;
	
	__asm__ __volatile__
	(
        ASM_REL
	ASM_LABEL (0)
		"ldrexd	%2, %3, [%6]\n\t"
		"strexd	%0, %4, %5, [%6]\n\t"
		"teq	%0, #0\n\t"
		"bne	0b\n\t"
	
		: "=&r" (success), "=m" (*p), "=&r" (l), "=&r" (h)
		: "r" (lo), "r" (hi), "r" (p)
		: "cc", "memory", "r2", "r3"
	);
#endif
}

static inline atomic_word2 atomic_exchange_explicit (volatile atomic_word2* p, atomic_word2 val, memory_order_acq_rel_t)
{
#if defined(_MSC_VER)
	__dmb(_ARM_BARRIER_ISH);
	atomic_word2 w;
	w.v = _InterlockedExchange64_acq ((__int64 volatile*) p, (__int64) val.v);
	return w;
#else
	register atomic_word l __asm__ ("r0");
	register atomic_word h __asm__ ("r1");
	register atomic_word lo __asm__ ("r2") = val.lo;
	register atomic_word hi __asm__ ("r3") = val.hi;
	atomic_word success;
	
	__asm__ __volatile__
	(
		ASM_REL
	ASM_LABEL (0)
		"ldrexd	%2, %3, [%6]\n\t"
		"strexd	%0, %5, %4, [%6]\n\t"
		"teq	%0, #0\n\t"
		"bne	0b\n\t"
		ASM_ISB
		
		: "=&r" (success), "=m" (*p), "=&r" (l), "=&r" (h)
		: "r" (hi), "r" (lo), "r" (p)
		: "cc", "memory", "r0", "r1", "r3"
	);
	val.lo = l;
	val.hi = h;
	
	return val;
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word2* p, atomic_word2* oldval, atomic_word2 newval, memory_order_acquire_t, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	__int64 tmp = _InterlockedCompareExchange64_acq ((volatile __int64*) p, newval.v, oldval->v);
	return oldval->v==tmp ? true : (oldval->v=tmp, false);
#else
	register atomic_word l __asm__ ("r2");
	register atomic_word h __asm__ ("r3");
	register atomic_word lo __asm__ ("r0") = newval.lo;
	register atomic_word hi __asm__ ("r1") = newval.hi;
	atomic_word success;
	
	__asm__ __volatile__
	(
	ASM_LABEL (0)
		"ldrexd	%2, %3, [%8]\n\t"
		"teq	%3, %5\n\t"
		"it		eq\n\t"
		"teqeq	%2, %4\n\t"
		"bne	1f\n\t"
		"strexd	%0, %6, %7, [%8]\n\t"
		"teq	%0, #0\n\t"
		"bne	0b\n\t"
		ASM_ISB
	ASM_LABEL (1)
		"clrex\n\t"
		
		: "=&r" (success), "+m" (*p), "=&r" (l), "=&r" (h)
		: "r" (oldval->lo), "r" (oldval->hi), "r" (lo), "r" (hi), "r" (p), "0" (1)
		: "cc", "memory", "r2", "r3"
	);
	if(success != 0)
	{
		oldval->lo = l;
		oldval->hi = h;
	}
	return success == 0;
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word2* p, atomic_word2* oldval, atomic_word2 newval, memory_order_release_t, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	__int64 tmp = _InterlockedCompareExchange64_rel ((volatile __int64*) p, newval.v, oldval->v);
	return oldval->v==tmp ? true : (oldval->v=tmp, false);
#else
	register atomic_word l __asm__ ("r2");
	register atomic_word h __asm__ ("r3");
	register atomic_word lo __asm__ ("r0") = newval.lo;
	register atomic_word hi __asm__ ("r1") = newval.hi;
	atomic_word success;
	
	__asm__ __volatile__
	(
        ASM_REL
	ASM_LABEL (0)
		"ldrexd	%2, %3, [%8]\n\t"
		"teq	%3, %5\n\t"
		"it		eq\n\t"
		"teqeq	%2, %4\n\t"
		"bne	1f\n\t"
		"strexd	%0, %6, %7, [%8]\n\t"
		"teq	%0, #0\n\t"
		"bne	0b\n\t"
	ASM_LABEL (1)
		"clrex\n\t"
		
		: "=&r" (success), "+m" (*p), "=&r" (l), "=&r" (h)
		: "r" (oldval->lo), "r" (oldval->hi), "r" (lo), "r" (hi), "r" (p), "0" (1)
		: "cc", "memory", "r2", "r3"
	);
	if(success != 0)
	{
		oldval->lo = l;
		oldval->hi = h;
	}
	return success == 0;
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word2* p, atomic_word2* oldval, atomic_word2 newval, memory_order_acq_rel_t, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	__dmb(_ARM_BARRIER_ISH);
	__int64 tmp = _InterlockedCompareExchange64_acq ((volatile __int64*) p, newval.v, oldval->v);
	return oldval->v==tmp ? true : (oldval->v=tmp, false);
#else
	register atomic_word l __asm__ ("r2");
	register atomic_word h __asm__ ("r3");
	register atomic_word lo __asm__ ("r0") = newval.lo;
	register atomic_word hi __asm__ ("r1") = newval.hi;
	atomic_word success;
	
	__asm__ __volatile__
	(
        ASM_REL
	ASM_LABEL (0)
		"ldrexd	%2, %3, [%8]\n\t"
		"teq	%3, %5\n\t"
		"it		eq\n\t"
		"teqeq	%2, %4\n\t"
		"bne	1f\n\t"
		"strexd	%0, %6, %7, [%8]\n\t"
		"teq	%0, #0\n\t"
		"bne	0b\n\t"
		ASM_ISB
	ASM_LABEL (1)
		"clrex\n\t"
		
		: "=&r" (success), "+m" (*p), "=&r" (l), "=&r" (h)
		: "r" (oldval->lo), "r" (oldval->hi), "r" (lo), "r" (hi), "r" (p), "0" (1)
		: "cc", "memory", "r2", "r3"
	);
	if(success != 0)
	{
		oldval->lo = l;
		oldval->hi = h;
	}
	return success == 0;
#endif
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word2* p, atomic_word2* oldval, atomic_word2 newval, memory_order_seq_cst_t, memory_order_relaxed_t)
{
#if defined(_MSC_VER)
	__int64 tmp = _InterlockedCompareExchange64_rel ((volatile __int64*) p, newval.v, oldval->v);
	__dmb(_ARM_BARRIER_ISH);
	return oldval->v==tmp ? true : (oldval->v=tmp, false);
#else
	register atomic_word l __asm__ ("r2");
	register atomic_word h __asm__ ("r3");
	register atomic_word lo __asm__ ("r0") = newval.lo;
	register atomic_word hi __asm__ ("r1") = newval.hi;
	atomic_word success;
	
	__asm__ __volatile__
	(
        ASM_REL
	ASM_LABEL (0)
		"ldrexd	%2, %3, [%8]\n\t"
		"teq	%3, %5\n\t"
		"it		eq\n\t"
		"teqeq	%2, %4\n\t"
		"bne	1f\n\t"
		"strexd	%0, %6, %7, [%8]\n\t"
		"teq	%0, #0\n\t"
		"bne	0b\n\t"
		ASM_DMB_ISH
	ASM_LABEL (1)
		"clrex\n\t"
		
		: "=&r" (success), "+m" (*p), "=&r" (l), "=&r" (h)
		: "r" (oldval->lo), "r" (oldval->hi), "r" (lo), "r" (hi), "r" (p), "0" (1)
		: "cc", "memory", "r2", "r3"
	);
	if(success != 0)
	{
		oldval->lo = l;
		oldval->hi = h;
	}
	return success == 0;
#endif
}
