#define ASM_DMB_ISH         "dmb    ish\n\t"

#if defined(__ARM_ARCH_7S__)
	// this is sufficient for Swift processors
#	define ASM_REL         "dmb    ishst\n\t"
#else
#	define ASM_REL         "dmb    ish\n\t"
#endif

static inline void atomic_pause ()
{
}

static inline void atomic_thread_fence (memory_order_relaxed_t)
{
}

static inline void atomic_thread_fence (memory_order_acquire_t)
{
    __asm__ __volatile__ ("dmb ld\n\t" : : : "memory");
}

static inline void atomic_thread_fence (memory_order_release_t)
{
    __asm__ __volatile__ (ASM_REL : : : "memory");
}

static inline void atomic_thread_fence (memory_order_acq_rel_t)
{
    __asm__ __volatile__ (ASM_DMB_ISH : : : "memory");
}

static inline void atomic_thread_fence (int /* memory_order_seq_cst_t */)
{
    __asm__ __volatile__ (ASM_DMB_ISH : : : "memory");
}

#define ATOMIC_LOAD(opc) \
	atomic_word res; \
	__asm__ __volatile__ \
	( \
		opc "   %0, %1\n\t" \
		: "=r" (res) \
		: "m" (*p) \
	); \
	return res;

/*
 *	int support
 */

static inline int atomic_load_explicit (const volatile int* p, memory_order_relaxed_t)
{
	int res;
	__asm__ __volatile__
	(
		"ldr   %w0, %w1\n\t"
		: "=r" (res)
		: "m" (*p)
	);
	return res;
}

static inline int atomic_load_explicit (const volatile int* p, memory_order_acquire_t)
{
	int res;
	__asm__ __volatile__
	(
		"ldar   %w0, %w1\n\t"
		: "=r" (res)
		: "m" (*p)
	);
	return res;
}

static inline int atomic_load_explicit (const volatile int* p, int /* memory_order_seq_cst_t */)
{
	int res;
	__asm__ __volatile__
	(
		"ldar   %w0, %w1\n\t"
		: "=r" (res)
		: "m" (*p)
	);
	return res;
}

/*
 *	native word support
 */

static inline atomic_word atomic_load_explicit (const volatile atomic_word* p, memory_order_relaxed_t)
{
    ATOMIC_LOAD("ldr")
}

static inline atomic_word atomic_load_explicit (const volatile atomic_word* p, memory_order_acquire_t)
{
    ATOMIC_LOAD("ldar")
}

static inline atomic_word atomic_load_explicit (const volatile atomic_word* p, int /* memory_order_seq_cst_t */)
{
    ATOMIC_LOAD("ldar")
}

#define ATOMIC_STORE(opc) \
	__asm__ __volatile__ \
	( \
		opc "   %1, %0\n\t" \
		: "=m" (*p) \
		: "r" (v) \
		: "memory" \
	);

/*
 *	int support
 */

static inline void atomic_store_explicit (volatile int* p, int v, memory_order_relaxed_t)
{
	__asm__ __volatile__
	(
		"str   %w1, %w0\n\t"
		: "=m" (*p)
		: "r" (v)
		: "memory"
	);
}

static inline void atomic_store_explicit (volatile int* p, int v, memory_order_release_t)
{
	__asm__ __volatile__
	(
		"stlr   %w1, %w0\n\t"
		: "=m" (*p)
		: "r" (v)
		: "memory"
	);
}

static inline void atomic_store_explicit (volatile int* p, int v, int /* memory_order_seq_cst_t */)
{
	__asm__ __volatile__
	(
		"stlr   %w1, %w0\n\t"
		: "=m" (*p)
		: "r" (v)
		: "memory"
	);
}

/*
 *	native word support
 */

static inline void atomic_store_explicit (volatile atomic_word* p, atomic_word v, memory_order_relaxed_t)
{
    ATOMIC_STORE("str")
}

static inline void atomic_store_explicit (volatile atomic_word* p, atomic_word v, memory_order_release_t)
{
    ATOMIC_STORE("stlr")
}

static inline void atomic_store_explicit (volatile atomic_word* p, atomic_word v, int /* memory_order_seq_cst_t */)
{
    ATOMIC_STORE("stlr")
}

#define ATOMIC_XCHG(LD, ST) \
	atomic_word res; \
	atomic_word success; \
	__asm__ __volatile__ \
	( \
	"0:\n\t" \
		LD "    %2, [%4]\n\t" \
		ST "    %w0, %3, [%4]\n\t" \
		"cbnz	%w0, 0b\n\t" \
		: "=&r" (success), "+m" (*p), "=&r" (res) \
		: "r" (v), "r" (p) \
		: "memory" \
	); \
	return res;

static inline atomic_word atomic_exchange_explicit (volatile atomic_word* p, atomic_word v, memory_order_relaxed_t)
{
    ATOMIC_XCHG("ldxr", "stxr")
}

static inline atomic_word atomic_exchange_explicit (volatile atomic_word* p, atomic_word v, memory_order_acquire_t)
{
    ATOMIC_XCHG("ldaxr", "stxr")
}

static inline atomic_word atomic_exchange_explicit (volatile atomic_word* p, atomic_word v, memory_order_release_t)
{
    ATOMIC_XCHG("ldxr", "stlxr")
}

static inline atomic_word atomic_exchange_explicit (volatile atomic_word* p, atomic_word v, memory_order_acq_rel_t)
{
    ATOMIC_XCHG("ldaxr", "stlxr")
}

static inline atomic_word atomic_exchange_explicit (volatile atomic_word* p, atomic_word v, int /* memory_order_seq_cst_t */)
{
    ATOMIC_XCHG("ldaxr", "stlxr")
}

// atomic_compare_exchange_weak_explicit: can fail spuriously even if *p == *oldval

#define ATOMIC_CMP_XCHG(LD, ST) \
	atomic_word res; \
	atomic_word success = 0; \
	__asm__ __volatile__ \
	( \
		LD "    %2, [%4]\n\t" \
        "cmp    %2, %5\n\t" \
        "b.ne   1f\n\t" \
		ST "    %w0, %3, [%4]\n" \
    "1:\n\t" \
		"clrex\n\t" \
		: "=&r" (success), "+m" (*p), "=&r" (res) \
		: "r" (newval), "r" (p), "r" (*oldval) \
		: "cc", "memory" \
	); \
    *oldval = res; \
	return success == 0;


static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_relaxed_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG("ldxr", "stxr")
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acquire_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG("ldaxr", "stxr")
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_release_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG("ldxr", "stlxr")
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acq_rel_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG("ldaxr", "stlxr")
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, int /* memory_order_seq_cst_t */, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG("ldaxr", "stlxr")
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acquire_t, memory_order_acquire_t)
{
    ATOMIC_CMP_XCHG("ldaxr", "stxr")
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_release_t, memory_order_release_t)
{
    ATOMIC_CMP_XCHG("ldxr", "stlxr")
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acq_rel_t, memory_order_acq_rel_t)
{
    ATOMIC_CMP_XCHG("ldaxr", "stlxr")
}

static inline bool atomic_compare_exchange_weak_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, int /* memory_order_seq_cst_t */, int /* memory_order_seq_cst_t */)
{
    ATOMIC_CMP_XCHG("ldaxr", "stlxr")
}

// atomic_compare_exchange_strong_explicit: does loop and only returns false if *p != *oldval

#undef ATOMIC_CMP_XCHG

#define ATOMIC_CMP_XCHG(LD, ST) \
	atomic_word res; \
	atomic_word success = 0; \
	__asm__ __volatile__ \
	( \
    "0:\n\t" \
		LD "    %2, [%4]\n\t" \
        "cmp    %2, %5\n\t" \
        "b.ne   1f\n\t" \
		ST "    %w0, %3, [%4]\n" \
        "cbnz   %w0, 0b\n\t" \
    "1:\n\t" \
		"clrex\n\t" \
		: "=&r" (success), "+m" (*p), "=&r" (res) \
		: "r" (newval), "r" (p), "r" (*oldval) \
		: "cc", "memory" \
	); \
    *oldval = res; \
	return success == 0;

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_relaxed_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG("ldxr", "stxr")
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acquire_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG("ldaxr", "stxr")
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_release_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG("ldxr", "stlxr")
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acq_rel_t, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG("ldaxr", "stlxr")
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, int /* memory_order_seq_cst_t */, memory_order_relaxed_t)
{
    ATOMIC_CMP_XCHG("ldaxr", "stlxr")
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acquire_t, memory_order_acquire_t)
{
    ATOMIC_CMP_XCHG("ldaxr", "stxr")
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_release_t, memory_order_release_t)
{
    ATOMIC_CMP_XCHG("ldxr", "stlxr")
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, memory_order_acq_rel_t, memory_order_acq_rel_t)
{
    ATOMIC_CMP_XCHG("ldaxr", "stlxr")
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word* p, atomic_word *oldval, atomic_word newval, int /* memory_order_seq_cst_t */, int /* memory_order_seq_cst_t */)
{
    ATOMIC_CMP_XCHG("ldaxr", "stlxr")
}

#define ATOMIC_PFIX_int			"%w"
#define ATOMIC_PFIX_atomic_word	"%"
#define ATOMIC_PFIX(WORD)		ATOMIC_PFIX_##WORD

#define ATOMIC_OP(WORD, LD, ST, OP) \
	long long res, tmp; \
	int success; \
	__asm__ __volatile__ \
	( \
    "0:\n\t" \
		LD "    " ATOMIC_PFIX(WORD) "2, [%5]\n\t" \
        OP "    " ATOMIC_PFIX(WORD)	"3, " ATOMIC_PFIX(WORD) "2, " ATOMIC_PFIX(WORD) "4\n\t" \
		ST "    %w0, " ATOMIC_PFIX(WORD) "3, [%5]\n" \
        "cbnz   %w0, 0b\n\t" \
		: "=&r" (success), "+m" (*p), "=&r" (res), "=&r" (tmp) \
		: "Ir" ((long long) v), "r" (p) \
		: "cc", "memory" \
	); \
	return (WORD) res;

static inline int atomic_fetch_add_explicit (volatile int* p, int v, memory_order_relaxed_t)
{
    ATOMIC_OP (int, "ldxr", "stxr", "add")
}

static inline int atomic_fetch_add_explicit (volatile int* p, int v, memory_order_acquire_t)
{
    ATOMIC_OP (int, "ldaxr", "stxr", "add")
}

static inline int atomic_fetch_add_explicit (volatile int* p, int v, memory_order_release_t)
{
    ATOMIC_OP (int, "ldxr", "stlxr", "add")
}

static inline int atomic_fetch_add_explicit (volatile int* p, int v, memory_order_acq_rel_t)
{
    ATOMIC_OP (int, "ldaxr", "stlxr", "add")
}

static inline int atomic_fetch_add_explicit (volatile int* p, int v, int /* memory_order_seq_cst_t */)
{
    ATOMIC_OP (int, "ldaxr", "stlxr", "add")
}

static inline atomic_word atomic_fetch_add_explicit (volatile atomic_word* p, atomic_word v, memory_order_relaxed_t)
{
    ATOMIC_OP (atomic_word, "ldxr", "stxr", "add")
}

static inline atomic_word atomic_fetch_add_explicit (volatile atomic_word* p, atomic_word v, memory_order_acquire_t)
{
    ATOMIC_OP (atomic_word, "ldaxr", "stxr", "add")
}

static inline atomic_word atomic_fetch_add_explicit (volatile atomic_word* p, atomic_word v, memory_order_release_t)
{
    ATOMIC_OP (atomic_word, "ldxr", "stlxr", "add")
}

static inline atomic_word atomic_fetch_add_explicit (volatile atomic_word* p, atomic_word v, memory_order_acq_rel_t)
{
    ATOMIC_OP (atomic_word, "ldaxr", "stlxr", "add")
}

static inline atomic_word atomic_fetch_add_explicit (volatile atomic_word* p, atomic_word v, int /* memory_order_seq_cst_t */)
{
    ATOMIC_OP (atomic_word, "ldaxr", "stlxr", "add")
}

static inline int atomic_fetch_sub_explicit (volatile int* p, int v, memory_order_relaxed_t)
{
    ATOMIC_OP (int, "ldxr", "stxr", "sub")
}

static inline int atomic_fetch_sub_explicit (volatile int* p, int v, memory_order_acquire_t)
{
    ATOMIC_OP (int, "ldaxr", "stxr", "sub")
}

static inline int atomic_fetch_sub_explicit (volatile int* p, int v, memory_order_release_t)
{
    ATOMIC_OP (int, "ldxr", "stlxr", "sub")
}

static inline int atomic_fetch_sub_explicit (volatile int* p, int v, memory_order_acq_rel_t)
{
    ATOMIC_OP (int, "ldaxr", "stlxr", "sub")
}

static inline int atomic_fetch_sub_explicit (volatile int* p, int v, int /* memory_order_seq_cst_t */)
{
    ATOMIC_OP (int, "ldaxr", "stlxr", "sub")
}

static inline atomic_word atomic_fetch_sub_explicit (volatile atomic_word* p, atomic_word v, memory_order_relaxed_t)
{
    ATOMIC_OP (atomic_word, "ldxr", "stxr", "sub")
}

static inline atomic_word atomic_fetch_sub_explicit (volatile atomic_word* p, atomic_word v, memory_order_acquire_t)
{
    ATOMIC_OP (atomic_word, "ldaxr", "stxr", "sub")
}

static inline atomic_word atomic_fetch_sub_explicit (volatile atomic_word* p, atomic_word v, memory_order_release_t)
{
    ATOMIC_OP (atomic_word, "ldxr", "stlxr", "sub")
}

static inline atomic_word atomic_fetch_sub_explicit (volatile atomic_word* p, atomic_word v, memory_order_acq_rel_t)
{
    ATOMIC_OP (atomic_word, "ldaxr", "stlxr", "sub")
}

static inline atomic_word atomic_fetch_sub_explicit (volatile atomic_word* p, atomic_word v, int /* memory_order_seq_cst_t */)
{
    ATOMIC_OP (atomic_word, "ldaxr", "stlxr", "sub")
}

/*
 *  extensions
 */

static inline void atomic_retain (volatile int* p)
{
    atomic_fetch_add_explicit (p, 1, memory_order_relaxed);
}

static inline bool atomic_release (volatile int* p)
{
	bool res = atomic_fetch_sub_explicit (p, 1, memory_order_release) == 1;
    if (res)
	{
		atomic_thread_fence (memory_order_acquire);
	}
	return res;
}

/*
 *	double word
 */

// Note: the only way to get atomic 128-bit memory accesses on ARM64 is to use ldxp/stxp with a loop
// (ldxp and stxp instructions are not guaranteed to appear atomic)

static inline atomic_word2 atomic_load_explicit (const volatile atomic_word2* p, memory_order_relaxed_t)
{
	atomic_word2 v;
	atomic_word success;
	__asm__ __volatile__
	(
	"0:\n\t"
		"ldxp	%1, %2, [%3]\n\t"
		"stxp	%w0, %1, %2, [%3]\n\t"
		"cbnz	%w0, 0b\n\t"
	
		: "=&r" (success), "=&r" (v.lo), "=&r" (v.hi)
		: "r" (p)
	);
	return v;
}

static inline atomic_word2 atomic_load_explicit (const volatile atomic_word2* p, memory_order_acquire_t)
{
	atomic_word2 v;
	atomic_word success;
	__asm__ __volatile__
	(
	"0:\n\t"
		"ldaxp	%1, %2, [%3]\n\t"
		"stxp	%w0, %1, %2, [%3]\n\t"
		"cbnz	%w0, 0b\n\t"
	
		: "=&r" (success), "=&r" (v.lo), "=&r" (v.hi)
		: "r" (p)
	);
	return v;
}

static inline void atomic_store_explicit (volatile atomic_word2* p, atomic_word2 v, memory_order_relaxed_t)
{
	atomic_word lo;
	atomic_word hi;
	atomic_word success;
	__asm__ __volatile__
	(
	"0:\n\t"
		"ldxp	%2, %3, [%6]\n\t"
		"stxp	%w0, %4, %5, [%6]\n\t"
		"cbnz	%w0, 0b\n\t"
	
		: "=&r" (success), "=m" (*p), "=&r" (lo), "=&r" (hi)
		: "r" (v.lo), "r" (v.hi), "r" (p)
		: "memory"
	);
}

static inline void atomic_store_explicit (volatile atomic_word2* p, atomic_word2 v, memory_order_release_t)
{
	atomic_word lo;
	atomic_word hi;
	atomic_word success;
	__asm__ __volatile__
	(
	"0:\n\t"
		"ldxp	%2, %3, [%6]\n\t"
		"stlxp	%w0, %4, %5, [%6]\n\t"
		"cbnz	%w0, 0b\n\t"
	
		: "=&r" (success), "=m" (*p), "=&r" (lo), "=&r" (hi)
		: "r" (v.lo), "r" (v.hi), "r" (p)
		: "memory"
	);
}

static inline atomic_word2 atomic_exchange_explicit (volatile atomic_word2* p, atomic_word2 val, memory_order_acq_rel_t)
{
	atomic_word2 oldval;
	atomic_word success;
	__asm__ __volatile__
	(
	"0:\n\t"
		"ldaxp	%2, %3, [%6]\n\t"
		"stlxp	%w0, %5, %4, [%6]\n\t"
		"cbnz	%w0, 0b\n\t"
	
		: "=&r" (success), "+m" (*p), "=&r" (oldval.lo), "=&r" (oldval.hi)
		: "r" (val.hi), "r" (val.lo), "r" (p)
		: "memory"
	);

	return oldval;
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word2* p, atomic_word2* oldval, atomic_word2 newval, memory_order_acquire_t, memory_order_relaxed_t)
{
	atomic_word lo = oldval->lo;
	atomic_word hi = oldval->hi;
	atomic_word success;
	__asm__ __volatile__
	(
	"0:\n\t"
		"ldaxp	%2, %3, [%8]\n\t"
		"cmp	%3, %5\n\t"
		"b.ne	1f\n\t"
		"cmp	%2, %4\n\t"
		"b.ne	1f\n\t"
		"stxp	%w0, %6, %7, [%8]\n\t"
		"cbnz	%w0, 0b\n\t"
	"1:\n\t"
		"clrex\n\t"
	
		: "=&r" (success), "+m" (*p), "=&r" (oldval->lo), "=&r" (oldval->hi)
		: "r" (lo), "r" (hi), "r" (newval.lo), "r" (newval.hi), "r" (p), "0" (1)
		: "cc", "memory"
	);

	return success == 0;
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word2* p, atomic_word2* oldval, atomic_word2 newval, memory_order_release_t, memory_order_relaxed_t)
{
	atomic_word lo = oldval->lo;
	atomic_word hi = oldval->hi;
	atomic_word success;
	__asm__ __volatile__
	(
	"0:\n\t"
		"ldxp	%2, %3, [%8]\n\t"
		"cmp	%3, %5\n\t"
		"b.ne	1f\n\t"
		"cmp	%2, %4\n\t"
		"b.ne	1f\n\t"
		"stlxp	%w0, %6, %7, [%8]\n\t"
		"cbnz	%w0, 0b\n\t"
	"1:\n\t"
		"clrex\n\t"
	
		: "=&r" (success), "+m" (*p), "=&r" (oldval->lo), "=&r" (oldval->hi)
		: "r" (lo), "r" (hi), "r" (newval.lo), "r" (newval.hi), "r" (p), "0" (1)
		: "cc", "memory"
	);

	return success == 0;
}

static inline bool atomic_compare_exchange_strong_explicit (volatile atomic_word2* p, atomic_word2* oldval, atomic_word2 newval, int /*memory_order_acq_rel_t*/, memory_order_relaxed_t)
{
	atomic_word lo = oldval->lo;
	atomic_word hi = oldval->hi;
	atomic_word success;
	__asm__ __volatile__
	(
	"0:\n\t"
		"ldaxp	%2, %3, [%8]\n\t"
		"cmp	%3, %5\n\t"
		"b.ne	1f\n\t"
		"cmp	%2, %4\n\t"
		"b.ne	1f\n\t"
		"stlxp	%w0, %6, %7, [%8]\n\t"
		"cbnz	%w0, 0b\n\t"
	"1:\n\t"
		"clrex\n\t"
	
		: "=&r" (success), "+m" (*p), "=&r" (oldval->lo), "=&r" (oldval->hi)
		: "r" (lo), "r" (hi), "r" (newval.lo), "r" (newval.hi), "r" (p), "0" (1)
		: "cc", "memory"
	);

	return success == 0;
}

