diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S b/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S index b2a9440..cf6af07 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S @@ -32,13 +32,14 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_cas_16) + dmb ish mov x4, x0 /* we need r0 for return value */ 1: ldxrh w0, [x4] /* load old value */ cmp w0, w1 /* compare? */ b.ne 2f stxrh w3, w2, [x4] /* store new value */ cbnz w3, 1b /* succeed? nope, try again. */ - dmb st /* data memory barrier */ + dmb ishst /* data memory barrier */ 2: ret /* return. */ END(_atomic_cas_16) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S index 7190c81..0f25d56 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S @@ -32,13 +32,14 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_cas_32) + dmb ish mov x4, x0 /* we need r0 for return value */ 1: ldxr w0, [x4] /* load old value */ cmp w0, w1 /* compare? */ b.ne 2f /* return if different */ stxr w3, w2, [x4] /* store new value */ cbnz w3, 1b /* succeed? nope, try again. */ - dmb st + dmb ishst 2: ret /* return. */ END(_atomic_cas_32) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S index 1bc90b6..19ae54d 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S @@ -32,13 +32,14 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_cas_64) + dmb ish mov x4, x0 /* we need r0 for return value */ 1: ldxr x0, [x4] /* load old value */ cmp x0, x1 /* compare? */ b.ne 2f /* return if different */ stxr w3, x2, [x4] /* store new value */ cbnz w3, 1b /* succeed? nope, try again. */ - dmb st + dmb ishst 2: ret /* return. */ END(_atomic_cas_64) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S b/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S index 7c8f139..ec7fa2e 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S @@ -32,13 +32,14 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_cas_8) + dmb ish mov x4, x0 /* we need r0 for return value */ 1: ldxrb w0, [x4] /* load old value */ cmp w0, w1 /* compare? */ b.ne 2f stxrb w3, w2, [x4] /* store new value */ cbnz w3, 1b /* succeed? nope, try again. */ - dmb st /* data memory barrier */ + dmb ishst /* data memory barrier */ 2: ret /* return. */ END(_atomic_cas_8) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S index 6ae4cfb..c69f692 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_dec_32.S @@ -32,11 +32,12 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_dec_32) + dmb ish 1: ldxr w3, [x0] /* load old value (return value) */ sub w3, w3, #1 /* calculate new value */ stxr w2, w3, [x0] /* try to store */ cbnz w2, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_dec_32) @@ -45,12 +46,13 @@ ATOMIC_OP_ALIAS(atomic_dec_uint,_atomic_dec_32) STRONG_ALIAS(_atomic_dec_uint,_atomic_dec_32) ENTRY_NP(_atomic_dec_32_nv) + dmb ish mov x4, x0 /* need r0 for return value */ 1: ldxr w0, [x4] /* load old value */ sub w0, w0, #1 /* calculate new value (return value) */ stxr w3, w0, [x4] /* try to store */ cbnz w3, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_dec_32_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S index cde2968..eb87557 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_dec_64.S @@ -32,11 +32,12 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_dec_64) + dmb ish 1: ldxr x2, [x0] /* load old value (return value) */ sub x2, x2, #1 /* calculate new value */ stxr w3, x2, [x0] /* try to store */ cbnz w3, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_dec_64) @@ -47,12 +48,13 @@ STRONG_ALIAS(_atomic_dec_ptr,_atomic_dec_64) STRONG_ALIAS(_atomic_dec_ulong,_atomic_dec_64) ENTRY_NP(_atomic_dec_64_nv) + dmb ish mov x4, x0 /* need r0 for return value */ 1: ldxr x0, [x4] /* load old value */ sub x0, x0, #1 /* calculate new value (return value) */ stxr w3, x0, [x4] /* try to store */ cbnz w3, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_dec_64_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S index 303693b..9ddd8dd 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_inc_32.S @@ -32,11 +32,12 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_inc_32) + dmb ish 1: ldxr w3, [x0] /* load old value (return value) */ add w3, w3, #1 /* calculate new value */ stxr w2, w3, [x0] /* try to store */ cbnz w2, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_inc_32) @@ -45,12 +46,13 @@ ATOMIC_OP_ALIAS(atomic_inc_uint,_atomic_inc_32) STRONG_ALIAS(_atomic_inc_uint,_atomic_inc_32) ENTRY_NP(_atomic_inc_32_nv) + dmb ish mov x4, x0 /* need r0 for return value */ 1: ldxr w0, [x4] /* load old value */ add w0, w0, #1 /* calculate new value (return value) */ stxr w3, w0, [x4] /* try to store */ cbnz w3, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_inc_32_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S index 0d623c5..cfed1d9 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_inc_64.S @@ -32,11 +32,12 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_inc_64) + dmb ish 1: ldxr x2, [x0] /* load old value (return value) */ add x2, x2, #1 /* calculate new value */ stxr w3, x2, [x0] /* try to store */ cbnz w3, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_inc_64) @@ -47,12 +48,13 @@ STRONG_ALIAS(_atomic_inc_ptr,_atomic_inc_64) STRONG_ALIAS(_atomic_inc_ulong,_atomic_inc_64) ENTRY_NP(_atomic_inc_64_nv) + dmb ish mov x4, x0 /* need r0 for return value */ 1: ldxr x0, [x4] /* load old value */ add x0, x0, #1 /* calculate new value (return value) */ stxr w3, x0, [x4] /* try to store */ cbnz w3, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_inc_64_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S b/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S index 9ac7562..17d47f1 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S @@ -32,13 +32,14 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_nand_16) + dmb ish mov x4, x0 1: ldxrh w0, [x4] /* load old value (to be returned) */ mvn w3, w0 /* complement source */ and w3, w3, w1 /* calculate new value */ stxrh w2, w3, [x4] /* try to store */ cbnz w2, 1b /* succeed? no, try again */ - dmb st + dmb ishst ret /* return old value */ END(_atomic_nand_16) @@ -48,13 +49,14 @@ STRONG_ALIAS(__sync_fetch_and_nand_2,_atomic_nand_16) STRONG_ALIAS(_atomic_nand_ushort,_atomic_nand_16) ENTRY_NP(_atomic_nand_16_nv) + dmb ish mov x4, x0 /* need r0 for return value */ 1: ldxrh w0, [x4] /* load old value */ mvn w0, w0 /* complement source */ and w0, w0, w1 /* calculate new value (return value) */ stxrh w2, w0, [x4] /* try to store */ cbnz w2, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_nand_16_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S index 3b22fcf..1ecb6ee 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S @@ -32,13 +32,14 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_nand_32) + dmb ish mov x4, x0 1: ldxr w0, [x4] /* load old value (to be returned) */ mvn w3, w0 /* complement source */ and w3, w3, w1 /* calculate new value */ stxr w2, w3, [x4] /* try to store */ cbnz w2, 1b /* succeed? no, try again */ - dmb st + dmb ishst ret /* return old value */ END(_atomic_nand_32) @@ -48,13 +49,14 @@ STRONG_ALIAS(__sync_fetch_and_nand_4,_atomic_nand_32) STRONG_ALIAS(_atomic_nand_uint,_atomic_nand_32) ENTRY_NP(_atomic_nand_32_nv) + dmb ish mov x4, x0 /* need r0 for return value */ 1: ldxr w0, [x4] /* load old value */ mvn w0, w0 /* complement source */ and w0, w0, w1 /* calculate new value (return value) */ stxr w2, w0, [x4] /* try to store */ cbnz w2, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_nand_32_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S index 8868fdb..6cffec7 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S @@ -32,13 +32,14 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_nand_64) + dmb ish mov x4, x0 1: ldxr x0, [x4] /* load old value (to be returned) */ mvn x2, x0 /* complement source */ and x2, x2, x1 /* calculate new value */ stxr w3, x2, [x4] /* try to store */ cbnz w3, 1b /* succeed? no, try again */ - dmb st + dmb ishst ret /* return old value */ END(_atomic_nand_64) @@ -48,13 +49,14 @@ STRONG_ALIAS(__sync_fetch_and_nand_8,_atomic_nand_64) STRONG_ALIAS(_atomic_nand_ulong,_atomic_nand_64) ENTRY_NP(_atomic_nand_64_nv) + dmb ish mov x4, x0 /* need r0 for return value */ 1: ldxr x0, [x4] /* load old value */ mvn x0, x0 /* complement source */ and x0, x0, x1 /* calculate new value (return value) */ stxr w3, x0, [x4] /* try to store */ cbnz w3, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_nand_64_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S b/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S index fdd282c..890f431 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S @@ -32,13 +32,14 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_nand_8) + dmb ish mov x4, x0 1: ldxrb w0, [x4] /* load old value (to be returned) */ mvn w3, w0 /* complement source */ and w3, w3, w1 /* calculate new value */ stxrb w2, w3, [x4] /* try to store */ cbnz w2, 1b /* succeed? no, try again */ - dmb st + dmb ishst ret /* return old value */ END(_atomic_nand_8) @@ -48,13 +49,14 @@ STRONG_ALIAS(__sync_fetch_and_nand_1,_atomic_nand_8) STRONG_ALIAS(_atomic_nand_uchar,_atomic_nand_8) ENTRY_NP(_atomic_nand_8_nv) + dmb ish mov x4, x0 /* need r0 for return value */ 1: ldxrb w0, [x4] /* load old value */ mvn w0, w0 /* complement source */ and w0, w0, w1 /* calculate new value (return value) */ stxrb w2, w0, [x4] /* try to store */ cbnz w2, 1b /* succeed? no, try again? */ - dmb st + dmb ishst ret /* return new value */ END(_atomic_nand_8_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h b/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h index f31c43b..64e53d5 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h +++ b/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h @@ -36,89 +36,97 @@ #define ATOMIC_OP8(OP, INSN) \ ENTRY_NP(_atomic_##OP##_8) ;\ + dmb ish ;\ mov x4, x0 ;\ 1: ldxrb w0, [x4] /* load old value */ ;\ INSN w2, w0, w1 /* calculate new value */ ;\ stxrb w3, w2, [x4] /* try to store */ ;\ cbnz w3, 1b /* succeed? no, try again */ ;\ - dmb st ;\ + dmb ishst ;\ ret /* return old value */ ;\ END(_atomic_##OP##_8) #define ATOMIC_OP8_NV(OP, INSN) \ ENTRY_NP(_atomic_##OP##_8_nv) ;\ + dmb ish ;\ mov x4, x0 /* need r0 for return value */ ;\ 1: ldxrb w0, [x4] /* load old value */ ;\ INSN w0, w0, w1 /* calc new (return) value */ ;\ stxrb w3, w0, [x4] /* try to store */ ;\ cbnz w3, 1b /* succeed? no, try again */ ;\ - dmb sy ;\ + dmb ishst ;\ ret /* return new value */ ;\ END(_atomic_##OP##_8_nv) #define ATOMIC_OP16(OP, INSN) \ ENTRY_NP(_atomic_##OP##_16) ;\ + dmb ish ;\ mov x4, x0 ;\ 1: ldxrh w0, [x4] /* load old value */ ;\ INSN w2, w0, w1 /* calculate new value */ ;\ stxrh w3, w2, [x4] /* try to store */ ;\ cbnz w3, 1b /* succeed? no, try again */ ;\ - dmb st ;\ + dmb ishst ;\ ret /* return old value */ ;\ END(_atomic_##OP##_16) #define ATOMIC_OP16_NV(OP, INSN) \ ENTRY_NP(_atomic_##OP##_16_nv) ;\ + dmb ish ;\ mov x4, x0 /* need r0 for return value */ ;\ 1: ldxrh w0, [x4] /* load old value */ ;\ INSN w0, w0, w1 /* calc new (return) value */ ;\ stxrh w3, w0, [x4] /* try to store */ ;\ cbnz w3, 1b /* succeed? no, try again */ ;\ - dmb sy ;\ + dmb ishst ;\ ret /* return new value */ ;\ END(_atomic_##OP##_16_nv) #define ATOMIC_OP32(OP, INSN) \ ENTRY_NP(_atomic_##OP##_32) ;\ + dmb ish ;\ mov x4, x0 ;\ 1: ldxr w0, [x4] /* load old value */ ;\ INSN w2, w0, w1 /* calculate new value */ ;\ stxr w3, w2, [x4] /* try to store */ ;\ cbnz w3, 1b /* succeed? no, try again */ ;\ - dmb st ;\ + dmb ishst ;\ ret /* return old value */ ;\ END(_atomic_##OP##_32) #define ATOMIC_OP32_NV(OP, INSN) \ ENTRY_NP(_atomic_##OP##_32_nv) ;\ + dmb ish ;\ mov x4, x0 /* need r0 for return value */ ;\ 1: ldxr w0, [x4] /* load old value */ ;\ INSN w0, w0, w1 /* calc new (return) value */ ;\ stxr w3, w0, [x4] /* try to store */ ;\ cbnz w3, 1b /* succeed? no, try again? */ ;\ - dmb sy ;\ + dmb ishst ;\ ret /* return new value */ ;\ END(_atomic_##OP##_32_nv) #define ATOMIC_OP64(OP, INSN) \ ENTRY_NP(_atomic_##OP##_64) ;\ + dmb ish ;\ mov x4, x0 ;\ 1: ldxr x0, [x4] /* load old value */ ;\ INSN x2, x0, x1 /* calculate new value */ ;\ stxr w3, x2, [x4] /* try to store */ ;\ cbnz w3, 1b /* succeed? no, try again */ ;\ - dmb st ;\ + dmb ishst ;\ ret /* return old value */ ;\ END(_atomic_##OP##_64) #define ATOMIC_OP64_NV(OP, INSN) \ ENTRY_NP(_atomic_##OP##_64_nv) ;\ + dmb ish ;\ mov x4, x0 /* need r0 for return value */ ;\ 1: ldxr x0, [x4] /* load old value */ ;\ INSN x0, x0, x1 /* calc new (return) value */ ;\ stxr w3, x0, [x4] /* try to store */ ;\ cbnz w3, 1b /* succeed? no, try again? */ ;\ - dmb sy ;\ + dmb ishst ;\ ret /* return new value */ ;\ END(_atomic_##OP##_64_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S b/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S index 5b90dc7..c5c1bf6 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S @@ -32,11 +32,12 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_swap_16) + dmb ish mov x4, x0 1: ldxrh w0, [x4] stxrh w3, w1, [x4] cbnz w3, 1b - dmb st + dmb ishst ret END(_atomic_swap_16) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S index 42aa904..039d3b0 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S @@ -32,11 +32,12 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_swap_32) + dmb ish mov x4, x0 1: ldxr w0, [x4] stxr w3, w1, [x4] cbnz w3, 1b - dmb st + dmb ishst ret END(_atomic_swap_32) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S index d950d80..ff04f98 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S @@ -32,11 +32,12 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_swap_64) + dmb ish mov x4, x0 1: ldxr x0, [x4] stxr w3, x1, [x4] cbnz w3, 1b - dmb st + dmb ishst ret END(_atomic_swap_64) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S b/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S index 4f25f44..c17216a 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S @@ -32,11 +32,12 @@ #include "atomic_op_asm.h" ENTRY_NP(_atomic_swap_8) + dmb ish mov x4, x0 1: ldxrb w0, [x4] stxrb w3, w1, [x4] cbnz w3, 1b - dmb st + dmb ishst ret END(_atomic_swap_8)