diff --git a/common/lib/libc/arch/aarch64/atomic/Makefile.inc b/common/lib/libc/arch/aarch64/atomic/Makefile.inc index ec0685d8cca2..bc00d6cf4679 100644 --- a/common/lib/libc/arch/aarch64/atomic/Makefile.inc +++ b/common/lib/libc/arch/aarch64/atomic/Makefile.inc @@ -1,8 +1,7 @@ # $NetBSD: Makefile.inc,v 1.4 2021/04/27 09:14:24 skrll Exp $ -.if defined(LIB) && (${LIB} == "kern" || ${LIB} == "c" || ${LIB} == "pthread" \ - || ${LIB} == "rump") - +.if defined(LIB) +.if (${LIB} == "c" || ${LIB} == "pthread" || ${LIB} == "rump" || ${LIB} == "kern") .for op in add and cas nand or sub swap xor .for sz in 8 16 32 64 SRCS.atomic+= atomic_${op}_${sz}.S @@ -12,6 +11,7 @@ SRCS.atomic+= atomic_dec_32.S atomic_dec_64.S SRCS.atomic+= atomic_inc_32.S atomic_inc_64.S SRCS.atomic+= membar_ops.S #and cas nand or sub swap xor +.if (${LIB} == "kern") .for op in swp cas clr set eor add .for sz in 1 2 4 8 .for ar in _relax _acq _rel _acq_rel @@ -30,6 +30,7 @@ __aarch64_${op}${ar}.S: __aarch64_lse.S SRCS.gen+= __aarch64_${op}${ar}.S .endfor .endfor +.endif #.for op in add and nand or sub xor #SRCS.atomic+= sync_fetch_and_${op}_8.S #.endfor @@ -37,6 +38,7 @@ SRCS.gen+= __aarch64_${op}${ar}.S #SRCS.atomic+= sync_bool_compare_and_swap_${sz}.S #.endfor +.endif .endif SRCS.atomic+= atomic_init_cas.c diff --git a/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S b/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S index 8fa85e9102d5..1ea50979c100 100644 --- a/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S +++ b/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S @@ -204,4 +204,3 @@ ENTRY_NP(INSN_FUNC) 2: b 1b END(INSN_FUNC) #endif - diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S b/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S index a088a2bb070f..b7b5a4254a43 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_16.S @@ -33,14 +33,16 @@ ATOMIC_OP16(add, add) +SYNC_FETCH_OP16(add, add) + ATOMIC_OP_ALIAS(atomic_add_16,_atomic_add_16) ATOMIC_OP_ALIAS(atomic_add_short,_atomic_add_16) -STRONG_ALIAS(__sync_fetch_and_add_2,_atomic_add_16) STRONG_ALIAS(_atomic_add_short,_atomic_add_16) ATOMIC_OP16_NV(add, add) +SYNC_OP16_FETCH(add, add) + ATOMIC_OP_ALIAS(atomic_add_16_nv,_atomic_add_16_nv) ATOMIC_OP_ALIAS(atomic_add_short_nv,_atomic_add_16_nv) -STRONG_ALIAS(__sync_add_and_fetch_2,_atomic_add_16_nv) STRONG_ALIAS(_atomic_add_short_nv,_atomic_add_16_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S index c157f6895a5f..31455d010206 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_32.S @@ -33,14 +33,16 @@ ATOMIC_OP32(add, add) +SYNC_FETCH_OP32(add, add) + ATOMIC_OP_ALIAS(atomic_add_32,_atomic_add_32) ATOMIC_OP_ALIAS(atomic_add_int,_atomic_add_32) -STRONG_ALIAS(__sync_fetch_and_add_4,_atomic_add_32) STRONG_ALIAS(_atomic_add_int,_atomic_add_32) ATOMIC_OP32_NV(add, add) +SYNC_OP32_FETCH(add, add) + ATOMIC_OP_ALIAS(atomic_add_32_nv,_atomic_add_32_nv) ATOMIC_OP_ALIAS(atomic_add_int_nv,_atomic_add_32_nv) -STRONG_ALIAS(__sync_add_and_fetch_4,_atomic_add_32_nv) STRONG_ALIAS(_atomic_add_int_nv,_atomic_add_32_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S index d815b8db456a..c389f7609168 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_64.S @@ -33,18 +33,20 @@ ATOMIC_OP64(add, add) +SYNC_FETCH_OP64(add, add) + ATOMIC_OP_ALIAS(atomic_add_64_nv,_atomic_add_64_nv) ATOMIC_OP_ALIAS(atomic_add_ptr_nv,_atomic_add_64_nv) ATOMIC_OP_ALIAS(atomic_add_long_nv,_atomic_add_64_nv) STRONG_ALIAS(_atomic_add_ptr_nv,_atomic_add_64_nv) STRONG_ALIAS(_atomic_add_long_nv,_atomic_add_64_nv) -STRONG_ALIAS(__sync_add_and_fetch_8,_atomic_add_64_nv) ATOMIC_OP64_NV(add, add) +SYNC_OP64_FETCH(add, add) + ATOMIC_OP_ALIAS(atomic_add_64,_atomic_add_64) ATOMIC_OP_ALIAS(atomic_add_ptr,_atomic_add_64) ATOMIC_OP_ALIAS(atomic_add_long,_atomic_add_64) STRONG_ALIAS(_atomic_add_ptr,_atomic_add_64) STRONG_ALIAS(_atomic_add_long,_atomic_add_64) -STRONG_ALIAS(__sync_fetch_and_add_8,_atomic_add_64) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S b/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S index 3dc5a2a55e42..6360b63af7b6 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_add_8.S @@ -33,14 +33,16 @@ ATOMIC_OP8(add, add) +SYNC_FETCH_OP8(add, add) + ATOMIC_OP_ALIAS(atomic_add_8,_atomic_add_8) ATOMIC_OP_ALIAS(atomic_add_char,_atomic_add_8) -STRONG_ALIAS(__sync_fetch_and_add_1,_atomic_add_8) STRONG_ALIAS(_atomic_add_char,_atomic_add_8) ATOMIC_OP8_NV(add, add) +SYNC_OP8_FETCH(add, add) + ATOMIC_OP_ALIAS(atomic_add_8_nv,_atomic_add_8_nv) ATOMIC_OP_ALIAS(atomic_add_char_nv,_atomic_add_8_nv) -STRONG_ALIAS(__sync_add_and_fetch_1,_atomic_add_8_nv) STRONG_ALIAS(_atomic_add_char_nv,_atomic_add_8_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S b/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S index f7427fe45b06..9e391c4c228d 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_16.S @@ -33,14 +33,16 @@ ATOMIC_OP16(and, and) +SYNC_FETCH_OP16(and, and) + ATOMIC_OP_ALIAS(atomic_and_16,_atomic_and_16) ATOMIC_OP_ALIAS(atomic_and_ushort,_atomic_and_16) -STRONG_ALIAS(__sync_fetch_and_and_2,_atomic_and_16) STRONG_ALIAS(_atomic_and_ushort,_atomic_and_16) ATOMIC_OP16_NV(and, and) +SYNC_OP16_FETCH(and, and) + ATOMIC_OP_ALIAS(atomic_and_16_nv,_atomic_and_16_nv) ATOMIC_OP_ALIAS(atomic_and_ushort_nv,_atomic_and_16_nv) -STRONG_ALIAS(__sync_and_and_fetch_2,_atomic_and_16_nv) STRONG_ALIAS(_atomic_and_ushort_nv,_atomic_and_16_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S index 44ea5f04a589..4344d8424fdb 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_32.S @@ -33,14 +33,16 @@ ATOMIC_OP32(and, and) +SYNC_FETCH_OP32(and, and) + ATOMIC_OP_ALIAS(atomic_and_32,_atomic_and_32) ATOMIC_OP_ALIAS(atomic_and_uint,_atomic_and_32) -STRONG_ALIAS(__sync_fetch_and_and_4,_atomic_and_32) STRONG_ALIAS(_atomic_and_uint,_atomic_and_32) ATOMIC_OP32_NV(and, and) +SYNC_OP32_FETCH(and, and) + ATOMIC_OP_ALIAS(atomic_and_32_nv,_atomic_and_32_nv) ATOMIC_OP_ALIAS(atomic_and_uint_nv,_atomic_and_32_nv) -STRONG_ALIAS(__sync_and_and_fetch_4,_atomic_and_32_nv) STRONG_ALIAS(_atomic_and_uint_nv,_atomic_and_32_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S index 9e3a86eb0c35..9e75de64eebe 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_64.S @@ -33,14 +33,16 @@ ATOMIC_OP64(and, and) +SYNC_FETCH_OP64(and, and) + ATOMIC_OP_ALIAS(atomic_and_64,_atomic_and_64) ATOMIC_OP_ALIAS(atomic_and_ulong,_atomic_and_64) STRONG_ALIAS(_atomic_and_ulong,_atomic_and_64) -STRONG_ALIAS(__sync_fetch_and_and_8,_atomic_and_64) ATOMIC_OP64_NV(and, and) +SYNC_OP64_FETCH(and, and) + ATOMIC_OP_ALIAS(atomic_and_64_nv,_atomic_and_64_nv) ATOMIC_OP_ALIAS(atomic_and_ulong_nv,_atomic_and_64_nv) STRONG_ALIAS(_atomic_and_ulong_nv,_atomic_and_64_nv) -STRONG_ALIAS(__sync_and_and_fetch_8,_atomic_and_64_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S b/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S index bad4e7e02cf2..1c21f56eec79 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_and_8.S @@ -33,14 +33,16 @@ ATOMIC_OP8(and, and) +SYNC_FETCH_OP8(and, and) + ATOMIC_OP_ALIAS(atomic_and_8,_atomic_and_8) ATOMIC_OP_ALIAS(atomic_and_uchar,_atomic_and_8) -STRONG_ALIAS(__sync_fetch_and_and_1,_atomic_and_8) STRONG_ALIAS(_atomic_and_uchar,_atomic_and_8) ATOMIC_OP8_NV(and, and) +SYNC_OP8_FETCH(and, and) + ATOMIC_OP_ALIAS(atomic_and_8_nv,_atomic_and_8_nv) ATOMIC_OP_ALIAS(atomic_and_uchar_nv,_atomic_and_8_nv) -STRONG_ALIAS(__sync_and_and_fetch_1,_atomic_and_8_nv) STRONG_ALIAS(_atomic_and_uchar_nv,_atomic_and_8_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S b/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S index a499ff6dbef9..56cd52f2ebc5 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_16.S @@ -46,3 +46,14 @@ ATOMIC_OP_ALIAS(atomic_cas_16,_atomic_cas_16) STRONG_ALIAS(_atomic_cas_short,_atomic_cas_16) STRONG_ALIAS(_atomic_cas_ushort,_atomic_cas_16) STRONG_ALIAS(__sync_val_compare_and_swap_2,_atomic_cas_16) + +ENTRY_NP(__sync_val_compare_and_swap_2) + mov x4, x0 /* we need x0 for return value */ +1: ldaxrh w0, [x4] /* load (acquire) old value */ + cmp w0, w1 /* compare? */ + b.ne 2f + stlxrh w3, w2, [x4] /* store (release) new value */ + cbnz w3, 3f /* succeed? nope, try again. */ +2: ret /* return. */ +3: b 1b +END(__sync_val_compare_and_swap_2) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S index ec538f16b70f..9b365e30707e 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_32.S @@ -46,7 +46,17 @@ ATOMIC_OP_ALIAS(atomic_cas_32,_atomic_cas_32) ATOMIC_OP_ALIAS(atomic_cas_uint,_atomic_cas_32) ATOMIC_OP_ALIAS(atomic_cas_32_ni,_atomic_cas_32) ATOMIC_OP_ALIAS(atomic_cas_uint_ni,_atomic_cas_32) -STRONG_ALIAS(__sync_val_compare_and_swap_4,_atomic_cas_32) STRONG_ALIAS(_atomic_cas_uint,_atomic_cas_32) STRONG_ALIAS(_atomic_cas_32_ni,_atomic_cas_32) STRONG_ALIAS(_atomic_cas_uint_ni,_atomic_cas_32) + +ENTRY_NP(__sync_val_compare_and_swap_4) + mov x4, x0 /* we need x0 for return value */ +1: ldaxr w0, [x4] /* load (acquire) old value */ + cmp w0, w1 /* compare? */ + b.ne 2f /* return if different */ + stlxr w3, w2, [x4] /* store (release) new value */ + cbnz w3, 3f /* succeed? nope, try again. */ +2: ret /* return. */ +3: b 1b +END(__sync_val_compare_and_swap_4) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S index a46545dd050f..9839f1be0fc8 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_64.S @@ -48,9 +48,19 @@ ATOMIC_OP_ALIAS(atomic_cas_ptr,_atomic_cas_64) ATOMIC_OP_ALIAS(atomic_cas_64_ni,_atomic_cas_64) ATOMIC_OP_ALIAS(atomic_cas_ulong_ni,_atomic_cas_64) ATOMIC_OP_ALIAS(atomic_cas_ptr_ni,_atomic_cas_64) -STRONG_ALIAS(__sync_val_compare_and_swap_8,_atomic_cas_64) STRONG_ALIAS(_atomic_cas_ulong,_atomic_cas_64) STRONG_ALIAS(_atomic_cas_ptr,_atomic_cas_64) STRONG_ALIAS(_atomic_cas_64_ni,_atomic_cas_64) STRONG_ALIAS(_atomic_cas_ulong_ni,_atomic_cas_64) STRONG_ALIAS(_atomic_cas_ptr_ni,_atomic_cas_64) + +ENTRY_NP(__sync_val_compare_and_swap_8) + mov x4, x0 /* we need x0 for return value */ +1: ldaxr x0, [x4] /* load (acquite) old value */ + cmp x0, x1 /* compare? */ + b.ne 2f /* return if different */ + stlxr w3, x2, [x4] /* store (release) new value */ + cbnz w3, 3f /* succeed? nope, try again. */ +2: ret /* return. */ +3: b 1b +END(__sync_val_compare_and_swap_8) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S b/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S index 939ff2a8db3a..56cb50a73a24 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_cas_8.S @@ -45,4 +45,14 @@ END(_atomic_cas_8) ATOMIC_OP_ALIAS(atomic_cas_8,_atomic_cas_8) STRONG_ALIAS(_atomic_cas_char,_atomic_cas_8) STRONG_ALIAS(_atomic_cas_uchar,_atomic_cas_8) -STRONG_ALIAS(__sync_val_compare_and_swap_1,_atomic_cas_8) + +ENTRY_NP(__sync_val_compare_and_swap_1) + mov x4, x0 /* we need x0 for return value */ +1: ldaxrb w0, [x4] /* load (acquire) old value */ + cmp w0, w1 /* compare? */ + b.ne 2f + stlxrb w3, w2, [x4] /* store (release) new value */ + cbnz w3, 3f /* succeed? nope, try again. */ +2: ret /* return. */ +3: b 1b +END(__sync_val_compare_and_swap_1) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S b/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S index df65b9f4d55c..54338ef04eac 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_16.S @@ -31,34 +31,61 @@ #include "atomic_op_asm.h" +/* + * { tmp = *ptr; *ptr = ~(tmp & value); return tmp; } // nand + */ ENTRY_NP(_atomic_nand_16) - mov x4, x0 -1: ldxrh w0, [x4] /* load old value (to be returned) */ - mvn w3, w0 /* complement source */ - and w3, w3, w1 /* calculate new value */ + mov x4, x0 /* need r0 for return value */ +1: ldxrh w0, [x4] /* load old value (*ptr) */ + and w3, w0, w1 /* w3 = (*ptr & value) */ + mvn w3, w3 /* w3 = ~(*ptr & value) */ stxrh w2, w3, [x4] /* try to store */ cbnz w2, 2f /* succeed? no, try again */ ret /* return old value */ 2: b 1b END(_atomic_nand_16) +ENTRY_NP(__sync_fetch_and_nand_2) + mov x4, x0 /* need r0 for return value */ +1: ldaxrh w0, [x4] /* load (acquire) old value (*ptr) */ + and w3, w0, w1 /* w3 = (*ptr & value) */ + mvn w3, w3 /* w3 = ~(*ptr & value) */ + stlxrh w2, w3, [x4] /* try to store release */ + cbnz w2, 2f /* succeed? no, try again */ + ret /* return old value */ +2: b 1b +END(__sync_fetch_and_nand_2) + ATOMIC_OP_ALIAS(atomic_nand_16,_atomic_nand_16) ATOMIC_OP_ALIAS(atomic_nand_ushort,_atomic_nand_16) -STRONG_ALIAS(__sync_fetch_and_nand_2,_atomic_nand_16) STRONG_ALIAS(_atomic_nand_ushort,_atomic_nand_16) + +/* + * { tmp = ~(*ptr & value); *ptr = tmp; return *ptr; } // nand + */ ENTRY_NP(_atomic_nand_16_nv) mov x4, x0 /* need r0 for return value */ -1: ldxrh w0, [x4] /* load old value */ - mvn w0, w0 /* complement source */ - and w0, w0, w1 /* calculate new value (return value) */ +1: ldxrh w0, [x4] /* load old value (*ptr) */ + and w0, w0, w1 /* w0 = (*ptr & value) */ + mvn w0, w0 /* w0 = ~(*ptr & value), return */ stxrh w2, w0, [x4] /* try to store */ cbnz w2, 2f /* succeed? no, try again? */ ret /* return new value */ 2: b 1b END(_atomic_nand_16_nv) +ENTRY_NP(__sync_nand_and_fetch_2) + mov x4, x0 /* need r0 for return value */ +1: ldaxrh w0, [x4] /* load (acquire) old value (*ptr) */ + and w0, w0, w1 /* w0 = (*ptr & value) */ + mvn w0, w0 /* w0 = ~(*ptr & value), return */ + stlxrh w2, w0, [x4] /* try to store (release) */ + cbnz w2, 2f /* succeed? no, try again? */ + ret /* return new value */ +2: b 1b +END(__sync_nand_and_fetch_2) + ATOMIC_OP_ALIAS(atomic_nand_16_nv,_atomic_nand_16_nv) ATOMIC_OP_ALIAS(atomic_nand_ushort_nv,_atomic_nand_16_nv) -STRONG_ALIAS(__sync_nand_and_fetch_2,_atomic_nand_16_nv) STRONG_ALIAS(_atomic_nand_ushort_nv,_atomic_nand_16_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S index bb5724cc5a54..bce86e8bddca 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_32.S @@ -31,11 +31,14 @@ #include "atomic_op_asm.h" +/* + * { tmp = *ptr; *ptr = ~(tmp & value); return tmp; } // nand + */ ENTRY_NP(_atomic_nand_32) - mov x4, x0 -1: ldxr w0, [x4] /* load old value (to be returned) */ - mvn w3, w0 /* complement source */ - and w3, w3, w1 /* calculate new value */ + mov x4, x0 /* need r0 for return value */ +1: ldxr w0, [x4] /* load old value (*ptr) */ + and w3, w0, w1 /* w3 = (tmp & value) */ + mvn w3, w3 /* w3 = ~(tmp & value) */ stxr w2, w3, [x4] /* try to store */ cbnz w2, 2f /* succeed? no, try again */ ret /* return old value */ @@ -44,14 +47,28 @@ END(_atomic_nand_32) ATOMIC_OP_ALIAS(atomic_nand_32,_atomic_nand_32) ATOMIC_OP_ALIAS(atomic_nand_uint,_atomic_nand_32) -STRONG_ALIAS(__sync_fetch_and_nand_4,_atomic_nand_32) STRONG_ALIAS(_atomic_nand_uint,_atomic_nand_32) +ENTRY_NP(__sync_fetch_and_nand_4) + mov x4, x0 /* need r0 for return value */ +1: ldaxr w0, [x4] /* load (acquire) old value (*ptr) */ + and w3, w0, w1 /* w3 = (tmp & value) */ + mvn w3, w3 /* w3 = ~(tmp & value) */ + stlxr w2, w3, [x4] /* try to store (release) */ + cbnz w2, 2f /* succeed? no, try again */ + ret /* return old value */ +2: b 1b +END(__sync_fetch_and_nand_4) + + +/* + * { tmp = ~(*ptr & value); *ptr = tmp; return *ptr; } // nand + */ ENTRY_NP(_atomic_nand_32_nv) mov x4, x0 /* need r0 for return value */ -1: ldxr w0, [x4] /* load old value */ - mvn w0, w0 /* complement source */ - and w0, w0, w1 /* calculate new value (return value) */ +1: ldxr w0, [x4] /* load old value (tmp) */ + and w0, w0, w1 /* w0 = (tmp & value) */ + mvn w0, w0 /* w0 = ~(tmp & value), return */ stxr w2, w0, [x4] /* try to store */ cbnz w2, 2f /* succeed? no, try again? */ ret /* return new value */ @@ -60,5 +77,15 @@ END(_atomic_nand_32_nv) ATOMIC_OP_ALIAS(atomic_nand_32_nv,_atomic_nand_32_nv) ATOMIC_OP_ALIAS(atomic_nand_uint_nv,_atomic_nand_32_nv) -STRONG_ALIAS(__sync_nand_and_fetch_4,_atomic_nand_32_nv) STRONG_ALIAS(_atomic_nand_uint_nv,_atomic_nand_32_nv) + +ENTRY_NP(__sync_nand_and_fetch_4) + mov x4, x0 /* need r0 for return value */ +1: ldaxr w0, [x4] /* load (acquire) old value (tmp) */ + and w0, w0, w1 /* w0 = (tmp & value) */ + mvn w0, w0 /* w0 = ~(tmp & value), return */ + stlxr w2, w0, [x4] /* try to store (release) */ + cbnz w2, 2f /* succeed? no, try again? */ + ret /* return new value */ +2: b 1b +END(__sync_nand_and_fetch_4) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S index f21498dcfe6d..80119d6a09aa 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_64.S @@ -31,11 +31,14 @@ #include "atomic_op_asm.h" +/* + * { tmp = *ptr; *ptr = ~(tmp & value); return tmp; } // nand + */ ENTRY_NP(_atomic_nand_64) - mov x4, x0 -1: ldxr x0, [x4] /* load old value (to be returned) */ - mvn x2, x0 /* complement source */ - and x2, x2, x1 /* calculate new value */ + mov x4, x0 /* need r0 for return value */ +1: ldxr x0, [x4] /* load old value (*ptr) */ + and x2, x2, x1 /* x3 = (*ptr & value) */ + mvn x2, x0 /* x3 = ~(*ptr & value) */ stxr w3, x2, [x4] /* try to store */ cbnz w3, 2f /* succeed? no, try again */ ret /* return old value */ @@ -44,14 +47,28 @@ END(_atomic_nand_64) ATOMIC_OP_ALIAS(atomic_nand_64,_atomic_nand_64) ATOMIC_OP_ALIAS(atomic_nand_ulong,_atomic_nand_64) -STRONG_ALIAS(__sync_fetch_and_nand_8,_atomic_nand_64) STRONG_ALIAS(_atomic_nand_ulong,_atomic_nand_64) +ENTRY_NP(__sync_fetch_and_nand_8) + mov x4, x0 /* need r0 for return value */ +1: ldaxr x0, [x4] /* load (acquire) old value (*ptr) */ + and x2, x2, x1 /* x3 = (*ptr & value) */ + mvn x2, x0 /* x3 = ~(*ptr & value) */ + stlxr w3, x2, [x4] /* try to store release */ + cbnz w3, 2f /* succeed? no, try again */ + ret /* return old value */ +2: b 1b +END(__sync_fetch_and_nand_8) + + +/* + * { tmp = ~(*ptr & value); *ptr = tmp; return *ptr; } // nand + */ ENTRY_NP(_atomic_nand_64_nv) mov x4, x0 /* need r0 for return value */ -1: ldxr x0, [x4] /* load old value */ - mvn x0, x0 /* complement source */ - and x0, x0, x1 /* calculate new value (return value) */ +1: ldxr x0, [x4] /* load old value (*ptr) */ + and x0, x0, x1 /* x0 = (*ptr & value) */ + mvn x0, x0 /* x0 = ~(*ptr & value) */ stxr w3, x0, [x4] /* try to store */ cbnz w3, 2f /* succeed? no, try again? */ ret /* return new value */ @@ -60,5 +77,15 @@ END(_atomic_nand_64_nv) ATOMIC_OP_ALIAS(atomic_nand_64_nv,_atomic_nand_64_nv) ATOMIC_OP_ALIAS(atomic_nand_ulong_nv,_atomic_nand_64_nv) -STRONG_ALIAS(__sync_nand_and_fetch_8,_atomic_nand_64_nv) STRONG_ALIAS(_atomic_nand_ulong_nv,_atomic_nand_64_nv) + +ENTRY_NP(__sync_nand_and_fetch_8) + mov x4, x0 /* need r0 for return value */ +1: ldaxr x0, [x4] /* load (acquire) old value (*ptr) */ + and x0, x0, x1 /* x0 = (*ptr & value) */ + mvn x0, x0 /* x0 = ~(*ptr & value) */ + stlxr w3, x0, [x4] /* try to store (release) */ + cbnz w3, 2f /* succeed? no, try again? */ + ret /* return new value */ +2: b 1b +END(__sync_nand_and_fetch_8) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S b/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S index 8b473e28b1de..bcebd94a8c36 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_nand_8.S @@ -31,11 +31,14 @@ #include "atomic_op_asm.h" +/* + * { tmp = *ptr; *ptr = ~(tmp & value); return tmp; } // nand + */ ENTRY_NP(_atomic_nand_8) - mov x4, x0 -1: ldxrb w0, [x4] /* load old value (to be returned) */ - mvn w3, w0 /* complement source */ - and w3, w3, w1 /* calculate new value */ + mov x4, x0 /* need r0 for return value */ +1: ldxrb w0, [x4] /* load old value (*ptr) */ + and w3, w3, w1 /* w3 = (*ptr & value) */ + mvn w3, w0 /* w3 = ~(*ptr & value) */ stxrb w2, w3, [x4] /* try to store */ cbnz w2, 2f /* succeed? no, try again */ ret /* return old value */ @@ -44,14 +47,28 @@ END(_atomic_nand_8) ATOMIC_OP_ALIAS(atomic_nand_8,_atomic_nand_8) ATOMIC_OP_ALIAS(atomic_nand_uchar,_atomic_nand_8) -STRONG_ALIAS(__sync_fetch_and_nand_1,_atomic_nand_8) STRONG_ALIAS(_atomic_nand_uchar,_atomic_nand_8) +ENTRY_NP(__sync_fetch_and_nand_1) + mov x4, x0 /* need r0 for return value */ +1: ldaxrb w0, [x4] /* load old value (*ptr) */ + and w3, w3, w1 /* w3 = (*ptr & value) */ + mvn w3, w0 /* w3 = ~(*ptr & value) */ + stlxrb w2, w3, [x4] /* try to store */ + cbnz w2, 2f /* succeed? no, try again */ + ret /* return old value */ +2: b 1b +END(__sync_fetch_and_nand_1) + + +/* + * { tmp = ~(*ptr & value); *ptr = tmp; return *ptr; } // nand + */ ENTRY_NP(_atomic_nand_8_nv) mov x4, x0 /* need r0 for return value */ -1: ldxrb w0, [x4] /* load old value */ - mvn w0, w0 /* complement source */ - and w0, w0, w1 /* calculate new value (return value) */ +1: ldxrb w0, [x4] /* load old value (*ptr) */ + and w0, w0, w1 /* w0 = (*ptr & value) */ + mvn w0, w0 /* w0 = ~(*ptr & value) */ stxrb w2, w0, [x4] /* try to store */ cbnz w2, 2f /* succeed? no, try again? */ ret /* return new value */ @@ -60,5 +77,15 @@ END(_atomic_nand_8_nv) ATOMIC_OP_ALIAS(atomic_nand_8_nv,_atomic_nand_8_nv) ATOMIC_OP_ALIAS(atomic_nand_uchar_nv,_atomic_nand_8_nv) -STRONG_ALIAS(__sync_nand_and_fetch_1,_atomic_nand_8_nv) STRONG_ALIAS(_atomic_nand_uchar_nv,_atomic_nand_8_nv) + +ENTRY_NP(__sync_nand_and_fetch_1) + mov x4, x0 /* need r0 for return value */ +1: ldaxrb w0, [x4] /* load (acquire) old value (*ptr) */ + and w0, w0, w1 /* w0 = (*ptr & value) */ + mvn w0, w0 /* w0 = ~(*ptr & value) */ + stlxrb w2, w0, [x4] /* try to store (release) */ + cbnz w2, 2f /* succeed? no, try again? */ + ret /* return new value */ +2: b 1b +END(__sync_nand_and_fetch_1) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h b/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h index 37ae06002ca8..d7307acf2248 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h +++ b/common/lib/libc/arch/aarch64/atomic/atomic_op_asm.h @@ -44,6 +44,16 @@ ENTRY_NP(_atomic_##OP##_8) ;\ ret /* return old value */ ;\ END(_atomic_##OP##_8) +#define SYNC_FETCH_OP8(OP, INSN) \ +ENTRY_NP(__sync_fetch_and_##OP##_1) ;\ + mov x4, x0 ;\ +1: ldaxrb w0, [x4] /* load (acquire) old value */ ;\ + INSN w2, w0, w1 /* calculate new value */ ;\ + stlxrb w3, w2, [x4] /* try to store (release) */ ;\ + cbnz w3, 1b /* succeed? no, try again */ ;\ + ret /* return old value */ ;\ +END(__sync_fetch_and_##OP##_1) + #define ATOMIC_OP8_NV(OP, INSN) \ ENTRY_NP(_atomic_##OP##_8_nv) ;\ mov x4, x0 /* need x0 for return value */ ;\ @@ -54,6 +64,16 @@ ENTRY_NP(_atomic_##OP##_8_nv) ;\ ret /* return new value */ ;\ END(_atomic_##OP##_8_nv) +#define SYNC_OP8_FETCH(OP, INSN) \ +ENTRY_NP(__sync_##OP##_and_fetch_1) ;\ + mov x4, x0 /* need x0 for return value */ ;\ +1: ldaxrb w0, [x4] /* load (acquire) old value */ ;\ + INSN w0, w0, w1 /* calc new (return) value */ ;\ + stlxrb w3, w0, [x4] /* try to store (release) */ ;\ + cbnz w3, 1b /* succeed? no, try again */ ;\ + ret /* return new value */ ;\ +END(__sync_##OP##_and_fetch_1) + #define ATOMIC_OP16(OP, INSN) \ ENTRY_NP(_atomic_##OP##_16) ;\ mov x4, x0 ;\ @@ -64,6 +84,16 @@ ENTRY_NP(_atomic_##OP##_16) ;\ ret /* return old value */ ;\ END(_atomic_##OP##_16) +#define SYNC_FETCH_OP16(OP, INSN) \ +ENTRY_NP(__sync_fetch_and_##OP##_2) ;\ + mov x4, x0 ;\ +1: ldaxrh w0, [x4] /* load (acquire) old value */ ;\ + INSN w2, w0, w1 /* calculate new value */ ;\ + stlxrh w3, w2, [x4] /* try to store (release) */ ;\ + cbnz w3, 1b /* succeed? no, try again */ ;\ + ret /* return old value */ ;\ +END(__sync_fetch_and_##OP##_2) + #define ATOMIC_OP16_NV(OP, INSN) \ ENTRY_NP(_atomic_##OP##_16_nv) ;\ mov x4, x0 /* need x0 for return value */ ;\ @@ -74,6 +104,16 @@ ENTRY_NP(_atomic_##OP##_16_nv) ;\ ret /* return new value */ ;\ END(_atomic_##OP##_16_nv) +#define SYNC_OP16_FETCH(OP, INSN) \ +ENTRY_NP(__sync__##OP##_and_fetch_2) ;\ + mov x4, x0 /* need x0 for return value */ ;\ +1: ldaxrh w0, [x4] /* load (acquire) old value */ ;\ + INSN w0, w0, w1 /* calc new (return) value */ ;\ + stlxrh w3, w0, [x4] /* try to store (release) */ ;\ + cbnz w3, 1b /* succeed? no, try again */ ;\ + ret /* return new value */ ;\ +END(__sync__##OP##_and_fetch_2) + #define ATOMIC_OP32(OP, INSN) \ ENTRY_NP(_atomic_##OP##_32) ;\ mov x4, x0 ;\ @@ -84,6 +124,16 @@ ENTRY_NP(_atomic_##OP##_32) ;\ ret /* return old value */ ;\ END(_atomic_##OP##_32) +#define SYNC_FETCH_OP32(OP, INSN) \ +ENTRY_NP(__sync_fetch_and_##OP##_4) ;\ + mov x4, x0 ;\ +1: ldaxr w0, [x4] /* load (acquire) old value */ ;\ + INSN w2, w0, w1 /* calculate new value */ ;\ + stlxr w3, w2, [x4] /* try to store (release) */ ;\ + cbnz w3, 1b /* succeed? no, try again */ ;\ + ret /* return old value */ ;\ +END(__sync_fetch_and_##OP##_4) + #define ATOMIC_OP32_NV(OP, INSN) \ ENTRY_NP(_atomic_##OP##_32_nv) ;\ mov x4, x0 /* need x0 for return value */ ;\ @@ -94,6 +144,16 @@ ENTRY_NP(_atomic_##OP##_32_nv) ;\ ret /* return new value */ ;\ END(_atomic_##OP##_32_nv) +#define SYNC_OP32_FETCH(OP, INSN) \ +ENTRY_NP(__sync__##OP##_and_fetch_4) ;\ + mov x4, x0 /* need x0 for return value */ ;\ +1: ldaxr w0, [x4] /* load (acquire) old value */ ;\ + INSN w0, w0, w1 /* calc new (return) value */ ;\ + stlxr w3, w0, [x4] /* try to store (release) */ ;\ + cbnz w3, 1b /* succeed? no, try again? */ ;\ + ret /* return new value */ ;\ +END(__sync__##OP##_and_fetch_4) + #define ATOMIC_OP64(OP, INSN) \ ENTRY_NP(_atomic_##OP##_64) ;\ mov x4, x0 ;\ @@ -104,6 +164,16 @@ ENTRY_NP(_atomic_##OP##_64) ;\ ret /* return old value */ ;\ END(_atomic_##OP##_64) +#define SYNC_FETCH_OP64(OP, INSN) \ +ENTRY_NP(__sync_fetch_and_##OP##_8) ;\ + mov x4, x0 ;\ +1: ldaxr x0, [x4] /* load (acquire) old value */ ;\ + INSN x2, x0, x1 /* calculate new value */ ;\ + stlxr w3, x2, [x4] /* try to store (release) */ ;\ + cbnz w3, 1b /* succeed? no, try again */ ;\ + ret /* return old value */ ;\ +END(__sync_fetch_and_##OP##_8) + #define ATOMIC_OP64_NV(OP, INSN) \ ENTRY_NP(_atomic_##OP##_64_nv) ;\ mov x4, x0 /* need x0 for return value */ ;\ @@ -114,6 +184,16 @@ ENTRY_NP(_atomic_##OP##_64_nv) ;\ ret /* return new value */ ;\ END(_atomic_##OP##_64_nv) +#define SYNC_OP64_FETCH(OP, INSN) \ +ENTRY_NP(__sync_##OP##_and_fetch_8) ;\ + mov x4, x0 /* need x0 for return value */ ;\ +1: ldaxr x0, [x4] /* load (acquire) old value */ ;\ + INSN x0, x0, x1 /* calc new (return) value */ ;\ + stlxr w3, x0, [x4] /* try to store (release) */ ;\ + cbnz w3, 1b /* succeed? no, try again? */ ;\ + ret /* return new value */ ;\ +END(__sync_##OP##_and_fetch_8) + #if defined(_KERNEL) #define ATOMIC_OP_ALIAS(a,s) STRONG_ALIAS(a,s) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_or_16.S b/common/lib/libc/arch/aarch64/atomic/atomic_or_16.S index e167734759cc..2d17df52eb46 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_or_16.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_or_16.S @@ -33,14 +33,16 @@ ATOMIC_OP16(or, orr) +SYNC_FETCH_OP16(or, orr) + ATOMIC_OP_ALIAS(atomic_or_16,_atomic_or_16) ATOMIC_OP_ALIAS(atomic_or_ushort,_atomic_or_16) -STRONG_ALIAS(__sync_fetch_and_or_2,_atomic_or_16) STRONG_ALIAS(_atomic_or_ushort,_atomic_or_16) ATOMIC_OP16_NV(or, orr) +SYNC_OP16_FETCH(or, orr) + ATOMIC_OP_ALIAS(atomic_or_16_nv,_atomic_or_16_nv) ATOMIC_OP_ALIAS(atomic_or_ushort_nv,_atomic_or_16_nv) -STRONG_ALIAS(__sync_or_and_fetch_2,_atomic_or_16_nv) STRONG_ALIAS(_atomic_or_ushort_nv,_atomic_or_16_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_or_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_or_32.S index 2411812ca7d3..e913e97d1578 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_or_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_or_32.S @@ -33,14 +33,16 @@ ATOMIC_OP32(or, orr) +SYNC_FETCH_OP32(or, orr) + ATOMIC_OP_ALIAS(atomic_or_32,_atomic_or_32) ATOMIC_OP_ALIAS(atomic_or_uint,_atomic_or_32) -STRONG_ALIAS(__sync_fetch_and_or_4,_atomic_or_32) STRONG_ALIAS(_atomic_or_uint,_atomic_or_32) ATOMIC_OP32_NV(or, orr) +SYNC_OP32_FETCH(or, orr) + ATOMIC_OP_ALIAS(atomic_or_32_nv,_atomic_or_32_nv) ATOMIC_OP_ALIAS(atomic_or_uint_nv,_atomic_or_32_nv) -STRONG_ALIAS(__sync_or_and_fetch_4,_atomic_or_32_nv) STRONG_ALIAS(_atomic_or_uint_nv,_atomic_or_32_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_or_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_or_64.S index c99351930c33..35ed8df09b6c 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_or_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_or_64.S @@ -33,14 +33,16 @@ ATOMIC_OP64(or, orr) +SYNC_FETCH_OP64(or, orr) + ATOMIC_OP_ALIAS(atomic_or_64,_atomic_or_64) ATOMIC_OP_ALIAS(atomic_or_ulong,_atomic_or_64) STRONG_ALIAS(_atomic_or_ulong,_atomic_or_64) -STRONG_ALIAS(__sync_fetch_and_or_8,_atomic_or_64) ATOMIC_OP64_NV(or, orr) +SYNC_OP64_FETCH(or, orr) + STRONG_ALIAS(_atomic_or_ulong_nv,_atomic_or_64_nv) ATOMIC_OP_ALIAS(atomic_or_64_nv,_atomic_or_64_nv) ATOMIC_OP_ALIAS(atomic_or_ulong_nv,_atomic_or_64_nv) -STRONG_ALIAS(__sync_or_and_fetch_8,_atomic_or_64_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_or_8.S b/common/lib/libc/arch/aarch64/atomic/atomic_or_8.S index 3f4e5615f277..316b20a50678 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_or_8.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_or_8.S @@ -33,14 +33,16 @@ ATOMIC_OP8(or, orr) +SYNC_FETCH_OP8(or, orr) + ATOMIC_OP_ALIAS(atomic_or_8,_atomic_or_8) ATOMIC_OP_ALIAS(atomic_or_uchar,_atomic_or_8) -STRONG_ALIAS(__sync_fetch_and_or_1,_atomic_or_8) STRONG_ALIAS(_atomic_or_uchar,_atomic_or_8) ATOMIC_OP8_NV(or, orr) +SYNC_OP8_FETCH(or, orr) + ATOMIC_OP_ALIAS(atomic_or_8_nv,_atomic_or_8_nv) ATOMIC_OP_ALIAS(atomic_or_uchar_nv,_atomic_or_8_nv) -STRONG_ALIAS(__sync_or_and_fetch_1,_atomic_or_8_nv) STRONG_ALIAS(_atomic_or_uchar_nv,_atomic_or_8_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S b/common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S index 44554e475013..0ff0c35fbe90 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_sub_16.S @@ -33,18 +33,20 @@ ATOMIC_OP16(sub, sub) +SYNC_FETCH_OP16(sub, sub) + ATOMIC_OP_ALIAS(atomic_sub_16,_atomic_sub_16) ATOMIC_OP_ALIAS(atomic_sub_short,_atomic_sub_16) ATOMIC_OP_ALIAS(atomic_sub_ushort,_atomic_sub_16) -STRONG_ALIAS(__sync_fetch_and_sub_2,_atomic_sub_16) STRONG_ALIAS(_atomic_sub_short,_atomic_sub_16) STRONG_ALIAS(_atomic_sub_ushort,_atomic_sub_16) ATOMIC_OP16_NV(sub, sub) +SYNC_OP16_FETCH(sub, sub) + ATOMIC_OP_ALIAS(atomic_sub_16_nv,_atomic_sub_16_nv) ATOMIC_OP_ALIAS(atomic_sub_short_nv,_atomic_sub_16_nv) ATOMIC_OP_ALIAS(atomic_sub_ushort_nv,_atomic_sub_16_nv) -STRONG_ALIAS(__sync_sub_and_fetch_2,_atomic_sub_16_nv) STRONG_ALIAS(_atomic_sub_short_nv,_atomic_sub_16_nv) STRONG_ALIAS(_atomic_sub_ushort_nv,_atomic_sub_16_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S index b59c1d9399db..7b902a9e9890 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_sub_32.S @@ -33,14 +33,16 @@ ATOMIC_OP32(sub, sub) +SYNC_FETCH_OP32(sub, sub) + ATOMIC_OP_ALIAS(atomic_sub_32,_atomic_sub_32) ATOMIC_OP_ALIAS(atomic_sub_int,_atomic_sub_32) -STRONG_ALIAS(__sync_fetch_and_sub_4,_atomic_sub_32) STRONG_ALIAS(_atomic_sub_int,_atomic_sub_32) ATOMIC_OP32_NV(sub, sub) +SYNC_OP32_FETCH(sub, sub) + ATOMIC_OP_ALIAS(atomic_sub_32_nv,_atomic_sub_32_nv) ATOMIC_OP_ALIAS(atomic_sub_int_nv,_atomic_sub_32_nv) -STRONG_ALIAS(__sync_sub_and_fetch_4,_atomic_sub_32_nv) STRONG_ALIAS(_atomic_sub_int_nv,_atomic_sub_32_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S index ce1706a04d4c..7da367bf66a4 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_sub_64.S @@ -33,18 +33,20 @@ ATOMIC_OP64(sub, sub) +SYNC_FETCH_OP64(sub, sub) + ATOMIC_OP_ALIAS(atomic_sub_64_nv,_atomic_sub_64_nv) ATOMIC_OP_ALIAS(atomic_sub_ptr_nv,_atomic_sub_64_nv) ATOMIC_OP_ALIAS(atomic_sub_long_nv,_atomic_sub_64_nv) STRONG_ALIAS(_atomic_sub_ptr_nv,_atomic_sub_64_nv) STRONG_ALIAS(_atomic_sub_long_nv,_atomic_sub_64_nv) -STRONG_ALIAS(__sync_sub_and_fetch_8,_atomic_sub_64_nv) ATOMIC_OP64_NV(sub, sub) +SYNC_OP64_FETCH(sub, sub) + ATOMIC_OP_ALIAS(atomic_sub_64,_atomic_sub_64) ATOMIC_OP_ALIAS(atomic_sub_ptr,_atomic_sub_64) ATOMIC_OP_ALIAS(atomic_sub_long,_atomic_sub_64) STRONG_ALIAS(_atomic_sub_ptr,_atomic_sub_64) STRONG_ALIAS(_atomic_sub_long,_atomic_sub_64) -STRONG_ALIAS(__sync_fetch_and_sub_8,_atomic_sub_64) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S b/common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S index 6bdb77483790..6add242be907 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_sub_8.S @@ -33,14 +33,16 @@ ATOMIC_OP8(sub, sub) +SYNC_FETCH_OP8(sub, sub) + ATOMIC_OP_ALIAS(atomic_sub_8,_atomic_sub_8) ATOMIC_OP_ALIAS(atomic_sub_char,_atomic_sub_8) -STRONG_ALIAS(__sync_fetch_and_sub_1,_atomic_sub_8) STRONG_ALIAS(_atomic_sub_char,_atomic_sub_8) ATOMIC_OP8_NV(sub, sub) +SYNC_OP8_FETCH(sub, sub) + ATOMIC_OP_ALIAS(atomic_sub_8_nv,_atomic_sub_8_nv) ATOMIC_OP_ALIAS(atomic_sub_char_nv,_atomic_sub_8_nv) -STRONG_ALIAS(__sync_sub_and_fetch_1,_atomic_sub_8_nv) STRONG_ALIAS(_atomic_sub_char_nv,_atomic_sub_8_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S b/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S index 1f3cd95dfb42..94ad1adfa1a9 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_swap_16.S @@ -43,10 +43,18 @@ END(_atomic_swap_16) ATOMIC_OP_ALIAS(atomic_swap_16,_atomic_swap_16) ATOMIC_OP_ALIAS(atomic_swap_short,_atomic_swap_16) ATOMIC_OP_ALIAS(atomic_swap_ushort,_atomic_swap_16) -STRONG_ALIAS(__sync_lock_test_and_set_2,_atomic_swap_16) STRONG_ALIAS(_atomic_swap_short,_atomic_swap_16) STRONG_ALIAS(_atomic_swap_ushort,_atomic_swap_16) +ENTRY_NP(__sync_lock_test_and_set_2) + mov x4, x0 +1: ldaxrh w0, [x4] + stxrh w3, w1, [x4] + cbnz w3, 2f + ret +2: b 1b +END(__sync_lock_test_and_set_2) + ENTRY_NP(__sync_lock_release_2) stlrh wzr, [x0] ret diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S index 16863d6d0827..b9c6c97964fa 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_swap_32.S @@ -42,9 +42,17 @@ END(_atomic_swap_32) ATOMIC_OP_ALIAS(atomic_swap_32,_atomic_swap_32) ATOMIC_OP_ALIAS(atomic_swap_uint,_atomic_swap_32) -STRONG_ALIAS(__sync_lock_test_and_set_4,_atomic_swap_32) STRONG_ALIAS(_atomic_swap_uint,_atomic_swap_32) +ENTRY_NP(__sync_lock_test_and_set_4) + mov x4, x0 +1: ldaxr w0, [x4] + stxr w3, w1, [x4] + cbnz w3, 2f + ret +2: b 1b +END(__sync_lock_test_and_set_4) + ENTRY_NP(__sync_lock_release_4) stlr wzr, [x0] ret diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S index e9ac40c2a888..aae2e07224fd 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_swap_64.S @@ -43,10 +43,18 @@ END(_atomic_swap_64) ATOMIC_OP_ALIAS(atomic_swap_64,_atomic_swap_64) ATOMIC_OP_ALIAS(atomic_swap_ulong,_atomic_swap_64) ATOMIC_OP_ALIAS(atomic_swap_ptr,_atomic_swap_64) -STRONG_ALIAS(__sync_lock_test_and_set_8,_atomic_swap_64) STRONG_ALIAS(_atomic_swap_ulong,_atomic_swap_64) STRONG_ALIAS(_atomic_swap_ptr,_atomic_swap_64) +ENTRY_NP(__sync_lock_test_and_set_8) + mov x4, x0 +1: ldaxr x0, [x4] + stxr w3, x1, [x4] + cbnz w3, 2f + ret +2: b 1b +END(_atomic_swap_64) + ENTRY_NP(__sync_lock_release_8) stlr xzr, [x0] ret diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S b/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S index f2ba13e9fc5f..1d8a700a5265 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_swap_8.S @@ -43,10 +43,18 @@ END(_atomic_swap_8) ATOMIC_OP_ALIAS(atomic_swap_8,_atomic_swap_8) ATOMIC_OP_ALIAS(atomic_swap_char,_atomic_swap_8) ATOMIC_OP_ALIAS(atomic_swap_uchar,_atomic_swap_8) -STRONG_ALIAS(__sync_lock_test_and_set_1,_atomic_swap_8) STRONG_ALIAS(_atomic_swap_char,_atomic_swap_8) STRONG_ALIAS(_atomic_swap_uchar,_atomic_swap_8) +ENTRY_NP(__sync_lock_test_and_set_1) + mov x4, x0 +1: ldaxrb w0, [x4] + stxrb w3, w1, [x4] + cbnz w3, 2f + ret +2: b 1b +END(__sync_lock_test_and_set_1) + ENTRY_NP(__sync_lock_release_1) stlrb wzr, [x0] ret diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S b/common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S index 564ba86859d0..65b84567361f 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_xor_16.S @@ -33,14 +33,16 @@ ATOMIC_OP16(xor, eor) +SYNC_FETCH_OP16(xor, eor) + ATOMIC_OP_ALIAS(atomic_xor_16,_atomic_xor_16) ATOMIC_OP_ALIAS(atomic_xor_ushort,_atomic_xor_16) -STRONG_ALIAS(__sync_fetch_and_xor_2,_atomic_xor_16) STRONG_ALIAS(_atomic_xor_ushort,_atomic_xor_16) ATOMIC_OP16_NV(xor, eor) +SYNC_OP16_FETCH(xor, eor) + ATOMIC_OP_ALIAS(atomic_xor_16_nv,_atomic_xor_16_nv) ATOMIC_OP_ALIAS(atomic_xor_ushort_nv,_atomic_xor_16_nv) -STRONG_ALIAS(__sync_xor_and_fetch_2,_atomic_xor_16_nv) STRONG_ALIAS(_atomic_xor_ushort_nv,_atomic_xor_16_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S b/common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S index f1bfdc4f43c2..1368416e8355 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_xor_32.S @@ -33,14 +33,16 @@ ATOMIC_OP32(xor, eor) +SYNC_FETCH_OP32(xor, eor) + ATOMIC_OP_ALIAS(atomic_xor_32,_atomic_xor_32) ATOMIC_OP_ALIAS(atomic_xor_uint,_atomic_xor_32) -STRONG_ALIAS(__sync_fetch_and_xor_4,_atomic_xor_32) STRONG_ALIAS(_atomic_xor_uint,_atomic_xor_32) ATOMIC_OP32_NV(xor, eor) +SYNC_OP32_FETCH(xor, eor) + ATOMIC_OP_ALIAS(atomic_xor_32_nv,_atomic_xor_32_nv) ATOMIC_OP_ALIAS(atomic_xor_uint_nv,_atomic_xor_32_nv) -STRONG_ALIAS(__sync_xor_and_fetch_4,_atomic_xor_32_nv) STRONG_ALIAS(_atomic_xor_uint_nv,_atomic_xor_32_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S b/common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S index 588d8a0ef47b..b1689bd83b8c 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_xor_64.S @@ -33,12 +33,14 @@ ATOMIC_OP64(xor, eor) +SYNC_FETCH_OP64(xor, eor) + ATOMIC_OP_ALIAS(atomic_xor_64,_atomic_xor_64) ATOMIC_OP_ALIAS(atomic_xor_ulong,_atomic_xor_64) -STRONG_ALIAS(__sync_fetch_and_xor_8,_atomic_xor_64) ATOMIC_OP64_NV(xor, eor) +SYNC_OP64_FETCH(xor, eor) + ATOMIC_OP_ALIAS(atomic_xor_64_nv,_atomic_xor_64_nv) ATOMIC_OP_ALIAS(atomic_xor_ulong_nv,_atomic_xor_64_nv) -STRONG_ALIAS(__sync_xor_and_fetch_8,_atomic_xor_64_nv) diff --git a/common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S b/common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S index 9eeff7e519db..f08b18ef5899 100644 --- a/common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S +++ b/common/lib/libc/arch/aarch64/atomic/atomic_xor_8.S @@ -33,14 +33,16 @@ ATOMIC_OP8(xor, eor) +SYNC_FETCH_OP8(xor, eor) + ATOMIC_OP_ALIAS(atomic_xor_8,_atomic_xor_8) ATOMIC_OP_ALIAS(atomic_xor_uchar,_atomic_xor_8) -STRONG_ALIAS(__sync_fetch_and_xor_1,_atomic_xor_8) STRONG_ALIAS(_atomic_xor_uchar,_atomic_xor_8) ATOMIC_OP8_NV(xor, eor) +SYNC_OP8_FETCH(xor, eor) + ATOMIC_OP_ALIAS(atomic_xor_8_nv,_atomic_xor_8_nv) ATOMIC_OP_ALIAS(atomic_xor_uchar_nv,_atomic_xor_8_nv) -STRONG_ALIAS(__sync_xor_and_fetch_1,_atomic_xor_8_nv) STRONG_ALIAS(_atomic_xor_uchar_nv,_atomic_xor_8_nv)