Replace fence instructions with Zalasr instructions during
smp_load_acquire() and smp_store_release() operations.
|----------------------------------|
| | __smp_store_release |
| |-----------------------------|
| | zalasr | !zalasr |
| rl |-----------------------------|
| | s{b|h|w|d}.rl | fence rw, w |
| | | s{b|h|w|d} |
|----------------------------------|
| | __smp_load_acquire |
| |-----------------------------|
| | zalasr | !zalasr |
| aq |-----------------------------|
| | l{b|h|w|d}.rl | l{b|h|w|d} |
| | | fence r, rw |
|----------------------------------|
Signed-off-by: Xu Lu <luxu.kernel@bytedance.com>
---
arch/riscv/include/asm/barrier.h | 91 ++++++++++++++++++++++++++++----
1 file changed, 80 insertions(+), 11 deletions(-)
diff --git a/arch/riscv/include/asm/barrier.h b/arch/riscv/include/asm/barrier.h
index b8c5726d86acb..9eaf94a028096 100644
--- a/arch/riscv/include/asm/barrier.h
+++ b/arch/riscv/include/asm/barrier.h
@@ -51,19 +51,88 @@
*/
#define smp_mb__after_spinlock() RISCV_FENCE(iorw, iorw)
-#define __smp_store_release(p, v) \
-do { \
- compiletime_assert_atomic_type(*p); \
- RISCV_FENCE(rw, w); \
- WRITE_ONCE(*p, v); \
+extern void __bad_size_call_parameter(void);
+
+#define __smp_store_release(p, v) \
+do { \
+ typeof(p) __p = (p); \
+ union { typeof(*p) __val; char __c[1]; } __u = \
+ { .__val = (__force typeof(*p)) (v) }; \
+ compiletime_assert_atomic_type(*p); \
+ switch (sizeof(*p)) { \
+ case 1: \
+ asm volatile(ALTERNATIVE("fence rw, w;\t\nsb %0, 0(%1)\t\n", \
+ SB_RL(%0, %1) "\t\nnop\t\n", \
+ 0, RISCV_ISA_EXT_ZALASR, 1) \
+ : : "r" (*(__u8 *)__u.__c), "r" (__p) \
+ : "memory"); \
+ break; \
+ case 2: \
+ asm volatile(ALTERNATIVE("fence rw, w;\t\nsh %0, 0(%1)\t\n", \
+ SH_RL(%0, %1) "\t\nnop\t\n", \
+ 0, RISCV_ISA_EXT_ZALASR, 1) \
+ : : "r" (*(__u16 *)__u.__c), "r" (__p) \
+ : "memory"); \
+ break; \
+ case 4: \
+ asm volatile(ALTERNATIVE("fence rw, w;\t\nsw %0, 0(%1)\t\n", \
+ SW_RL(%0, %1) "\t\nnop\t\n", \
+ 0, RISCV_ISA_EXT_ZALASR, 1) \
+ : : "r" (*(__u32 *)__u.__c), "r" (__p) \
+ : "memory"); \
+ break; \
+ case 8: \
+ asm volatile(ALTERNATIVE("fence rw, w;\t\nsd %0, 0(%1)\t\n", \
+ SD_RL(%0, %1) "\t\nnop\t\n", \
+ 0, RISCV_ISA_EXT_ZALASR, 1) \
+ : : "r" (*(__u64 *)__u.__c), "r" (__p) \
+ : "memory"); \
+ break; \
+ default: \
+ __bad_size_call_parameter(); \
+ break; \
+ } \
} while (0)
-#define __smp_load_acquire(p) \
-({ \
- typeof(*p) ___p1 = READ_ONCE(*p); \
- compiletime_assert_atomic_type(*p); \
- RISCV_FENCE(r, rw); \
- ___p1; \
+#define __smp_load_acquire(p) \
+({ \
+ union { typeof(*p) __val; char __c[1]; } __u; \
+ typeof(p) __p = (p); \
+ compiletime_assert_atomic_type(*p); \
+ switch (sizeof(*p)) { \
+ case 1: \
+ asm volatile(ALTERNATIVE("lb %0, 0(%1)\t\nfence r, rw\t\n", \
+ LB_AQ(%0, %1) "\t\nnop\t\n", \
+ 0, RISCV_ISA_EXT_ZALASR, 1) \
+ : "=r" (*(__u8 *)__u.__c) : "r" (__p) \
+ : "memory"); \
+ break; \
+ case 2: \
+ asm volatile(ALTERNATIVE("lh %0, 0(%1)\t\nfence r, rw\t\n", \
+ LH_AQ(%0, %1) "\t\nnop\t\n", \
+ 0, RISCV_ISA_EXT_ZALASR, 1) \
+ : "=r" (*(__u16 *)__u.__c) : "r" (__p) \
+ : "memory"); \
+ break; \
+ case 4: \
+ asm volatile(ALTERNATIVE("lw %0, 0(%1)\t\nfence r, rw\t\n", \
+ LW_AQ(%0, %1) "\t\nnop\t\n", \
+ 0, RISCV_ISA_EXT_ZALASR, 1) \
+ : "=r" (*(__u32 *)__u.__c) : "r" (__p) \
+ : "memory"); \
+ break; \
+ case 8: \
+ asm volatile(ALTERNATIVE("ld %0, 0(%1)\t\nfence r, rw\t\n", \
+ LD_AQ(%0, %1) "\t\nnop\t\n", \
+ 0, RISCV_ISA_EXT_ZALASR, 1) \
+ : "=r" (*(__u64 *)__u.__c) : "r" (__p) \
+ : "memory"); \
+ break; \
+ default: \
+ __bad_size_call_parameter(); \
+ break; \
+ } \
+ __u.__val; \
})
#ifdef CONFIG_RISCV_ISA_ZAWRS
--
2.20.1