arch/riscv/Kconfig | 10 ++++++ arch/riscv/include/asm/atomic.h | 52 +++++++++++++++++++++++++++++++- arch/riscv/include/asm/bitops.h | 45 +++++++++++++++++++++++++++ arch/riscv/include/asm/cmpxchg.h | 16 ++++++++++ arch/riscv/include/asm/futex.h | 46 ++++++++++++++++++++++++++++ 5 files changed, 168 insertions(+), 1 deletion(-)
From: Chao-ying Fu <cfu@mips.com>
Use lr and sc to implement all atomic functions. Some CPUs have
native support for lr and sc, but emulate AMO instructions through
trap handlers that are slow.
Add config RISCV_ISA_ZALRSC_ONLY.
Signed-off-by: Chao-ying Fu <cfu@mips.com>
Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com>
---
arch/riscv/Kconfig | 10 ++++++
arch/riscv/include/asm/atomic.h | 52 +++++++++++++++++++++++++++++++-
arch/riscv/include/asm/bitops.h | 45 +++++++++++++++++++++++++++
arch/riscv/include/asm/cmpxchg.h | 16 ++++++++++
arch/riscv/include/asm/futex.h | 46 ++++++++++++++++++++++++++++
5 files changed, 168 insertions(+), 1 deletion(-)
diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index cc63aef41e94..767538c27875 100644
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -715,6 +715,16 @@ config RISCV_ISA_ZACAS
If you don't know what to do here, say Y.
+config RISCV_ISA_ZALRSC_ONLY
+ bool "Zalrsc extension support only"
+ default n
+ help
+ Use lr and sc to build all atomic functions. Some CPUs have
+ native support for lr and sc, but emulate amo instructions through
+ trap handlers that are slow.
+
+ If you don't know what to do here, say n.
+
config TOOLCHAIN_HAS_ZBB
bool
default y
diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h
index 5b96c2f61adb..f484babecb9e 100644
--- a/arch/riscv/include/asm/atomic.h
+++ b/arch/riscv/include/asm/atomic.h
@@ -50,6 +50,7 @@ static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
* have the AQ or RL bits set. These don't return anything, so there's only
* one version to worry about.
*/
+#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
static __always_inline \
void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
@@ -59,7 +60,23 @@ void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
: "+A" (v->counter) \
: "r" (I) \
: "memory"); \
-} \
+}
+#else
+#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
+static __always_inline \
+void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
+{ \
+ register c_type ret, temp; \
+ __asm__ __volatile__ ( \
+ "1: lr." #asm_type " %1, %0\n" \
+ " " #asm_op " %2, %1, %3\n" \
+ " sc." #asm_type " %2, %2, %0\n" \
+ " bnez %2, 1b\n" \
+ : "+A" (v->counter), "=&r" (ret), "=&r" (temp) \
+ : "r" (I) \
+ : "memory"); \
+}
+#endif
#ifdef CONFIG_GENERIC_ATOMIC64
#define ATOMIC_OPS(op, asm_op, I) \
@@ -84,6 +101,7 @@ ATOMIC_OPS(xor, xor, i)
* There's two flavors of these: the arithmatic ops have both fetch and return
* versions, while the logical ops only have fetch versions.
*/
+#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
#define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \
static __always_inline \
c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \
@@ -108,6 +126,38 @@ c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \
: "memory"); \
return ret; \
}
+#else
+#define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \
+static __always_inline \
+c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \
+ atomic##prefix##_t *v) \
+{ \
+ register c_type ret, temp; \
+ __asm__ __volatile__ ( \
+ "1: lr." #asm_type " %1, %0\n" \
+ " " #asm_op " %2, %1, %3\n" \
+ " sc." #asm_type " %2, %2, %0\n" \
+ " bnez %2, 1b\n" \
+ : "+A" (v->counter), "=&r" (ret), "=&r" (temp) \
+ : "r" (I) \
+ : "memory"); \
+ return ret; \
+} \
+static __always_inline \
+c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \
+{ \
+ register c_type ret, temp; \
+ __asm__ __volatile__ ( \
+ "1: lr." #asm_type ".aqrl %1, %0\n" \
+ " " #asm_op " %2, %1, %3\n" \
+ " sc." #asm_type ".aqrl %2, %2, %0\n" \
+ " bnez %2, 1b\n" \
+ : "+A" (v->counter), "=&r" (ret), "=&r" (temp) \
+ : "r" (I) \
+ : "memory"); \
+ return ret; \
+}
+#endif
#define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \
static __always_inline \
diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h
index fae152ea0508..b51cb18f7d9e 100644
--- a/arch/riscv/include/asm/bitops.h
+++ b/arch/riscv/include/asm/bitops.h
@@ -187,12 +187,17 @@ static __always_inline int variable_fls(unsigned int x)
#if (BITS_PER_LONG == 64)
#define __AMO(op) "amo" #op ".d"
+#define __LR "lr.d"
+#define __SC "sc.d"
#elif (BITS_PER_LONG == 32)
#define __AMO(op) "amo" #op ".w"
+#define __LR "lr.w"
+#define __SC "sc.w"
#else
#error "Unexpected BITS_PER_LONG"
#endif
+#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
#define __test_and_op_bit_ord(op, mod, nr, addr, ord) \
({ \
unsigned long __res, __mask; \
@@ -211,6 +216,33 @@ static __always_inline int variable_fls(unsigned int x)
: "+A" (addr[BIT_WORD(nr)]) \
: "r" (mod(BIT_MASK(nr))) \
: "memory");
+#else
+#define __test_and_op_bit_ord(op, mod, nr, addr, ord) \
+({ \
+ unsigned long __res, __mask, __temp; \
+ __mask = BIT_MASK(nr); \
+ __asm__ __volatile__ ( \
+ "1: " __LR #ord " %0, %1\n" \
+ #op " %2, %0, %3\n" \
+ __SC #ord " %2, %2, %1\n" \
+ "bnez %2, 1b\n" \
+ : "=&r" (__res), "+A" (addr[BIT_WORD(nr)]), "=&r" (__temp) \
+ : "r" (mod(__mask)) \
+ : "memory"); \
+ ((__res & __mask) != 0); \
+})
+
+#define __op_bit_ord(op, mod, nr, addr, ord) \
+ unsigned long __res, __temp; \
+ __asm__ __volatile__ ( \
+ "1: " __LR #ord " %0, %1\n" \
+ #op " %2, %0, %3\n" \
+ __SC #ord " %2, %2, %1\n" \
+ "bnez %2, 1b\n" \
+ : "=&r" (__res), "+A" (addr[BIT_WORD(nr)]), "=&r" (__temp) \
+ : "r" (mod(BIT_MASK(nr))) \
+ : "memory")
+#endif
#define __test_and_op_bit(op, mod, nr, addr) \
__test_and_op_bit_ord(op, mod, nr, addr, .aqrl)
@@ -354,12 +386,25 @@ static inline void arch___clear_bit_unlock(
static inline bool arch_xor_unlock_is_negative_byte(unsigned long mask,
volatile unsigned long *addr)
{
+#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
unsigned long res;
__asm__ __volatile__ (
__AMO(xor) ".rl %0, %2, %1"
: "=r" (res), "+A" (*addr)
: "r" (__NOP(mask))
: "memory");
+#else
+ unsigned long res, temp;
+
+ __asm__ __volatile__ (
+ "1: " __LR ".rl %0, %1\n"
+ "xor %2, %0, %3\n"
+ __SC ".rl %2, %2, %1\n"
+ "bnez %2, 1b\n"
+ : "=&r" (res), "+A" (*addr), "=&r" (temp)
+ : "r" (__NOP(mask))
+ : "memory");
+#endif
return (res & BIT(7)) != 0;
}
diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h
index 4cadc56220fe..881082b05110 100644
--- a/arch/riscv/include/asm/cmpxchg.h
+++ b/arch/riscv/include/asm/cmpxchg.h
@@ -51,6 +51,7 @@
} \
})
+#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
#define __arch_xchg(sfx, prepend, append, r, p, n) \
({ \
__asm__ __volatile__ ( \
@@ -61,6 +62,21 @@
: "r" (n) \
: "memory"); \
})
+#else
+#define __arch_xchg(sfx, prepend, append, r, p, n) \
+({ \
+ __typeof__(*(__ptr)) temp; \
+ __asm__ __volatile__ ( \
+ prepend \
+ "1: lr" sfx " %0, %1\n" \
+ " sc" sfx " %2, %3, %1\n" \
+ " bnez %2, 1b\n" \
+ append \
+ : "=&r" (r), "+A" (*(p)), "=&r" (temp) \
+ : "r" (n) \
+ : "memory"); \
+})
+#endif
#define _arch_xchg(ptr, new, sc_sfx, swap_sfx, prepend, \
sc_append, swap_append) \
diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h
index fc8130f995c1..47297f47ec35 100644
--- a/arch/riscv/include/asm/futex.h
+++ b/arch/riscv/include/asm/futex.h
@@ -19,6 +19,7 @@
#define __disable_user_access() do { } while (0)
#endif
+#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \
{ \
__enable_user_access(); \
@@ -32,16 +33,39 @@
: "memory"); \
__disable_user_access(); \
}
+#else
+#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \
+{ \
+ __enable_user_access(); \
+ __asm__ __volatile__ ( \
+ "1: lr.w.aqrl %[ov], %[u]\n" \
+ " " insn "\n" \
+ " sc.w.aqrl %[t], %[t], %[u]\n" \
+ " bnez %[t], 1b\n" \
+ "2:\n" \
+ _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %[r]) \
+ : [r] "+r" (ret), [ov] "=&r" (oldval), \
+ [t] "=&r" (temp), [u] "+m" (*uaddr) \
+ : [op] "Jr" (oparg) \
+ : "memory"); \
+ __disable_user_access(); \
+}
+#endif
static inline int
arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
{
+#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
int oldval = 0, ret = 0;
+#else
+ int oldval = 0, ret = 0, temp = 0;
+#endif
if (!access_ok(uaddr, sizeof(u32)))
return -EFAULT;
switch (op) {
+#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
case FUTEX_OP_SET:
__futex_atomic_op("amoswap.w.aqrl %[ov],%z[op],%[u]",
ret, oldval, uaddr, oparg);
@@ -62,6 +86,28 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
__futex_atomic_op("amoxor.w.aqrl %[ov],%z[op],%[u]",
ret, oldval, uaddr, oparg);
break;
+#else
+ case FUTEX_OP_SET:
+ __futex_atomic_op("mv %[t], %z[op]",
+ ret, oldval, uaddr, oparg);
+ break;
+ case FUTEX_OP_ADD:
+ __futex_atomic_op("add %[t], %[ov], %z[op]",
+ ret, oldval, uaddr, oparg);
+ break;
+ case FUTEX_OP_OR:
+ __futex_atomic_op("or %[t], %[ov], %z[op]",
+ ret, oldval, uaddr, oparg);
+ break;
+ case FUTEX_OP_ANDN:
+ __futex_atomic_op("and %[t], %[ov], %z[op]",
+ ret, oldval, uaddr, ~oparg);
+ break;
+ case FUTEX_OP_XOR:
+ __futex_atomic_op("xor %[t], %[ov], %z[op]",
+ ret, oldval, uaddr, oparg);
+ break;
+#endif
default:
ret = -ENOSYS;
}
--
2.25.1Hey, On Fri, Nov 29, 2024 at 03:43:19PM +0100, Aleksandar Rikalo wrote: > From: Chao-ying Fu <cfu@mips.com> > > Use lr and sc to implement all atomic functions. Some CPUs have > native support for lr and sc, but emulate AMO instructions through > trap handlers that are slow. > > Add config RISCV_ISA_ZALRSC_ONLY. > > Signed-off-by: Chao-ying Fu <cfu@mips.com> > Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com> > --- > arch/riscv/Kconfig | 10 ++++++ > arch/riscv/include/asm/atomic.h | 52 +++++++++++++++++++++++++++++++- > arch/riscv/include/asm/bitops.h | 45 +++++++++++++++++++++++++++ > arch/riscv/include/asm/cmpxchg.h | 16 ++++++++++ > arch/riscv/include/asm/futex.h | 46 ++++++++++++++++++++++++++++ > 5 files changed, 168 insertions(+), 1 deletion(-) > > diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig > index cc63aef41e94..767538c27875 100644 > --- a/arch/riscv/Kconfig > +++ b/arch/riscv/Kconfig > @@ -715,6 +715,16 @@ config RISCV_ISA_ZACAS > > If you don't know what to do here, say Y. > > +config RISCV_ISA_ZALRSC_ONLY > + bool "Zalrsc extension support only" I don't agree with the naming of and description for this option. From the description below I'd imagine that you only care about a platform where use of AMO instructions is only undesirable, but overall the option implies that it can be used to build a kernel that can be used on systems that only implement Zalrsc, which, even with your patch applied, it cannot. I think, if we are going to merge something like this, we should go the whole way and permit platforms that don't even emulate the a extension. If not, the Kconfig option should be explicitly clear that the a extension is still mandatory. > + default n "default n" is the default, you shouldn't need this line. > + help > + Use lr and sc to build all atomic functions. Some CPUs have > + native support for lr and sc, but emulate amo instructions through > + trap handlers that are slow. Since you mention trap handlers here, it sounds like it may not be the CPU itself that emulates it, but rather firmware? Cheers, Conor.
On Fri, Nov 29, 2024 at 5:29 PM Conor Dooley <conor@kernel.org> wrote: > On Fri, Nov 29, 2024 at 03:43:19PM +0100, Aleksandar Rikalo wrote: > > --- a/arch/riscv/Kconfig > > +++ b/arch/riscv/Kconfig > > @@ -715,6 +715,16 @@ config RISCV_ISA_ZACAS > > > > If you don't know what to do here, say Y. > > > > +config RISCV_ISA_ZALRSC_ONLY > > + bool "Zalrsc extension support only" > > I don't agree with the naming of and description for this option. > From the description below I'd imagine that you only care about a > platform where use of AMO instructions is only undesirable, but overall > the option implies that it can be used to build a kernel that can be > used on systems that only implement Zalrsc, which, even with your patch > applied, it cannot. I think, if we are going to merge something like this, > we should go the whole way and permit platforms that don't even emulate > the a extension. If not, the Kconfig option should be explicitly clear > that the a extension is still mandatory. > Since you mention trap handlers here, it sounds like it may not be the > CPU itself that emulates it, but rather firmware? Hello, We can forget about emulation for now. This patch only affects the way the kernel implements atomic functions. It's needed for MIPS p8700. I've changed the description in v2: https://lore.kernel.org/linux-riscv/20241225082412.36727-1-arikalo@gmail.com Best Regards, Aleksandar
On Fri, Nov 29, 2024 at 10:43 PM Aleksandar Rikalo <arikalo@gmail.com> wrote:
>
> From: Chao-ying Fu <cfu@mips.com>
>
> Use lr and sc to implement all atomic functions. Some CPUs have
> native support for lr and sc, but emulate AMO instructions through
> trap handlers that are slow.
>
> Add config RISCV_ISA_ZALRSC_ONLY.
>
> Signed-off-by: Chao-ying Fu <cfu@mips.com>
> Signed-off-by: Aleksandar Rikalo <arikalo@gmail.com>
> ---
> arch/riscv/Kconfig | 10 ++++++
> arch/riscv/include/asm/atomic.h | 52 +++++++++++++++++++++++++++++++-
> arch/riscv/include/asm/bitops.h | 45 +++++++++++++++++++++++++++
> arch/riscv/include/asm/cmpxchg.h | 16 ++++++++++
> arch/riscv/include/asm/futex.h | 46 ++++++++++++++++++++++++++++
> 5 files changed, 168 insertions(+), 1 deletion(-)
>
> diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
> index cc63aef41e94..767538c27875 100644
> --- a/arch/riscv/Kconfig
> +++ b/arch/riscv/Kconfig
> @@ -715,6 +715,16 @@ config RISCV_ISA_ZACAS
>
> If you don't know what to do here, say Y.
>
> +config RISCV_ISA_ZALRSC_ONLY
> + bool "Zalrsc extension support only"
> + default n
> + help
> + Use lr and sc to build all atomic functions. Some CPUs have
> + native support for lr and sc, but emulate amo instructions through
> + trap handlers that are slow.
What's the actual hardware?
Is it emulated in m-mode?
Where is the code?
> +
> + If you don't know what to do here, say n.
> +
> config TOOLCHAIN_HAS_ZBB
> bool
> default y
> diff --git a/arch/riscv/include/asm/atomic.h b/arch/riscv/include/asm/atomic.h
> index 5b96c2f61adb..f484babecb9e 100644
> --- a/arch/riscv/include/asm/atomic.h
> +++ b/arch/riscv/include/asm/atomic.h
> @@ -50,6 +50,7 @@ static __always_inline void arch_atomic64_set(atomic64_t *v, s64 i)
> * have the AQ or RL bits set. These don't return anything, so there's only
> * one version to worry about.
> */
> +#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
> #define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
> static __always_inline \
> void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
> @@ -59,7 +60,23 @@ void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
> : "+A" (v->counter) \
> : "r" (I) \
> : "memory"); \
> -} \
> +}
> +#else
> +#define ATOMIC_OP(op, asm_op, I, asm_type, c_type, prefix) \
> +static __always_inline \
> +void arch_atomic##prefix##_##op(c_type i, atomic##prefix##_t *v) \
> +{ \
> + register c_type ret, temp; \
> + __asm__ __volatile__ ( \
> + "1: lr." #asm_type " %1, %0\n" \
> + " " #asm_op " %2, %1, %3\n" \
> + " sc." #asm_type " %2, %2, %0\n" \
> + " bnez %2, 1b\n" \
> + : "+A" (v->counter), "=&r" (ret), "=&r" (temp) \
> + : "r" (I) \
> + : "memory"); \
> +}
> +#endif
>
> #ifdef CONFIG_GENERIC_ATOMIC64
> #define ATOMIC_OPS(op, asm_op, I) \
> @@ -84,6 +101,7 @@ ATOMIC_OPS(xor, xor, i)
> * There's two flavors of these: the arithmatic ops have both fetch and return
> * versions, while the logical ops only have fetch versions.
> */
> +#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
> #define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \
> static __always_inline \
> c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \
> @@ -108,6 +126,38 @@ c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \
> : "memory"); \
> return ret; \
> }
> +#else
> +#define ATOMIC_FETCH_OP(op, asm_op, I, asm_type, c_type, prefix) \
> +static __always_inline \
> +c_type arch_atomic##prefix##_fetch_##op##_relaxed(c_type i, \
> + atomic##prefix##_t *v) \
> +{ \
> + register c_type ret, temp; \
> + __asm__ __volatile__ ( \
> + "1: lr." #asm_type " %1, %0\n" \
> + " " #asm_op " %2, %1, %3\n" \
> + " sc." #asm_type " %2, %2, %0\n" \
> + " bnez %2, 1b\n" \
> + : "+A" (v->counter), "=&r" (ret), "=&r" (temp) \
> + : "r" (I) \
> + : "memory"); \
> + return ret; \
> +} \
> +static __always_inline \
> +c_type arch_atomic##prefix##_fetch_##op(c_type i, atomic##prefix##_t *v) \
> +{ \
> + register c_type ret, temp; \
> + __asm__ __volatile__ ( \
> + "1: lr." #asm_type ".aqrl %1, %0\n" \
> + " " #asm_op " %2, %1, %3\n" \
> + " sc." #asm_type ".aqrl %2, %2, %0\n" \
> + " bnez %2, 1b\n" \
> + : "+A" (v->counter), "=&r" (ret), "=&r" (temp) \
> + : "r" (I) \
> + : "memory"); \
> + return ret; \
> +}
> +#endif
>
> #define ATOMIC_OP_RETURN(op, asm_op, c_op, I, asm_type, c_type, prefix) \
> static __always_inline \
> diff --git a/arch/riscv/include/asm/bitops.h b/arch/riscv/include/asm/bitops.h
> index fae152ea0508..b51cb18f7d9e 100644
> --- a/arch/riscv/include/asm/bitops.h
> +++ b/arch/riscv/include/asm/bitops.h
> @@ -187,12 +187,17 @@ static __always_inline int variable_fls(unsigned int x)
>
> #if (BITS_PER_LONG == 64)
> #define __AMO(op) "amo" #op ".d"
> +#define __LR "lr.d"
> +#define __SC "sc.d"
> #elif (BITS_PER_LONG == 32)
> #define __AMO(op) "amo" #op ".w"
> +#define __LR "lr.w"
> +#define __SC "sc.w"
> #else
> #error "Unexpected BITS_PER_LONG"
> #endif
>
> +#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
> #define __test_and_op_bit_ord(op, mod, nr, addr, ord) \
> ({ \
> unsigned long __res, __mask; \
> @@ -211,6 +216,33 @@ static __always_inline int variable_fls(unsigned int x)
> : "+A" (addr[BIT_WORD(nr)]) \
> : "r" (mod(BIT_MASK(nr))) \
> : "memory");
> +#else
> +#define __test_and_op_bit_ord(op, mod, nr, addr, ord) \
> +({ \
> + unsigned long __res, __mask, __temp; \
> + __mask = BIT_MASK(nr); \
> + __asm__ __volatile__ ( \
> + "1: " __LR #ord " %0, %1\n" \
> + #op " %2, %0, %3\n" \
> + __SC #ord " %2, %2, %1\n" \
> + "bnez %2, 1b\n" \
> + : "=&r" (__res), "+A" (addr[BIT_WORD(nr)]), "=&r" (__temp) \
> + : "r" (mod(__mask)) \
> + : "memory"); \
> + ((__res & __mask) != 0); \
> +})
> +
> +#define __op_bit_ord(op, mod, nr, addr, ord) \
> + unsigned long __res, __temp; \
> + __asm__ __volatile__ ( \
> + "1: " __LR #ord " %0, %1\n" \
> + #op " %2, %0, %3\n" \
> + __SC #ord " %2, %2, %1\n" \
> + "bnez %2, 1b\n" \
> + : "=&r" (__res), "+A" (addr[BIT_WORD(nr)]), "=&r" (__temp) \
> + : "r" (mod(BIT_MASK(nr))) \
> + : "memory")
> +#endif
>
> #define __test_and_op_bit(op, mod, nr, addr) \
> __test_and_op_bit_ord(op, mod, nr, addr, .aqrl)
> @@ -354,12 +386,25 @@ static inline void arch___clear_bit_unlock(
> static inline bool arch_xor_unlock_is_negative_byte(unsigned long mask,
> volatile unsigned long *addr)
> {
> +#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
> unsigned long res;
> __asm__ __volatile__ (
> __AMO(xor) ".rl %0, %2, %1"
> : "=r" (res), "+A" (*addr)
> : "r" (__NOP(mask))
> : "memory");
> +#else
> + unsigned long res, temp;
> +
> + __asm__ __volatile__ (
> + "1: " __LR ".rl %0, %1\n"
> + "xor %2, %0, %3\n"
> + __SC ".rl %2, %2, %1\n"
> + "bnez %2, 1b\n"
> + : "=&r" (res), "+A" (*addr), "=&r" (temp)
> + : "r" (__NOP(mask))
> + : "memory");
> +#endif
> return (res & BIT(7)) != 0;
> }
>
> diff --git a/arch/riscv/include/asm/cmpxchg.h b/arch/riscv/include/asm/cmpxchg.h
> index 4cadc56220fe..881082b05110 100644
> --- a/arch/riscv/include/asm/cmpxchg.h
> +++ b/arch/riscv/include/asm/cmpxchg.h
> @@ -51,6 +51,7 @@
> } \
> })
>
> +#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
> #define __arch_xchg(sfx, prepend, append, r, p, n) \
> ({ \
> __asm__ __volatile__ ( \
> @@ -61,6 +62,21 @@
> : "r" (n) \
> : "memory"); \
> })
> +#else
> +#define __arch_xchg(sfx, prepend, append, r, p, n) \
> +({ \
> + __typeof__(*(__ptr)) temp; \
> + __asm__ __volatile__ ( \
> + prepend \
> + "1: lr" sfx " %0, %1\n" \
> + " sc" sfx " %2, %3, %1\n" \
> + " bnez %2, 1b\n" \
> + append \
> + : "=&r" (r), "+A" (*(p)), "=&r" (temp) \
> + : "r" (n) \
> + : "memory"); \
> +})
> +#endif
>
> #define _arch_xchg(ptr, new, sc_sfx, swap_sfx, prepend, \
> sc_append, swap_append) \
> diff --git a/arch/riscv/include/asm/futex.h b/arch/riscv/include/asm/futex.h
> index fc8130f995c1..47297f47ec35 100644
> --- a/arch/riscv/include/asm/futex.h
> +++ b/arch/riscv/include/asm/futex.h
> @@ -19,6 +19,7 @@
> #define __disable_user_access() do { } while (0)
> #endif
>
> +#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
> #define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \
> { \
> __enable_user_access(); \
> @@ -32,16 +33,39 @@
> : "memory"); \
> __disable_user_access(); \
> }
> +#else
> +#define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \
> +{ \
> + __enable_user_access(); \
> + __asm__ __volatile__ ( \
> + "1: lr.w.aqrl %[ov], %[u]\n" \
> + " " insn "\n" \
> + " sc.w.aqrl %[t], %[t], %[u]\n" \
> + " bnez %[t], 1b\n" \
> + "2:\n" \
> + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %[r]) \
> + : [r] "+r" (ret), [ov] "=&r" (oldval), \
> + [t] "=&r" (temp), [u] "+m" (*uaddr) \
> + : [op] "Jr" (oparg) \
> + : "memory"); \
> + __disable_user_access(); \
> +}
> +#endif
>
> static inline int
> arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
> {
> +#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
> int oldval = 0, ret = 0;
> +#else
> + int oldval = 0, ret = 0, temp = 0;
> +#endif
>
> if (!access_ok(uaddr, sizeof(u32)))
> return -EFAULT;
>
> switch (op) {
> +#ifndef CONFIG_RISCV_ISA_ZALRSC_ONLY
> case FUTEX_OP_SET:
> __futex_atomic_op("amoswap.w.aqrl %[ov],%z[op],%[u]",
> ret, oldval, uaddr, oparg);
> @@ -62,6 +86,28 @@ arch_futex_atomic_op_inuser(int op, int oparg, int *oval, u32 __user *uaddr)
> __futex_atomic_op("amoxor.w.aqrl %[ov],%z[op],%[u]",
> ret, oldval, uaddr, oparg);
> break;
> +#else
> + case FUTEX_OP_SET:
> + __futex_atomic_op("mv %[t], %z[op]",
> + ret, oldval, uaddr, oparg);
> + break;
> + case FUTEX_OP_ADD:
> + __futex_atomic_op("add %[t], %[ov], %z[op]",
> + ret, oldval, uaddr, oparg);
> + break;
> + case FUTEX_OP_OR:
> + __futex_atomic_op("or %[t], %[ov], %z[op]",
> + ret, oldval, uaddr, oparg);
> + break;
> + case FUTEX_OP_ANDN:
> + __futex_atomic_op("and %[t], %[ov], %z[op]",
> + ret, oldval, uaddr, ~oparg);
> + break;
> + case FUTEX_OP_XOR:
> + __futex_atomic_op("xor %[t], %[ov], %z[op]",
> + ret, oldval, uaddr, oparg);
> + break;
> +#endif
> default:
> ret = -ENOSYS;
> }
> --
> 2.25.1
>
--
Best Regards
Guo Ren
© 2016 - 2026 Red Hat, Inc.