From nobody Tue Apr 7 16:16:01 2026 Received: from foss.arm.com (foss.arm.com [217.140.110.172]) by smtp.subspace.kernel.org (Postfix) with ESMTP id 75BB13FE663; Thu, 12 Mar 2026 17:53:02 +0000 (UTC) Authentication-Results: smtp.subspace.kernel.org; arc=none smtp.client-ip=217.140.110.172 ARC-Seal: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1773337988; cv=none; b=BGn0UOkJFtOO7eivBMY6ikYmQP6Vc2+4zpQP3u4Kwt1K2Qb0X11wOyCbDGD231/diCMRas9lCmCBiicN6p9N+gmCglgFYMKsTabDqH/QD0u9Eh/A/Tb0gMmxBmIgtDe+n8TO7l0+ZxGuJR3dGtGoH2Qs4zgukSu3bTo1cRqoBUQ= ARC-Message-Signature: i=1; a=rsa-sha256; d=subspace.kernel.org; s=arc-20240116; t=1773337988; c=relaxed/simple; bh=kIdLG9FEnLu3C0RYRmys5GQd3VdKVOfGPpAoHSJA1VE=; h=From:To:Cc:Subject:Date:Message-Id:In-Reply-To:References: MIME-Version; b=hrCGh/UsziySS1ke2YieK5B2kKN/F7ZsDjdIb7luwxnxfFIpf8WlEjJGR5nbkYyi+mvzBzHKRWedAo/iMv7KPimwmmhCg+mOLv9zfsvTcLd12VGco+vZTIFes5Q4mTTgJ6uLctcIXYa7BfSl9IJoneo6WNvvqMfJ52Yoy06cL9s= ARC-Authentication-Results: i=1; smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com; spf=pass smtp.mailfrom=arm.com; arc=none smtp.client-ip=217.140.110.172 Authentication-Results: smtp.subspace.kernel.org; dmarc=pass (p=none dis=none) header.from=arm.com Authentication-Results: smtp.subspace.kernel.org; spf=pass smtp.mailfrom=arm.com Received: from usa-sjc-imap-foss1.foss.arm.com (unknown [10.121.207.14]) by usa-sjc-mx-foss1.foss.arm.com (Postfix) with ESMTP id C68EB1A9A; Thu, 12 Mar 2026 10:52:55 -0700 (PDT) Received: from e129823.cambridge.arm.com (e129823.arm.com [10.1.197.6]) by usa-sjc-imap-foss1.foss.arm.com (Postfix) with ESMTPA id 746553F73B; Thu, 12 Mar 2026 10:52:59 -0700 (PDT) From: Yeoreum Yun To: linux-arm-kernel@lists.infradead.org, linux-kernel@vger.kernel.org, kvmarm@lists.linux.dev, kvm@vger.kernel.org, linux-kselftest@vger.kernel.org Cc: catalin.marinas@arm.com, will@kernel.org, maz@kernel.org, oupton@kernel.org, miko.lenczewski@arm.com, kevin.brodsky@arm.com, broonie@kernel.org, ardb@kernel.org, suzuki.poulose@arm.com, lpieralisi@kernel.org, joey.gouly@arm.com, yuzenghui@huawei.com, yeoreum.yun@arm.com Subject: [PATCH v16 5/8] arm64: futex: support futex with FEAT_LSUI Date: Thu, 12 Mar 2026 17:52:40 +0000 Message-Id: <20260312175243.1593864-6-yeoreum.yun@arm.com> X-Mailer: git-send-email 2.34.1 In-Reply-To: <20260312175243.1593864-1-yeoreum.yun@arm.com> References: <20260312175243.1593864-1-yeoreum.yun@arm.com> Precedence: bulk X-Mailing-List: linux-kernel@vger.kernel.org List-Id: List-Subscribe: List-Unsubscribe: MIME-Version: 1.0 Content-Transfer-Encoding: quoted-printable Content-Type: text/plain; charset="utf-8" Current futex atomic operations are implemented with ll/sc instructions and clearing PSTATE.PAN. Since Armv9.6, FEAT_LSUI supplies not only load/store instructions but also atomic operation for user memory access in kernel it doesn't need to clear PSTATE.PAN bit anymore. With theses instructions some of futex atomic operations don't need to be implmented with ldxr/stlxr pair instead can be implmented with one atomic operation supplied by FEAT_LSUI and don't enable mto like ldtr*/sttr* instructions usage. However, some of futex atomic operation don't have matched instructuion i.e) eor or cmpxchg with word size. For those operation, uses cas{al}t to implement them. Signed-off-by: Yeoreum Yun --- arch/arm64/include/asm/futex.h | 168 ++++++++++++++++++++++++++++++++- arch/arm64/include/asm/lsui.h | 27 ++++++ 2 files changed, 192 insertions(+), 3 deletions(-) create mode 100644 arch/arm64/include/asm/lsui.h diff --git a/arch/arm64/include/asm/futex.h b/arch/arm64/include/asm/futex.h index ba6a19de7823..20ee0b9b2546 100644 --- a/arch/arm64/include/asm/futex.h +++ b/arch/arm64/include/asm/futex.h @@ -7,9 +7,9 @@ =20 #include #include -#include =20 #include +#include =20 #define FUTEX_MAX_LOOPS 128 /* What's the largest number you can think of?= */ =20 @@ -90,11 +90,173 @@ __llsc_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u3= 2 newval, u32 *oval) return ret; } =20 +#ifdef CONFIG_ARM64_LSUI + +/* + * FEAT_LSUI is supported since Armv9.6, where FEAT_PAN is mandatory. + * However, this assumption may not always hold: + * + * - Some CPUs advertise FEAT_LSUI but lack FEAT_PAN. + * - Virtualisation or ID register overrides may expose invalid + * feature combinations. + * + * Rather than disabling FEAT_LSUI when FEAT_PAN is absent, wrap LSUI + * instructions with uaccess_ttbr0_enable()/disable() when + * ARM64_SW_TTBR0_PAN is enabled. + */ + +#define LSUI_FUTEX_ATOMIC_OP(op, asm_op) \ +static __always_inline int \ +__lsui_futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ +{ \ + int ret =3D 0; \ + int oldval; \ + \ + uaccess_ttbr0_enable(); \ + \ + asm volatile("// __lsui_futex_atomic_" #op "\n" \ + __LSUI_PREAMBLE \ +"1: " #asm_op "al %w[oparg], %w[oldval], %[uaddr]\n" \ +"2:\n" \ + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) \ + : [ret] "+r" (ret), [uaddr] "+Q" (*uaddr), \ + [oldval] "=3Dr" (oldval) \ + : [oparg] "r" (oparg) \ + : "memory"); \ + \ + uaccess_ttbr0_disable(); \ + \ + if (!ret) \ + *oval =3D oldval; \ + return ret; \ +} + +LSUI_FUTEX_ATOMIC_OP(add, ldtadd) +LSUI_FUTEX_ATOMIC_OP(or, ldtset) +LSUI_FUTEX_ATOMIC_OP(andnot, ldtclr) +LSUI_FUTEX_ATOMIC_OP(set, swpt) + +static __always_inline int +__lsui_cmpxchg64(u64 __user *uaddr, u64 *oldval, u64 newval) +{ + int ret =3D 0; + + uaccess_ttbr0_enable(); + + asm volatile("// __lsui_cmpxchg64\n" + __LSUI_PREAMBLE +"1: casalt %[oldval], %[newval], %[uaddr]\n" +"2:\n" + _ASM_EXTABLE_UACCESS_ERR(1b, 2b, %w[ret]) + : [ret] "+r" (ret), [uaddr] "+Q" (*uaddr), + [oldval] "+r" (*oldval) + : [newval] "r" (newval) + : "memory"); + + uaccess_ttbr0_disable(); + + return ret; +} + +static __always_inline int +__lsui_cmpxchg32(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + u64 __user *uaddr64; + bool futex_pos, other_pos; + int ret, i; + u32 other, orig_other; + union { + u32 futex[2]; + u64 raw; + } oval64, orig64, nval64; + + uaddr64 =3D (u64 __user *) PTR_ALIGN_DOWN(uaddr, sizeof(u64)); + futex_pos =3D !IS_ALIGNED((unsigned long)uaddr, sizeof(u64)); + other_pos =3D !futex_pos; + + oval64.futex[futex_pos] =3D oldval; + ret =3D get_user(oval64.futex[other_pos], (u32 __user *)uaddr64 + other_p= os); + if (ret) + return -EFAULT; + + ret =3D -EAGAIN; + for (i =3D 0; i < FUTEX_MAX_LOOPS; i++) { + orig64.raw =3D nval64.raw =3D oval64.raw; + + nval64.futex[futex_pos] =3D newval; + + if (__lsui_cmpxchg64(uaddr64, &oval64.raw, nval64.raw)) + return -EFAULT; + + oldval =3D oval64.futex[futex_pos]; + other =3D oval64.futex[other_pos]; + orig_other =3D orig64.futex[other_pos]; + + if (other =3D=3D orig_other) { + ret =3D 0; + break; + } + } + + if (!ret) + *oval =3D oldval; + + return ret; +} + +static __always_inline int +__lsui_futex_atomic_and(int oparg, u32 __user *uaddr, int *oval) +{ + /* + * Undo the bitwise negation applied to the oparg passed from + * arch_futex_atomic_op_inuser() with FUTEX_OP_ANDN. + */ + return __lsui_futex_atomic_andnot(~oparg, uaddr, oval); +} + +static __always_inline int +__lsui_futex_atomic_eor(int oparg, u32 __user *uaddr, int *oval) +{ + u32 oldval, newval, val; + int ret, i; + + if (get_user(oldval, uaddr)) + return -EFAULT; + + /* + * there are no ldteor/stteor instructions... + */ + for (i =3D 0; i < FUTEX_MAX_LOOPS; i++) { + newval =3D oldval ^ oparg; + + ret =3D __lsui_cmpxchg32(uaddr, oldval, newval, &val); + if (ret) + return ret; + + if (val =3D=3D oldval) { + *oval =3D val; + return 0; + } + + oldval =3D val; + } + + return -EAGAIN; +} + +static __always_inline int +__lsui_futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) +{ + return __lsui_cmpxchg32(uaddr, oldval, newval, oval); +} +#endif /* CONFIG_ARM64_LSUI */ + + #define FUTEX_ATOMIC_OP(op) \ static __always_inline int \ __futex_atomic_##op(int oparg, u32 __user *uaddr, int *oval) \ { \ - return __llsc_futex_atomic_##op(oparg, uaddr, oval); \ + return __lsui_llsc_body(futex_atomic_##op, oparg, uaddr, oval); \ } =20 FUTEX_ATOMIC_OP(add) @@ -106,7 +268,7 @@ FUTEX_ATOMIC_OP(set) static __always_inline int __futex_cmpxchg(u32 __user *uaddr, u32 oldval, u32 newval, u32 *oval) { - return __llsc_futex_cmpxchg(uaddr, oldval, newval, oval); + return __lsui_llsc_body(futex_cmpxchg, uaddr, oldval, newval, oval); } =20 static inline int diff --git a/arch/arm64/include/asm/lsui.h b/arch/arm64/include/asm/lsui.h new file mode 100644 index 000000000000..8f0d81953eb6 --- /dev/null +++ b/arch/arm64/include/asm/lsui.h @@ -0,0 +1,27 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_LSUI_H +#define __ASM_LSUI_H + +#include +#include +#include +#include +#include + +#define __LSUI_PREAMBLE ".arch_extension lsui\n" + +#ifdef CONFIG_ARM64_LSUI + +#define __lsui_llsc_body(op, ...) \ +({ \ + alternative_has_cap_unlikely(ARM64_HAS_LSUI) ? \ + __lsui_##op(__VA_ARGS__) : __llsc_##op(__VA_ARGS__); \ +}) + +#else /* CONFIG_ARM64_LSUI */ + +#define __lsui_llsc_body(op, ...) __llsc_##op(__VA_ARGS__) + +#endif /* CONFIG_ARM64_LSUI */ + +#endif /* __ASM_LSUI_H */ --=20 LEVI:{C3F47F37-75D8-414A-A8BA-3980EC8A46D7}